blob: 7b99e36b897370d64240791a73024c706d1c8d2e [file] [log] [blame]
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001/*
2 * ring buffer based function tracer
3 *
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
Nadia Yvette Chambers6d49e352012-12-06 10:39:54 +010012 * Copyright (C) 2004 Nadia Yvette Chambers
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020013 */
Steven Rostedt2cadf912008-12-01 22:20:19 -050014#include <linux/ring_buffer.h>
Sam Ravnborg273b2812009-10-18 00:52:28 +020015#include <generated/utsrelease.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050016#include <linux/stacktrace.h>
17#include <linux/writeback.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020018#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040020#include <linux/notifier.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050021#include <linux/irqflags.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020022#include <linux/debugfs.h>
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -050023#include <linux/tracefs.h>
Steven Rostedt4c11d7a2008-05-12 21:20:43 +020024#include <linux/pagemap.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020025#include <linux/hardirq.h>
26#include <linux/linkage.h>
27#include <linux/uaccess.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050028#include <linux/kprobes.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020029#include <linux/ftrace.h>
30#include <linux/module.h>
31#include <linux/percpu.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050032#include <linux/splice.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040033#include <linux/kdebug.h>
Frederic Weisbecker5f0c6c02009-03-27 14:22:10 +010034#include <linux/string.h>
Steven Rostedt (Red Hat)f76180b2015-01-20 15:48:46 -050035#include <linux/mount.h>
Lai Jiangshan7e53bd42010-01-06 20:08:50 +080036#include <linux/rwsem.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090037#include <linux/slab.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020038#include <linux/ctype.h>
39#include <linux/init.h>
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +020040#include <linux/poll.h>
Steven Rostedtb892e5c2012-03-01 22:06:48 -050041#include <linux/nmi.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020042#include <linux/fs.h>
Clark Williams8bd75c72013-02-07 09:47:07 -060043#include <linux/sched/rt.h>
Ingo Molnar86387f72008-05-12 21:20:51 +020044
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020045#include "trace.h"
Steven Rostedtf0868d12008-12-23 23:24:12 -050046#include "trace_output.h"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020047
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010048/*
Steven Rostedt73c51622009-03-11 13:42:01 -040049 * On boot up, the ring buffer is set to the minimum size, so that
50 * we do not waste memory on systems that are not using tracing.
51 */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -050052bool ring_buffer_expanded;
Steven Rostedt73c51622009-03-11 13:42:01 -040053
54/*
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010055 * We need to change this state when a selftest is running.
Frederic Weisbeckerff325042008-12-04 23:47:35 +010056 * A selftest will lurk into the ring-buffer to count the
57 * entries inserted during the selftest although some concurrent
Ingo Molnar5e1607a2009-03-05 10:24:48 +010058 * insertions into the ring-buffer such as trace_printk could occurred
Frederic Weisbeckerff325042008-12-04 23:47:35 +010059 * at the same time, giving false positive or negative results.
60 */
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010061static bool __read_mostly tracing_selftest_running;
Frederic Weisbeckerff325042008-12-04 23:47:35 +010062
Steven Rostedtb2821ae2009-02-02 21:38:32 -050063/*
64 * If a tracer is running, we do not want to run SELFTEST.
65 */
Li Zefan020e5f82009-07-01 10:47:05 +080066bool __read_mostly tracing_selftest_disabled;
Steven Rostedtb2821ae2009-02-02 21:38:32 -050067
Steven Rostedt (Red Hat)0daa23022014-12-12 22:27:10 -050068/* Pipe tracepoints to printk */
69struct trace_iterator *tracepoint_print_iter;
70int tracepoint_printk;
71
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +010072/* For tracers that don't implement custom flags */
73static struct tracer_opt dummy_tracer_opt[] = {
74 { }
75};
76
77static struct tracer_flags dummy_tracer_flags = {
78 .val = 0,
79 .opts = dummy_tracer_opt
80};
81
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -050082static int
83dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +010084{
85 return 0;
86}
Steven Rostedt0f048702008-11-05 16:05:44 -050087
88/*
Steven Rostedt7ffbd482012-10-11 12:14:25 -040089 * To prevent the comm cache from being overwritten when no
90 * tracing is active, only save the comm when a trace event
91 * occurred.
92 */
93static DEFINE_PER_CPU(bool, trace_cmdline_save);
94
95/*
Steven Rostedt0f048702008-11-05 16:05:44 -050096 * Kill all tracing for good (never come back).
97 * It is initialized to 1 but will turn to zero if the initialization
98 * of the tracer is successful. But that is the only place that sets
99 * this back to zero.
100 */
Hannes Eder4fd27352009-02-10 19:44:12 +0100101static int tracing_disabled = 1;
Steven Rostedt0f048702008-11-05 16:05:44 -0500102
Christoph Lameter9288f992009-10-07 19:17:45 -0400103DEFINE_PER_CPU(int, ftrace_cpu_disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -0400104
Jason Wessel955b61e2010-08-05 09:22:23 -0500105cpumask_var_t __read_mostly tracing_buffer_mask;
Steven Rostedtab464282008-05-12 21:21:00 +0200106
Steven Rostedt944ac422008-10-23 19:26:08 -0400107/*
108 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
109 *
110 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
111 * is set, then ftrace_dump is called. This will output the contents
112 * of the ftrace buffers to the console. This is very useful for
113 * capturing traces that lead to crashes and outputing it to a
114 * serial console.
115 *
116 * It is default off, but you can enable it with either specifying
117 * "ftrace_dump_on_oops" in the kernel command line, or setting
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200118 * /proc/sys/kernel/ftrace_dump_on_oops
119 * Set 1 if you want to dump buffers of all CPUs
120 * Set 2 if you want to dump the buffer of the CPU that triggered oops
Steven Rostedt944ac422008-10-23 19:26:08 -0400121 */
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200122
123enum ftrace_dump_mode ftrace_dump_on_oops;
Steven Rostedt944ac422008-10-23 19:26:08 -0400124
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400125/* When set, tracing will stop when a WARN*() is hit */
126int __disable_trace_on_warning;
127
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -0400128#ifdef CONFIG_TRACE_ENUM_MAP_FILE
129/* Map of enums to their values, for "enum_map" file */
130struct trace_enum_map_head {
131 struct module *mod;
132 unsigned long length;
133};
134
135union trace_enum_map_item;
136
137struct trace_enum_map_tail {
138 /*
139 * "end" is first and points to NULL as it must be different
140 * than "mod" or "enum_string"
141 */
142 union trace_enum_map_item *next;
143 const char *end; /* points to NULL */
144};
145
146static DEFINE_MUTEX(trace_enum_mutex);
147
148/*
149 * The trace_enum_maps are saved in an array with two extra elements,
150 * one at the beginning, and one at the end. The beginning item contains
151 * the count of the saved maps (head.length), and the module they
152 * belong to if not built in (head.mod). The ending item contains a
153 * pointer to the next array of saved enum_map items.
154 */
155union trace_enum_map_item {
156 struct trace_enum_map map;
157 struct trace_enum_map_head head;
158 struct trace_enum_map_tail tail;
159};
160
161static union trace_enum_map_item *trace_enum_maps;
162#endif /* CONFIG_TRACE_ENUM_MAP_FILE */
163
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -0500164static int tracing_set_tracer(struct trace_array *tr, const char *buf);
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500165
Li Zefanee6c2c12009-09-18 14:06:47 +0800166#define MAX_TRACER_SIZE 100
167static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500168static char *default_bootup_tracer;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100169
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500170static bool allocate_snapshot;
171
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200172static int __init set_cmdline_ftrace(char *str)
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100173{
Chen Gang67012ab2013-04-08 12:06:44 +0800174 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500175 default_bootup_tracer = bootup_tracer_buf;
Steven Rostedt73c51622009-03-11 13:42:01 -0400176 /* We are using ftrace early, expand it */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500177 ring_buffer_expanded = true;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100178 return 1;
179}
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200180__setup("ftrace=", set_cmdline_ftrace);
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100181
Steven Rostedt944ac422008-10-23 19:26:08 -0400182static int __init set_ftrace_dump_on_oops(char *str)
183{
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200184 if (*str++ != '=' || !*str) {
185 ftrace_dump_on_oops = DUMP_ALL;
186 return 1;
187 }
188
189 if (!strcmp("orig_cpu", str)) {
190 ftrace_dump_on_oops = DUMP_ORIG;
191 return 1;
192 }
193
194 return 0;
Steven Rostedt944ac422008-10-23 19:26:08 -0400195}
196__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
Steven Rostedt60a11772008-05-12 21:20:44 +0200197
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400198static int __init stop_trace_on_warning(char *str)
199{
Luis Claudio R. Goncalves933ff9f2014-11-12 21:14:00 -0200200 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
201 __disable_trace_on_warning = 1;
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400202 return 1;
203}
Luis Claudio R. Goncalves933ff9f2014-11-12 21:14:00 -0200204__setup("traceoff_on_warning", stop_trace_on_warning);
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400205
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400206static int __init boot_alloc_snapshot(char *str)
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500207{
208 allocate_snapshot = true;
209 /* We also need the main ring buffer expanded */
210 ring_buffer_expanded = true;
211 return 1;
212}
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400213__setup("alloc_snapshot", boot_alloc_snapshot);
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500214
Steven Rostedt7bcfaf52012-11-01 22:56:07 -0400215
216static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
217static char *trace_boot_options __initdata;
218
219static int __init set_trace_boot_options(char *str)
220{
Chen Gang67012ab2013-04-08 12:06:44 +0800221 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
Steven Rostedt7bcfaf52012-11-01 22:56:07 -0400222 trace_boot_options = trace_boot_options_buf;
223 return 0;
224}
225__setup("trace_options=", set_trace_boot_options);
226
Steven Rostedte1e232c2014-02-10 23:38:46 -0500227static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
228static char *trace_boot_clock __initdata;
229
230static int __init set_trace_boot_clock(char *str)
231{
232 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
233 trace_boot_clock = trace_boot_clock_buf;
234 return 0;
235}
236__setup("trace_clock=", set_trace_boot_clock);
237
Steven Rostedt (Red Hat)0daa23022014-12-12 22:27:10 -0500238static int __init set_tracepoint_printk(char *str)
239{
240 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
241 tracepoint_printk = 1;
242 return 1;
243}
244__setup("tp_printk", set_tracepoint_printk);
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400245
Lai Jiangshancf8e3472009-03-30 13:48:00 +0800246unsigned long long ns2usecs(cycle_t nsec)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200247{
248 nsec += 500;
249 do_div(nsec, 1000);
250 return nsec;
251}
252
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -0400253/* trace_flags holds trace_options default values */
254#define TRACE_DEFAULT_FLAGS \
255 (FUNCTION_DEFAULT_FLAGS | \
256 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
257 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
258 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
259 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
260
Steven Rostedt (Red Hat)16270142015-09-30 12:30:06 -0400261/* trace_options that are only supported by global_trace */
262#define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
263 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
264
265
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200266/*
267 * The global_trace is the descriptor that holds the tracing
268 * buffers for the live tracing. For each CPU, it contains
269 * a link list of pages that will store trace entries. The
270 * page descriptor of the pages in the memory is used to hold
271 * the link list by linking the lru item in the page descriptor
272 * to each of the pages in the buffer per CPU.
273 *
274 * For each active CPU there is a data field that holds the
275 * pages for the buffer for that CPU. Each CPU has the same number
276 * of pages allocated for its buffer.
277 */
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -0400278static struct trace_array global_trace = {
279 .trace_flags = TRACE_DEFAULT_FLAGS,
280};
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200281
Steven Rostedtae63b312012-05-03 23:09:03 -0400282LIST_HEAD(ftrace_trace_arrays);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200283
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -0400284int trace_array_get(struct trace_array *this_tr)
285{
286 struct trace_array *tr;
287 int ret = -ENODEV;
288
289 mutex_lock(&trace_types_lock);
290 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
291 if (tr == this_tr) {
292 tr->ref++;
293 ret = 0;
294 break;
295 }
296 }
297 mutex_unlock(&trace_types_lock);
298
299 return ret;
300}
301
302static void __trace_array_put(struct trace_array *this_tr)
303{
304 WARN_ON(!this_tr->ref);
305 this_tr->ref--;
306}
307
308void trace_array_put(struct trace_array *this_tr)
309{
310 mutex_lock(&trace_types_lock);
311 __trace_array_put(this_tr);
312 mutex_unlock(&trace_types_lock);
313}
314
Steven Rostedt (Red Hat)7f1d2f82015-05-05 10:09:53 -0400315int filter_check_discard(struct trace_event_file *file, void *rec,
Tom Zanussif306cc82013-10-24 08:34:17 -0500316 struct ring_buffer *buffer,
317 struct ring_buffer_event *event)
Tom Zanussieb02ce02009-04-08 03:15:54 -0500318{
Steven Rostedt (Red Hat)5d6ad962015-05-13 15:12:33 -0400319 if (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
Tom Zanussif306cc82013-10-24 08:34:17 -0500320 !filter_match_preds(file->filter, rec)) {
321 ring_buffer_discard_commit(buffer, event);
322 return 1;
323 }
324
325 return 0;
Tom Zanussieb02ce02009-04-08 03:15:54 -0500326}
Tom Zanussif306cc82013-10-24 08:34:17 -0500327EXPORT_SYMBOL_GPL(filter_check_discard);
328
Steven Rostedt (Red Hat)2425bcb2015-05-05 11:45:27 -0400329int call_filter_check_discard(struct trace_event_call *call, void *rec,
Tom Zanussif306cc82013-10-24 08:34:17 -0500330 struct ring_buffer *buffer,
331 struct ring_buffer_event *event)
332{
333 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
334 !filter_match_preds(call->filter, rec)) {
335 ring_buffer_discard_commit(buffer, event);
336 return 1;
337 }
338
339 return 0;
340}
341EXPORT_SYMBOL_GPL(call_filter_check_discard);
Tom Zanussieb02ce02009-04-08 03:15:54 -0500342
Fabian Frederickad1438a2014-04-17 21:44:42 +0200343static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
Steven Rostedt37886f62009-03-17 17:22:06 -0400344{
345 u64 ts;
346
347 /* Early boot up does not have a buffer yet */
Alexander Z Lam94571582013-08-02 18:36:16 -0700348 if (!buf->buffer)
Steven Rostedt37886f62009-03-17 17:22:06 -0400349 return trace_clock_local();
350
Alexander Z Lam94571582013-08-02 18:36:16 -0700351 ts = ring_buffer_time_stamp(buf->buffer, cpu);
352 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
Steven Rostedt37886f62009-03-17 17:22:06 -0400353
354 return ts;
355}
356
Alexander Z Lam94571582013-08-02 18:36:16 -0700357cycle_t ftrace_now(int cpu)
358{
359 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
360}
361
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400362/**
363 * tracing_is_enabled - Show if global_trace has been disabled
364 *
365 * Shows if the global trace has been enabled or not. It uses the
366 * mirror flag "buffer_disabled" to be used in fast paths such as for
367 * the irqsoff tracer. But it may be inaccurate due to races. If you
368 * need to know the accurate state, use tracing_is_on() which is a little
369 * slower, but accurate.
370 */
Steven Rostedt90369902008-11-05 16:05:44 -0500371int tracing_is_enabled(void)
372{
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400373 /*
374 * For quick access (irqsoff uses this in fast path), just
375 * return the mirror variable of the state of the ring buffer.
376 * It's a little racy, but we don't really care.
377 */
378 smp_rmb();
379 return !global_trace.buffer_disabled;
Steven Rostedt90369902008-11-05 16:05:44 -0500380}
381
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200382/*
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400383 * trace_buf_size is the size in bytes that is allocated
384 * for a buffer. Note, the number of bytes is always rounded
385 * to page size.
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400386 *
387 * This number is purposely set to a low number of 16384.
388 * If the dump on oops happens, it will be much appreciated
389 * to not have to wait for all that output. Anyway this can be
390 * boot time and run time configurable.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200391 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400392#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400393
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400394static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200395
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200396/* trace_types holds a link list of available tracers. */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200397static struct tracer *trace_types __read_mostly;
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200398
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200399/*
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200400 * trace_types_lock is used to protect the trace_types list.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200401 */
Alexander Z Lama8227412013-07-01 19:37:54 -0700402DEFINE_MUTEX(trace_types_lock);
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200403
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800404/*
405 * serialize the access of the ring buffer
406 *
407 * ring buffer serializes readers, but it is low level protection.
408 * The validity of the events (which returns by ring_buffer_peek() ..etc)
409 * are not protected by ring buffer.
410 *
411 * The content of events may become garbage if we allow other process consumes
412 * these events concurrently:
413 * A) the page of the consumed events may become a normal page
414 * (not reader page) in ring buffer, and this page will be rewrited
415 * by events producer.
416 * B) The page of the consumed events may become a page for splice_read,
417 * and this page will be returned to system.
418 *
419 * These primitives allow multi process access to different cpu ring buffer
420 * concurrently.
421 *
422 * These primitives don't distinguish read-only and read-consume access.
423 * Multi read-only access are also serialized.
424 */
425
426#ifdef CONFIG_SMP
427static DECLARE_RWSEM(all_cpu_access_lock);
428static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
429
430static inline void trace_access_lock(int cpu)
431{
Steven Rostedtae3b5092013-01-23 15:22:59 -0500432 if (cpu == RING_BUFFER_ALL_CPUS) {
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800433 /* gain it for accessing the whole ring buffer. */
434 down_write(&all_cpu_access_lock);
435 } else {
436 /* gain it for accessing a cpu ring buffer. */
437
Steven Rostedtae3b5092013-01-23 15:22:59 -0500438 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800439 down_read(&all_cpu_access_lock);
440
441 /* Secondly block other access to this @cpu ring buffer. */
442 mutex_lock(&per_cpu(cpu_access_lock, cpu));
443 }
444}
445
446static inline void trace_access_unlock(int cpu)
447{
Steven Rostedtae3b5092013-01-23 15:22:59 -0500448 if (cpu == RING_BUFFER_ALL_CPUS) {
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800449 up_write(&all_cpu_access_lock);
450 } else {
451 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
452 up_read(&all_cpu_access_lock);
453 }
454}
455
456static inline void trace_access_lock_init(void)
457{
458 int cpu;
459
460 for_each_possible_cpu(cpu)
461 mutex_init(&per_cpu(cpu_access_lock, cpu));
462}
463
464#else
465
466static DEFINE_MUTEX(access_lock);
467
468static inline void trace_access_lock(int cpu)
469{
470 (void)cpu;
471 mutex_lock(&access_lock);
472}
473
474static inline void trace_access_unlock(int cpu)
475{
476 (void)cpu;
477 mutex_unlock(&access_lock);
478}
479
480static inline void trace_access_lock_init(void)
481{
482}
483
484#endif
485
Steven Rostedt (Red Hat)d78a4612015-09-25 13:30:47 -0400486#ifdef CONFIG_STACKTRACE
487static void __ftrace_trace_stack(struct ring_buffer *buffer,
488 unsigned long flags,
489 int skip, int pc, struct pt_regs *regs);
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -0400490static inline void ftrace_trace_stack(struct trace_array *tr,
491 struct ring_buffer *buffer,
Steven Rostedt (Red Hat)73dddbb2015-09-29 15:38:55 -0400492 unsigned long flags,
493 int skip, int pc, struct pt_regs *regs);
Steven Rostedt (Red Hat)ca475e82015-09-28 09:41:11 -0400494
Steven Rostedt (Red Hat)d78a4612015-09-25 13:30:47 -0400495#else
496static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
497 unsigned long flags,
498 int skip, int pc, struct pt_regs *regs)
499{
500}
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -0400501static inline void ftrace_trace_stack(struct trace_array *tr,
502 struct ring_buffer *buffer,
Steven Rostedt (Red Hat)73dddbb2015-09-29 15:38:55 -0400503 unsigned long flags,
504 int skip, int pc, struct pt_regs *regs)
Steven Rostedt (Red Hat)ca475e82015-09-28 09:41:11 -0400505{
506}
507
Steven Rostedt (Red Hat)d78a4612015-09-25 13:30:47 -0400508#endif
509
Steven Rostedt (Red Hat)5280bce2013-07-02 19:59:57 -0400510static void tracer_tracing_on(struct trace_array *tr)
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400511{
512 if (tr->trace_buffer.buffer)
513 ring_buffer_record_on(tr->trace_buffer.buffer);
514 /*
515 * This flag is looked at when buffers haven't been allocated
516 * yet, or by some tracers (like irqsoff), that just want to
517 * know if the ring buffer has been disabled, but it can handle
518 * races of where it gets disabled but we still do a record.
519 * As the check is in the fast path of the tracers, it is more
520 * important to be fast than accurate.
521 */
522 tr->buffer_disabled = 0;
523 /* Make the flag seen by readers */
524 smp_wmb();
525}
526
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200527/**
Steven Rostedt499e5472012-02-22 15:50:28 -0500528 * tracing_on - enable tracing buffers
529 *
530 * This function enables tracing buffers that may have been
531 * disabled with tracing_off.
532 */
533void tracing_on(void)
534{
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400535 tracer_tracing_on(&global_trace);
Steven Rostedt499e5472012-02-22 15:50:28 -0500536}
537EXPORT_SYMBOL_GPL(tracing_on);
538
539/**
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500540 * __trace_puts - write a constant string into the trace buffer.
541 * @ip: The address of the caller
542 * @str: The constant string to write
543 * @size: The size of the string.
544 */
545int __trace_puts(unsigned long ip, const char *str, int size)
546{
547 struct ring_buffer_event *event;
548 struct ring_buffer *buffer;
549 struct print_entry *entry;
550 unsigned long irq_flags;
551 int alloc;
zhangwei(Jovi)8abfb872013-07-18 16:31:05 +0800552 int pc;
553
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -0400554 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
zhangwei(Jovi)f0160a52013-07-18 16:31:18 +0800555 return 0;
556
zhangwei(Jovi)8abfb872013-07-18 16:31:05 +0800557 pc = preempt_count();
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500558
Steven Rostedt (Red Hat)3132e102014-01-23 12:27:59 -0500559 if (unlikely(tracing_selftest_running || tracing_disabled))
560 return 0;
561
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500562 alloc = sizeof(*entry) + size + 2; /* possible \n added */
563
564 local_save_flags(irq_flags);
565 buffer = global_trace.trace_buffer.buffer;
566 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
zhangwei(Jovi)8abfb872013-07-18 16:31:05 +0800567 irq_flags, pc);
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500568 if (!event)
569 return 0;
570
571 entry = ring_buffer_event_data(event);
572 entry->ip = ip;
573
574 memcpy(&entry->buf, str, size);
575
576 /* Add a newline if necessary */
577 if (entry->buf[size - 1] != '\n') {
578 entry->buf[size] = '\n';
579 entry->buf[size + 1] = '\0';
580 } else
581 entry->buf[size] = '\0';
582
583 __buffer_unlock_commit(buffer, event);
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -0400584 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500585
586 return size;
587}
588EXPORT_SYMBOL_GPL(__trace_puts);
589
590/**
591 * __trace_bputs - write the pointer to a constant string into trace buffer
592 * @ip: The address of the caller
593 * @str: The constant string to write to the buffer to
594 */
595int __trace_bputs(unsigned long ip, const char *str)
596{
597 struct ring_buffer_event *event;
598 struct ring_buffer *buffer;
599 struct bputs_entry *entry;
600 unsigned long irq_flags;
601 int size = sizeof(struct bputs_entry);
zhangwei(Jovi)8abfb872013-07-18 16:31:05 +0800602 int pc;
603
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -0400604 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
zhangwei(Jovi)f0160a52013-07-18 16:31:18 +0800605 return 0;
606
zhangwei(Jovi)8abfb872013-07-18 16:31:05 +0800607 pc = preempt_count();
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500608
Steven Rostedt (Red Hat)3132e102014-01-23 12:27:59 -0500609 if (unlikely(tracing_selftest_running || tracing_disabled))
610 return 0;
611
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500612 local_save_flags(irq_flags);
613 buffer = global_trace.trace_buffer.buffer;
614 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
zhangwei(Jovi)8abfb872013-07-18 16:31:05 +0800615 irq_flags, pc);
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500616 if (!event)
617 return 0;
618
619 entry = ring_buffer_event_data(event);
620 entry->ip = ip;
621 entry->str = str;
622
623 __buffer_unlock_commit(buffer, event);
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -0400624 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500625
626 return 1;
627}
628EXPORT_SYMBOL_GPL(__trace_bputs);
629
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500630#ifdef CONFIG_TRACER_SNAPSHOT
631/**
632 * trace_snapshot - take a snapshot of the current buffer.
633 *
634 * This causes a swap between the snapshot buffer and the current live
635 * tracing buffer. You can use this to take snapshots of the live
636 * trace when some condition is triggered, but continue to trace.
637 *
638 * Note, make sure to allocate the snapshot with either
639 * a tracing_snapshot_alloc(), or by doing it manually
640 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
641 *
642 * If the snapshot buffer is not allocated, it will stop tracing.
643 * Basically making a permanent snapshot.
644 */
645void tracing_snapshot(void)
646{
647 struct trace_array *tr = &global_trace;
648 struct tracer *tracer = tr->current_trace;
649 unsigned long flags;
650
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500651 if (in_nmi()) {
652 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
653 internal_trace_puts("*** snapshot is being ignored ***\n");
654 return;
655 }
656
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500657 if (!tr->allocated_snapshot) {
Steven Rostedt (Red Hat)ca268da2013-03-09 00:40:58 -0500658 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
659 internal_trace_puts("*** stopping trace here! ***\n");
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500660 tracing_off();
661 return;
662 }
663
664 /* Note, snapshot can not be used when the tracer uses it */
665 if (tracer->use_max_tr) {
Steven Rostedt (Red Hat)ca268da2013-03-09 00:40:58 -0500666 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
667 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500668 return;
669 }
670
671 local_irq_save(flags);
672 update_max_tr(tr, current, smp_processor_id());
673 local_irq_restore(flags);
674}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500675EXPORT_SYMBOL_GPL(tracing_snapshot);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500676
677static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
678 struct trace_buffer *size_buf, int cpu_id);
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400679static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
680
681static int alloc_snapshot(struct trace_array *tr)
682{
683 int ret;
684
685 if (!tr->allocated_snapshot) {
686
687 /* allocate spare buffer */
688 ret = resize_buffer_duplicate_size(&tr->max_buffer,
689 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
690 if (ret < 0)
691 return ret;
692
693 tr->allocated_snapshot = true;
694 }
695
696 return 0;
697}
698
Fabian Frederickad1438a2014-04-17 21:44:42 +0200699static void free_snapshot(struct trace_array *tr)
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400700{
701 /*
702 * We don't free the ring buffer. instead, resize it because
703 * The max_tr ring buffer has some state (e.g. ring->clock) and
704 * we want preserve it.
705 */
706 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
707 set_buffer_entries(&tr->max_buffer, 1);
708 tracing_reset_online_cpus(&tr->max_buffer);
709 tr->allocated_snapshot = false;
710}
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500711
712/**
Tom Zanussi93e31ff2013-10-24 08:59:26 -0500713 * tracing_alloc_snapshot - allocate snapshot buffer.
714 *
715 * This only allocates the snapshot buffer if it isn't already
716 * allocated - it doesn't also take a snapshot.
717 *
718 * This is meant to be used in cases where the snapshot buffer needs
719 * to be set up for events that can't sleep but need to be able to
720 * trigger a snapshot.
721 */
722int tracing_alloc_snapshot(void)
723{
724 struct trace_array *tr = &global_trace;
725 int ret;
726
727 ret = alloc_snapshot(tr);
728 WARN_ON(ret < 0);
729
730 return ret;
731}
732EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
733
734/**
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500735 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
736 *
737 * This is similar to trace_snapshot(), but it will allocate the
738 * snapshot buffer if it isn't already allocated. Use this only
739 * where it is safe to sleep, as the allocation may sleep.
740 *
741 * This causes a swap between the snapshot buffer and the current live
742 * tracing buffer. You can use this to take snapshots of the live
743 * trace when some condition is triggered, but continue to trace.
744 */
745void tracing_snapshot_alloc(void)
746{
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500747 int ret;
748
Tom Zanussi93e31ff2013-10-24 08:59:26 -0500749 ret = tracing_alloc_snapshot();
750 if (ret < 0)
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400751 return;
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500752
753 tracing_snapshot();
754}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500755EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500756#else
757void tracing_snapshot(void)
758{
759 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
760}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500761EXPORT_SYMBOL_GPL(tracing_snapshot);
Tom Zanussi93e31ff2013-10-24 08:59:26 -0500762int tracing_alloc_snapshot(void)
763{
764 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
765 return -ENODEV;
766}
767EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500768void tracing_snapshot_alloc(void)
769{
770 /* Give warning */
771 tracing_snapshot();
772}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500773EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500774#endif /* CONFIG_TRACER_SNAPSHOT */
775
Steven Rostedt (Red Hat)5280bce2013-07-02 19:59:57 -0400776static void tracer_tracing_off(struct trace_array *tr)
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400777{
778 if (tr->trace_buffer.buffer)
779 ring_buffer_record_off(tr->trace_buffer.buffer);
780 /*
781 * This flag is looked at when buffers haven't been allocated
782 * yet, or by some tracers (like irqsoff), that just want to
783 * know if the ring buffer has been disabled, but it can handle
784 * races of where it gets disabled but we still do a record.
785 * As the check is in the fast path of the tracers, it is more
786 * important to be fast than accurate.
787 */
788 tr->buffer_disabled = 1;
789 /* Make the flag seen by readers */
790 smp_wmb();
791}
792
Steven Rostedt499e5472012-02-22 15:50:28 -0500793/**
794 * tracing_off - turn off tracing buffers
795 *
796 * This function stops the tracing buffers from recording data.
797 * It does not disable any overhead the tracers themselves may
798 * be causing. This function simply causes all recording to
799 * the ring buffers to fail.
800 */
801void tracing_off(void)
802{
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400803 tracer_tracing_off(&global_trace);
Steven Rostedt499e5472012-02-22 15:50:28 -0500804}
805EXPORT_SYMBOL_GPL(tracing_off);
806
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400807void disable_trace_on_warning(void)
808{
809 if (__disable_trace_on_warning)
810 tracing_off();
811}
812
Steven Rostedt499e5472012-02-22 15:50:28 -0500813/**
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400814 * tracer_tracing_is_on - show real state of ring buffer enabled
815 * @tr : the trace array to know if ring buffer is enabled
816 *
817 * Shows real state of the ring buffer if it is enabled or not.
818 */
Steven Rostedt (Red Hat)5280bce2013-07-02 19:59:57 -0400819static int tracer_tracing_is_on(struct trace_array *tr)
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400820{
821 if (tr->trace_buffer.buffer)
822 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
823 return !tr->buffer_disabled;
824}
825
Steven Rostedt499e5472012-02-22 15:50:28 -0500826/**
827 * tracing_is_on - show state of ring buffers enabled
828 */
829int tracing_is_on(void)
830{
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400831 return tracer_tracing_is_on(&global_trace);
Steven Rostedt499e5472012-02-22 15:50:28 -0500832}
833EXPORT_SYMBOL_GPL(tracing_is_on);
834
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400835static int __init set_buf_size(char *str)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200836{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400837 unsigned long buf_size;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200838
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200839 if (!str)
840 return 0;
Li Zefan9d612be2009-06-24 17:33:15 +0800841 buf_size = memparse(str, &str);
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200842 /* nr_entries can not be zero */
Li Zefan9d612be2009-06-24 17:33:15 +0800843 if (buf_size == 0)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200844 return 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400845 trace_buf_size = buf_size;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200846 return 1;
847}
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400848__setup("trace_buf_size=", set_buf_size);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200849
Tim Bird0e950172010-02-25 15:36:43 -0800850static int __init set_tracing_thresh(char *str)
851{
Wang Tianhong87abb3b2012-08-02 14:02:00 +0800852 unsigned long threshold;
Tim Bird0e950172010-02-25 15:36:43 -0800853 int ret;
854
855 if (!str)
856 return 0;
Daniel Walterbcd83ea2012-09-26 22:08:38 +0200857 ret = kstrtoul(str, 0, &threshold);
Tim Bird0e950172010-02-25 15:36:43 -0800858 if (ret < 0)
859 return 0;
Wang Tianhong87abb3b2012-08-02 14:02:00 +0800860 tracing_thresh = threshold * 1000;
Tim Bird0e950172010-02-25 15:36:43 -0800861 return 1;
862}
863__setup("tracing_thresh=", set_tracing_thresh);
864
Steven Rostedt57f50be2008-05-12 21:20:44 +0200865unsigned long nsecs_to_usecs(unsigned long nsecs)
866{
867 return nsecs / 1000;
868}
869
Steven Rostedt (Red Hat)a3418a32015-09-29 09:43:30 -0400870/*
871 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
872 * It uses C(a, b) where 'a' is the enum name and 'b' is the string that
873 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
874 * of strings in the order that the enums were defined.
875 */
876#undef C
877#define C(a, b) b
878
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200879/* These must match the bit postions in trace_iterator_flags */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200880static const char *trace_options[] = {
Steven Rostedt (Red Hat)a3418a32015-09-29 09:43:30 -0400881 TRACE_FLAGS
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200882 NULL
883};
884
Zhaolei5079f322009-08-25 16:12:56 +0800885static struct {
886 u64 (*func)(void);
887 const char *name;
David Sharp8be07092012-11-13 12:18:22 -0800888 int in_ns; /* is this clock in nanoseconds? */
Zhaolei5079f322009-08-25 16:12:56 +0800889} trace_clocks[] = {
Thomas Gleixner1b3e5c02014-07-16 21:05:25 +0000890 { trace_clock_local, "local", 1 },
891 { trace_clock_global, "global", 1 },
892 { trace_clock_counter, "counter", 0 },
Linus Torvaldse7fda6c2014-08-05 17:46:42 -0700893 { trace_clock_jiffies, "uptime", 0 },
Thomas Gleixner1b3e5c02014-07-16 21:05:25 +0000894 { trace_clock, "perf", 1 },
895 { ktime_get_mono_fast_ns, "mono", 1 },
Drew Richardsonaabfa5f2015-05-08 07:30:39 -0700896 { ktime_get_raw_fast_ns, "mono_raw", 1 },
David Sharp8cbd9cc2012-11-13 12:18:21 -0800897 ARCH_TRACE_CLOCKS
Zhaolei5079f322009-08-25 16:12:56 +0800898};
899
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200900/*
901 * trace_parser_get_init - gets the buffer for trace parser
902 */
903int trace_parser_get_init(struct trace_parser *parser, int size)
904{
905 memset(parser, 0, sizeof(*parser));
906
907 parser->buffer = kmalloc(size, GFP_KERNEL);
908 if (!parser->buffer)
909 return 1;
910
911 parser->size = size;
912 return 0;
913}
914
915/*
916 * trace_parser_put - frees the buffer for trace parser
917 */
918void trace_parser_put(struct trace_parser *parser)
919{
920 kfree(parser->buffer);
921}
922
923/*
924 * trace_get_user - reads the user input string separated by space
925 * (matched by isspace(ch))
926 *
927 * For each string found the 'struct trace_parser' is updated,
928 * and the function returns.
929 *
930 * Returns number of bytes read.
931 *
932 * See kernel/trace/trace.h for 'struct trace_parser' details.
933 */
934int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
935 size_t cnt, loff_t *ppos)
936{
937 char ch;
938 size_t read = 0;
939 ssize_t ret;
940
941 if (!*ppos)
942 trace_parser_clear(parser);
943
944 ret = get_user(ch, ubuf++);
945 if (ret)
946 goto out;
947
948 read++;
949 cnt--;
950
951 /*
952 * The parser is not finished with the last write,
953 * continue reading the user input without skipping spaces.
954 */
955 if (!parser->cont) {
956 /* skip white space */
957 while (cnt && isspace(ch)) {
958 ret = get_user(ch, ubuf++);
959 if (ret)
960 goto out;
961 read++;
962 cnt--;
963 }
964
965 /* only spaces were written */
966 if (isspace(ch)) {
967 *ppos += read;
968 ret = read;
969 goto out;
970 }
971
972 parser->idx = 0;
973 }
974
975 /* read the non-space input */
976 while (cnt && !isspace(ch)) {
Li Zefan3c235a32009-09-22 13:51:54 +0800977 if (parser->idx < parser->size - 1)
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200978 parser->buffer[parser->idx++] = ch;
979 else {
980 ret = -EINVAL;
981 goto out;
982 }
983 ret = get_user(ch, ubuf++);
984 if (ret)
985 goto out;
986 read++;
987 cnt--;
988 }
989
990 /* We either got finished input or we have to wait for another call. */
991 if (isspace(ch)) {
992 parser->buffer[parser->idx] = 0;
993 parser->cont = false;
Steven Rostedt057db842013-10-09 22:23:23 -0400994 } else if (parser->idx < parser->size - 1) {
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200995 parser->cont = true;
996 parser->buffer[parser->idx++] = ch;
Steven Rostedt057db842013-10-09 22:23:23 -0400997 } else {
998 ret = -EINVAL;
999 goto out;
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +02001000 }
1001
1002 *ppos += read;
1003 ret = read;
1004
1005out:
1006 return ret;
1007}
1008
Steven Rostedt (Red Hat)3a161d92014-06-25 15:54:42 -04001009/* TODO add a seq_buf_to_buffer() */
Dmitri Vorobievb8b94262009-03-22 19:11:11 +02001010static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02001011{
1012 int len;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02001013
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05001014 if (trace_seq_used(s) <= s->seq.readpos)
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02001015 return -EBUSY;
1016
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05001017 len = trace_seq_used(s) - s->seq.readpos;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02001018 if (cnt > len)
1019 cnt = len;
Steven Rostedt (Red Hat)3a161d92014-06-25 15:54:42 -04001020 memcpy(buf, s->buffer + s->seq.readpos, cnt);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02001021
Steven Rostedt (Red Hat)3a161d92014-06-25 15:54:42 -04001022 s->seq.readpos += cnt;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02001023 return cnt;
1024}
1025
Tim Bird0e950172010-02-25 15:36:43 -08001026unsigned long __read_mostly tracing_thresh;
1027
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001028#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001029/*
1030 * Copy the new maximum trace into the separate maximum-trace
1031 * structure. (this way the maximum trace is permanently saved,
1032 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
1033 */
1034static void
1035__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1036{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001037 struct trace_buffer *trace_buf = &tr->trace_buffer;
1038 struct trace_buffer *max_buf = &tr->max_buffer;
1039 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1040 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001041
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001042 max_buf->cpu = cpu;
1043 max_buf->time_start = data->preempt_timestamp;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001044
Steven Rostedt (Red Hat)6d9b3fa2014-01-14 11:28:38 -05001045 max_data->saved_latency = tr->max_latency;
Steven Rostedt8248ac02009-09-02 12:27:41 -04001046 max_data->critical_start = data->critical_start;
1047 max_data->critical_end = data->critical_end;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001048
Arnaldo Carvalho de Melo1acaa1b2010-03-05 18:23:50 -03001049 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
Steven Rostedt8248ac02009-09-02 12:27:41 -04001050 max_data->pid = tsk->pid;
Steven Rostedt (Red Hat)f17a5192013-05-30 21:10:37 -04001051 /*
1052 * If tsk == current, then use current_uid(), as that does not use
1053 * RCU. The irq tracer can be called out of RCU scope.
1054 */
1055 if (tsk == current)
1056 max_data->uid = current_uid();
1057 else
1058 max_data->uid = task_uid(tsk);
1059
Steven Rostedt8248ac02009-09-02 12:27:41 -04001060 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1061 max_data->policy = tsk->policy;
1062 max_data->rt_priority = tsk->rt_priority;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001063
1064 /* record this tasks comm */
1065 tracing_record_cmdline(tsk);
1066}
1067
Steven Rostedt4fcdae82008-05-12 21:21:00 +02001068/**
1069 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1070 * @tr: tracer
1071 * @tsk: the task with the latency
1072 * @cpu: The cpu that initiated the trace.
1073 *
1074 * Flip the buffers between the @tr and the max_tr and record information
1075 * about which task was the cause of this latency.
1076 */
Ingo Molnare309b412008-05-12 21:20:51 +02001077void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001078update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1079{
Steven Rostedt (Red Hat)2721e722013-03-12 11:32:32 -04001080 struct ring_buffer *buf;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001081
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001082 if (tr->stop_count)
Steven Rostedtb8de7bd12009-08-31 22:32:27 -04001083 return;
1084
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02001085 WARN_ON_ONCE(!irqs_disabled());
Steven Rostedt34600f02013-01-22 13:35:11 -05001086
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05001087 if (!tr->allocated_snapshot) {
Hiraku Toyookadebdd572012-12-26 11:53:00 +09001088 /* Only the nop tracer should hit this when disabling */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001089 WARN_ON_ONCE(tr->current_trace != &nop_trace);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09001090 return;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09001091 }
Steven Rostedt34600f02013-01-22 13:35:11 -05001092
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001093 arch_spin_lock(&tr->max_lock);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001094
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001095 buf = tr->trace_buffer.buffer;
1096 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1097 tr->max_buffer.buffer = buf;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001098
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001099 __update_max_tr(tr, tsk, cpu);
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001100 arch_spin_unlock(&tr->max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001101}
1102
1103/**
1104 * update_max_tr_single - only copy one trace over, and reset the rest
1105 * @tr - tracer
1106 * @tsk - task with the latency
1107 * @cpu - the cpu of the buffer to copy.
Steven Rostedt4fcdae82008-05-12 21:21:00 +02001108 *
1109 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001110 */
Ingo Molnare309b412008-05-12 21:20:51 +02001111void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001112update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1113{
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001114 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001115
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001116 if (tr->stop_count)
Steven Rostedtb8de7bd12009-08-31 22:32:27 -04001117 return;
1118
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02001119 WARN_ON_ONCE(!irqs_disabled());
Steven Rostedt6c244992013-04-29 20:08:14 -04001120 if (!tr->allocated_snapshot) {
Steven Rostedt (Red Hat)2930e042013-03-26 17:33:00 -04001121 /* Only the nop tracer should hit this when disabling */
Linus Torvalds9e8529a2013-04-29 13:55:38 -07001122 WARN_ON_ONCE(tr->current_trace != &nop_trace);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09001123 return;
Steven Rostedt (Red Hat)2930e042013-03-26 17:33:00 -04001124 }
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09001125
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001126 arch_spin_lock(&tr->max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001127
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001128 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001129
Steven Rostedte8165db2009-09-03 19:13:05 -04001130 if (ret == -EBUSY) {
1131 /*
1132 * We failed to swap the buffer due to a commit taking
1133 * place on this CPU. We fail to record, but we reset
1134 * the max trace buffer (no one writes directly to it)
1135 * and flag that it failed.
1136 */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001137 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
Steven Rostedte8165db2009-09-03 19:13:05 -04001138 "Failed to swap buffers due to commit in progress\n");
1139 }
1140
Steven Rostedte8165db2009-09-03 19:13:05 -04001141 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001142
1143 __update_max_tr(tr, tsk, cpu);
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001144 arch_spin_unlock(&tr->max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001145}
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001146#endif /* CONFIG_TRACER_MAX_TRACE */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001147
Rabin Vincente30f53a2014-11-10 19:46:34 +01001148static int wait_on_pipe(struct trace_iterator *iter, bool full)
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001149{
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05001150 /* Iterators are static, they should be filled or empty */
1151 if (trace_buffer_iter(iter, iter->cpu_file))
Steven Rostedt (Red Hat)8b8b3682014-06-10 09:46:00 -04001152 return 0;
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001153
Rabin Vincente30f53a2014-11-10 19:46:34 +01001154 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1155 full);
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001156}
1157
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001158#ifdef CONFIG_FTRACE_STARTUP_TEST
1159static int run_tracer_selftest(struct tracer *type)
1160{
1161 struct trace_array *tr = &global_trace;
1162 struct tracer *saved_tracer = tr->current_trace;
1163 int ret;
1164
1165 if (!type->selftest || tracing_selftest_disabled)
1166 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001167
1168 /*
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001169 * Run a selftest on this tracer.
1170 * Here we reset the trace buffer, and set the current
1171 * tracer to be this tracer. The tracer can then run some
1172 * internal tracing to verify that everything is in order.
1173 * If we fail, we do not register this tracer.
Steven Rostedt4fcdae82008-05-12 21:21:00 +02001174 */
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001175 tracing_reset_online_cpus(&tr->trace_buffer);
Steven Rostedt4fcdae82008-05-12 21:21:00 +02001176
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001177 tr->current_trace = type;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001178
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001179#ifdef CONFIG_TRACER_MAX_TRACE
1180 if (type->use_max_tr) {
1181 /* If we expanded the buffers, make sure the max is expanded too */
1182 if (ring_buffer_expanded)
1183 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1184 RING_BUFFER_ALL_CPUS);
1185 tr->allocated_snapshot = true;
1186 }
1187#endif
1188
1189 /* the test is responsible for initializing and enabling */
1190 pr_info("Testing tracer %s: ", type->name);
1191 ret = type->selftest(type, tr);
1192 /* the test is responsible for resetting too */
1193 tr->current_trace = saved_tracer;
1194 if (ret) {
1195 printk(KERN_CONT "FAILED!\n");
1196 /* Add the warning after printing 'FAILED' */
1197 WARN_ON(1);
1198 return -1;
1199 }
1200 /* Only reset on passing, to avoid touching corrupted buffers */
1201 tracing_reset_online_cpus(&tr->trace_buffer);
1202
1203#ifdef CONFIG_TRACER_MAX_TRACE
1204 if (type->use_max_tr) {
1205 tr->allocated_snapshot = false;
1206
1207 /* Shrink the max buffer again */
1208 if (ring_buffer_expanded)
1209 ring_buffer_resize(tr->max_buffer.buffer, 1,
1210 RING_BUFFER_ALL_CPUS);
1211 }
1212#endif
1213
1214 printk(KERN_CONT "PASSED\n");
1215 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001216}
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001217#else
1218static inline int run_tracer_selftest(struct tracer *type)
1219{
1220 return 0;
1221}
1222#endif /* CONFIG_FTRACE_STARTUP_TEST */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001223
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04001224static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1225
Steven Rostedt4fcdae82008-05-12 21:21:00 +02001226/**
1227 * register_tracer - register a tracer with the ftrace system.
1228 * @type - the plugin for the tracer
1229 *
1230 * Register a new plugin tracer.
1231 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001232int register_tracer(struct tracer *type)
1233{
1234 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001235 int ret = 0;
1236
1237 if (!type->name) {
1238 pr_info("Tracer must have a name\n");
1239 return -1;
1240 }
1241
Dan Carpenter24a461d2010-07-10 12:06:44 +02001242 if (strlen(type->name) >= MAX_TRACER_SIZE) {
Li Zefanee6c2c12009-09-18 14:06:47 +08001243 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1244 return -1;
1245 }
1246
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001247 mutex_lock(&trace_types_lock);
Ingo Molnar86fa2f62008-11-19 10:00:15 +01001248
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +01001249 tracing_selftest_running = true;
1250
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001251 for (t = trace_types; t; t = t->next) {
1252 if (strcmp(type->name, t->name) == 0) {
1253 /* already found */
Li Zefanee6c2c12009-09-18 14:06:47 +08001254 pr_info("Tracer %s already registered\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001255 type->name);
1256 ret = -1;
1257 goto out;
1258 }
1259 }
1260
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01001261 if (!type->set_flag)
1262 type->set_flag = &dummy_set_flag;
1263 if (!type->flags)
1264 type->flags = &dummy_tracer_flags;
1265 else
1266 if (!type->flags->opts)
1267 type->flags->opts = dummy_tracer_opt;
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01001268
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001269 ret = run_tracer_selftest(type);
1270 if (ret < 0)
1271 goto out;
Steven Rostedt60a11772008-05-12 21:20:44 +02001272
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001273 type->next = trace_types;
1274 trace_types = type;
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04001275 add_tracer_options(&global_trace, type);
Steven Rostedt60a11772008-05-12 21:20:44 +02001276
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001277 out:
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +01001278 tracing_selftest_running = false;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001279 mutex_unlock(&trace_types_lock);
1280
Steven Rostedtdac74942009-02-05 01:13:38 -05001281 if (ret || !default_bootup_tracer)
1282 goto out_unlock;
Steven Rostedtb2821ae2009-02-02 21:38:32 -05001283
Li Zefanee6c2c12009-09-18 14:06:47 +08001284 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
Steven Rostedtdac74942009-02-05 01:13:38 -05001285 goto out_unlock;
1286
1287 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1288 /* Do we want this tracer to start on bootup? */
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05001289 tracing_set_tracer(&global_trace, type->name);
Steven Rostedtdac74942009-02-05 01:13:38 -05001290 default_bootup_tracer = NULL;
1291 /* disable other selftests, since this will break it. */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05001292 tracing_selftest_disabled = true;
Steven Rostedtdac74942009-02-05 01:13:38 -05001293#ifdef CONFIG_FTRACE_STARTUP_TEST
1294 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1295 type->name);
1296#endif
1297
1298 out_unlock:
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001299 return ret;
1300}
1301
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001302void tracing_reset(struct trace_buffer *buf, int cpu)
Steven Rostedtf6339032009-09-04 12:35:16 -04001303{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001304 struct ring_buffer *buffer = buf->buffer;
Steven Rostedtf6339032009-09-04 12:35:16 -04001305
Hiraku Toyookaa5416412012-12-19 16:02:34 +09001306 if (!buffer)
1307 return;
1308
Steven Rostedtf6339032009-09-04 12:35:16 -04001309 ring_buffer_record_disable(buffer);
1310
1311 /* Make sure all commits have finished */
1312 synchronize_sched();
Steven Rostedt68179682012-05-08 20:57:53 -04001313 ring_buffer_reset_cpu(buffer, cpu);
Steven Rostedtf6339032009-09-04 12:35:16 -04001314
1315 ring_buffer_record_enable(buffer);
1316}
1317
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001318void tracing_reset_online_cpus(struct trace_buffer *buf)
Pekka J Enberg213cc062008-12-19 12:08:39 +02001319{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001320 struct ring_buffer *buffer = buf->buffer;
Pekka J Enberg213cc062008-12-19 12:08:39 +02001321 int cpu;
1322
Hiraku Toyookaa5416412012-12-19 16:02:34 +09001323 if (!buffer)
1324 return;
1325
Steven Rostedt621968c2009-09-04 12:02:35 -04001326 ring_buffer_record_disable(buffer);
1327
1328 /* Make sure all commits have finished */
1329 synchronize_sched();
1330
Alexander Z Lam94571582013-08-02 18:36:16 -07001331 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
Pekka J Enberg213cc062008-12-19 12:08:39 +02001332
1333 for_each_online_cpu(cpu)
Steven Rostedt68179682012-05-08 20:57:53 -04001334 ring_buffer_reset_cpu(buffer, cpu);
Steven Rostedt621968c2009-09-04 12:02:35 -04001335
1336 ring_buffer_record_enable(buffer);
Pekka J Enberg213cc062008-12-19 12:08:39 +02001337}
1338
Steven Rostedt (Red Hat)09d80912013-07-23 22:21:59 -04001339/* Must have trace_types_lock held */
Steven Rostedt (Red Hat)873c6422013-03-04 23:26:06 -05001340void tracing_reset_all_online_cpus(void)
Steven Rostedt9456f0f2009-05-06 21:54:09 -04001341{
Steven Rostedt (Red Hat)873c6422013-03-04 23:26:06 -05001342 struct trace_array *tr;
1343
Steven Rostedt (Red Hat)873c6422013-03-04 23:26:06 -05001344 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001345 tracing_reset_online_cpus(&tr->trace_buffer);
1346#ifdef CONFIG_TRACER_MAX_TRACE
1347 tracing_reset_online_cpus(&tr->max_buffer);
1348#endif
Steven Rostedt (Red Hat)873c6422013-03-04 23:26:06 -05001349 }
Steven Rostedt9456f0f2009-05-06 21:54:09 -04001350}
1351
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001352#define SAVED_CMDLINES_DEFAULT 128
Thomas Gleixner2c7eea42009-03-18 09:03:19 +01001353#define NO_CMDLINE_MAP UINT_MAX
Thomas Gleixneredc35bd2009-12-03 12:38:57 +01001354static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001355struct saved_cmdlines_buffer {
1356 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1357 unsigned *map_cmdline_to_pid;
1358 unsigned cmdline_num;
1359 int cmdline_idx;
1360 char *saved_cmdlines;
1361};
1362static struct saved_cmdlines_buffer *savedcmd;
Steven Rostedt25b0b442008-05-12 21:21:00 +02001363
Steven Rostedt25b0b442008-05-12 21:21:00 +02001364/* temporary disable recording */
Hannes Eder4fd27352009-02-10 19:44:12 +01001365static atomic_t trace_record_cmdline_disabled __read_mostly;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001366
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001367static inline char *get_saved_cmdlines(int idx)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001368{
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001369 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1370}
1371
1372static inline void set_cmdline(int idx, const char *cmdline)
1373{
1374 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1375}
1376
1377static int allocate_cmdlines_buffer(unsigned int val,
1378 struct saved_cmdlines_buffer *s)
1379{
1380 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1381 GFP_KERNEL);
1382 if (!s->map_cmdline_to_pid)
1383 return -ENOMEM;
1384
1385 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1386 if (!s->saved_cmdlines) {
1387 kfree(s->map_cmdline_to_pid);
1388 return -ENOMEM;
1389 }
1390
1391 s->cmdline_idx = 0;
1392 s->cmdline_num = val;
1393 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1394 sizeof(s->map_pid_to_cmdline));
1395 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1396 val * sizeof(*s->map_cmdline_to_pid));
1397
1398 return 0;
1399}
1400
1401static int trace_create_savedcmd(void)
1402{
1403 int ret;
1404
Namhyung Kima6af8fb2014-06-10 16:11:35 +09001405 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001406 if (!savedcmd)
1407 return -ENOMEM;
1408
1409 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1410 if (ret < 0) {
1411 kfree(savedcmd);
1412 savedcmd = NULL;
1413 return -ENOMEM;
1414 }
1415
1416 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001417}
1418
Carsten Emdeb5130b12009-09-13 01:43:07 +02001419int is_tracing_stopped(void)
1420{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001421 return global_trace.stop_count;
Carsten Emdeb5130b12009-09-13 01:43:07 +02001422}
1423
Steven Rostedt0f048702008-11-05 16:05:44 -05001424/**
1425 * tracing_start - quick start of the tracer
1426 *
1427 * If tracing is enabled but was stopped by tracing_stop,
1428 * this will start the tracer back up.
1429 */
1430void tracing_start(void)
1431{
1432 struct ring_buffer *buffer;
1433 unsigned long flags;
1434
1435 if (tracing_disabled)
1436 return;
1437
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001438 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1439 if (--global_trace.stop_count) {
1440 if (global_trace.stop_count < 0) {
Steven Rostedtb06a8302009-01-22 14:26:15 -05001441 /* Someone screwed up their debugging */
1442 WARN_ON_ONCE(1);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001443 global_trace.stop_count = 0;
Steven Rostedtb06a8302009-01-22 14:26:15 -05001444 }
Steven Rostedt0f048702008-11-05 16:05:44 -05001445 goto out;
1446 }
1447
Steven Rostedta2f80712010-03-12 19:56:00 -05001448 /* Prevent the buffers from switching */
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001449 arch_spin_lock(&global_trace.max_lock);
Steven Rostedt0f048702008-11-05 16:05:44 -05001450
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001451 buffer = global_trace.trace_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001452 if (buffer)
1453 ring_buffer_record_enable(buffer);
1454
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001455#ifdef CONFIG_TRACER_MAX_TRACE
1456 buffer = global_trace.max_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001457 if (buffer)
1458 ring_buffer_record_enable(buffer);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001459#endif
Steven Rostedt0f048702008-11-05 16:05:44 -05001460
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001461 arch_spin_unlock(&global_trace.max_lock);
Steven Rostedta2f80712010-03-12 19:56:00 -05001462
Steven Rostedt0f048702008-11-05 16:05:44 -05001463 out:
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001464 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1465}
1466
1467static void tracing_start_tr(struct trace_array *tr)
1468{
1469 struct ring_buffer *buffer;
1470 unsigned long flags;
1471
1472 if (tracing_disabled)
1473 return;
1474
1475 /* If global, we need to also start the max tracer */
1476 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1477 return tracing_start();
1478
1479 raw_spin_lock_irqsave(&tr->start_lock, flags);
1480
1481 if (--tr->stop_count) {
1482 if (tr->stop_count < 0) {
1483 /* Someone screwed up their debugging */
1484 WARN_ON_ONCE(1);
1485 tr->stop_count = 0;
1486 }
1487 goto out;
1488 }
1489
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001490 buffer = tr->trace_buffer.buffer;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001491 if (buffer)
1492 ring_buffer_record_enable(buffer);
1493
1494 out:
1495 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
Steven Rostedt0f048702008-11-05 16:05:44 -05001496}
1497
1498/**
1499 * tracing_stop - quick stop of the tracer
1500 *
1501 * Light weight way to stop tracing. Use in conjunction with
1502 * tracing_start.
1503 */
1504void tracing_stop(void)
1505{
1506 struct ring_buffer *buffer;
1507 unsigned long flags;
1508
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001509 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1510 if (global_trace.stop_count++)
Steven Rostedt0f048702008-11-05 16:05:44 -05001511 goto out;
1512
Steven Rostedta2f80712010-03-12 19:56:00 -05001513 /* Prevent the buffers from switching */
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001514 arch_spin_lock(&global_trace.max_lock);
Steven Rostedta2f80712010-03-12 19:56:00 -05001515
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001516 buffer = global_trace.trace_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001517 if (buffer)
1518 ring_buffer_record_disable(buffer);
1519
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001520#ifdef CONFIG_TRACER_MAX_TRACE
1521 buffer = global_trace.max_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001522 if (buffer)
1523 ring_buffer_record_disable(buffer);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001524#endif
Steven Rostedt0f048702008-11-05 16:05:44 -05001525
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001526 arch_spin_unlock(&global_trace.max_lock);
Steven Rostedta2f80712010-03-12 19:56:00 -05001527
Steven Rostedt0f048702008-11-05 16:05:44 -05001528 out:
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001529 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1530}
1531
1532static void tracing_stop_tr(struct trace_array *tr)
1533{
1534 struct ring_buffer *buffer;
1535 unsigned long flags;
1536
1537 /* If global, we need to also stop the max tracer */
1538 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1539 return tracing_stop();
1540
1541 raw_spin_lock_irqsave(&tr->start_lock, flags);
1542 if (tr->stop_count++)
1543 goto out;
1544
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001545 buffer = tr->trace_buffer.buffer;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001546 if (buffer)
1547 ring_buffer_record_disable(buffer);
1548
1549 out:
1550 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
Steven Rostedt0f048702008-11-05 16:05:44 -05001551}
1552
Ingo Molnare309b412008-05-12 21:20:51 +02001553void trace_stop_cmdline_recording(void);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001554
Steven Rostedt (Red Hat)379cfda2014-05-30 09:42:39 -04001555static int trace_save_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001556{
Carsten Emdea635cf02009-03-18 09:00:41 +01001557 unsigned pid, idx;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001558
1559 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
Steven Rostedt (Red Hat)379cfda2014-05-30 09:42:39 -04001560 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001561
1562 /*
1563 * It's not the end of the world if we don't get
1564 * the lock, but we also don't want to spin
1565 * nor do we want to disable interrupts,
1566 * so if we miss here, then better luck next time.
1567 */
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001568 if (!arch_spin_trylock(&trace_cmdline_lock))
Steven Rostedt (Red Hat)379cfda2014-05-30 09:42:39 -04001569 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001570
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001571 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
Thomas Gleixner2c7eea42009-03-18 09:03:19 +01001572 if (idx == NO_CMDLINE_MAP) {
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001573 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001574
Carsten Emdea635cf02009-03-18 09:00:41 +01001575 /*
1576 * Check whether the cmdline buffer at idx has a pid
1577 * mapped. We are going to overwrite that entry so we
1578 * need to clear the map_pid_to_cmdline. Otherwise we
1579 * would read the new comm for the old pid.
1580 */
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001581 pid = savedcmd->map_cmdline_to_pid[idx];
Carsten Emdea635cf02009-03-18 09:00:41 +01001582 if (pid != NO_CMDLINE_MAP)
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001583 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001584
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001585 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1586 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001587
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001588 savedcmd->cmdline_idx = idx;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001589 }
1590
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001591 set_cmdline(idx, tsk->comm);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001592
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001593 arch_spin_unlock(&trace_cmdline_lock);
Steven Rostedt (Red Hat)379cfda2014-05-30 09:42:39 -04001594
1595 return 1;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001596}
1597
Steven Rostedt (Red Hat)4c27e752014-05-30 10:49:46 -04001598static void __trace_find_cmdline(int pid, char comm[])
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001599{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001600 unsigned map;
1601
Steven Rostedt4ca53082009-03-16 19:20:15 -04001602 if (!pid) {
1603 strcpy(comm, "<idle>");
1604 return;
1605 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001606
Steven Rostedt74bf4072010-01-25 15:11:53 -05001607 if (WARN_ON_ONCE(pid < 0)) {
1608 strcpy(comm, "<XXX>");
1609 return;
1610 }
1611
Steven Rostedt4ca53082009-03-16 19:20:15 -04001612 if (pid > PID_MAX_DEFAULT) {
1613 strcpy(comm, "<...>");
1614 return;
1615 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001616
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001617 map = savedcmd->map_pid_to_cmdline[pid];
Thomas Gleixner50d88752009-03-18 08:58:44 +01001618 if (map != NO_CMDLINE_MAP)
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001619 strcpy(comm, get_saved_cmdlines(map));
Thomas Gleixner50d88752009-03-18 08:58:44 +01001620 else
1621 strcpy(comm, "<...>");
Steven Rostedt (Red Hat)4c27e752014-05-30 10:49:46 -04001622}
1623
1624void trace_find_cmdline(int pid, char comm[])
1625{
1626 preempt_disable();
1627 arch_spin_lock(&trace_cmdline_lock);
1628
1629 __trace_find_cmdline(pid, comm);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001630
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001631 arch_spin_unlock(&trace_cmdline_lock);
Heiko Carstens5b6045a2009-05-26 17:28:02 +02001632 preempt_enable();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001633}
1634
Ingo Molnare309b412008-05-12 21:20:51 +02001635void tracing_record_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001636{
Steven Rostedt0fb96562012-05-11 14:25:30 -04001637 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001638 return;
1639
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001640 if (!__this_cpu_read(trace_cmdline_save))
1641 return;
1642
Steven Rostedt (Red Hat)379cfda2014-05-30 09:42:39 -04001643 if (trace_save_cmdline(tsk))
1644 __this_cpu_write(trace_cmdline_save, false);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001645}
1646
Pekka Paalanen45dcd8b2008-09-16 21:56:41 +03001647void
Steven Rostedt38697052008-10-01 13:14:09 -04001648tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1649 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001650{
1651 struct task_struct *tsk = current;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001652
Steven Rostedt777e2082008-09-29 23:02:42 -04001653 entry->preempt_count = pc & 0xff;
1654 entry->pid = (tsk) ? tsk->pid : 0;
1655 entry->flags =
Steven Rostedt92444892008-10-24 09:42:59 -04001656#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
Steven Rostedt2e2ca152008-08-01 12:26:40 -04001657 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
Steven Rostedt92444892008-10-24 09:42:59 -04001658#else
1659 TRACE_FLAG_IRQS_NOSUPPORT |
1660#endif
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001661 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1662 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
Peter Zijlstrae5137b52013-10-04 17:28:26 +02001663 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1664 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001665}
Frederic Weisbeckerf413cdb2009-08-07 01:25:54 +02001666EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001667
Steven Rostedte77405a2009-09-02 14:17:06 -04001668struct ring_buffer_event *
1669trace_buffer_lock_reserve(struct ring_buffer *buffer,
1670 int type,
1671 unsigned long len,
1672 unsigned long flags, int pc)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001673{
1674 struct ring_buffer_event *event;
1675
Steven Rostedte77405a2009-09-02 14:17:06 -04001676 event = ring_buffer_lock_reserve(buffer, len);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001677 if (event != NULL) {
1678 struct trace_entry *ent = ring_buffer_event_data(event);
1679
1680 tracing_generic_entry_update(ent, flags, pc);
1681 ent->type = type;
1682 }
1683
1684 return event;
1685}
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001686
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001687void
1688__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1689{
1690 __this_cpu_write(trace_cmdline_save, true);
1691 ring_buffer_unlock_commit(buffer, event);
1692}
1693
Steven Rostedt (Red Hat)b7f0c952015-09-25 17:38:44 -04001694void trace_buffer_unlock_commit(struct trace_array *tr,
1695 struct ring_buffer *buffer,
1696 struct ring_buffer_event *event,
1697 unsigned long flags, int pc)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001698{
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001699 __buffer_unlock_commit(buffer, event);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001700
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -04001701 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
Steven Rostedte77405a2009-09-02 14:17:06 -04001702 ftrace_trace_userstack(buffer, flags, pc);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001703}
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001704EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001705
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04001706static struct ring_buffer *temp_buffer;
1707
Steven Rostedtef5580d2009-02-27 19:38:04 -05001708struct ring_buffer_event *
Steven Rostedtccb469a2012-08-02 10:32:10 -04001709trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
Steven Rostedt (Red Hat)7f1d2f82015-05-05 10:09:53 -04001710 struct trace_event_file *trace_file,
Steven Rostedtccb469a2012-08-02 10:32:10 -04001711 int type, unsigned long len,
1712 unsigned long flags, int pc)
1713{
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04001714 struct ring_buffer_event *entry;
1715
Steven Rostedt (Red Hat)7f1d2f82015-05-05 10:09:53 -04001716 *current_rb = trace_file->tr->trace_buffer.buffer;
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04001717 entry = trace_buffer_lock_reserve(*current_rb,
Steven Rostedtccb469a2012-08-02 10:32:10 -04001718 type, len, flags, pc);
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04001719 /*
1720 * If tracing is off, but we have triggers enabled
1721 * we still need to look at the event data. Use the temp_buffer
1722 * to store the trace event for the tigger to use. It's recusive
1723 * safe and will not be recorded anywhere.
1724 */
Steven Rostedt (Red Hat)5d6ad962015-05-13 15:12:33 -04001725 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04001726 *current_rb = temp_buffer;
1727 entry = trace_buffer_lock_reserve(*current_rb,
1728 type, len, flags, pc);
1729 }
1730 return entry;
Steven Rostedtccb469a2012-08-02 10:32:10 -04001731}
1732EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1733
1734struct ring_buffer_event *
Steven Rostedte77405a2009-09-02 14:17:06 -04001735trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1736 int type, unsigned long len,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001737 unsigned long flags, int pc)
1738{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001739 *current_rb = global_trace.trace_buffer.buffer;
Steven Rostedte77405a2009-09-02 14:17:06 -04001740 return trace_buffer_lock_reserve(*current_rb,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001741 type, len, flags, pc);
1742}
Steven Rostedt94487d62009-05-05 19:22:53 -04001743EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001744
Steven Rostedt (Red Hat)b7f0c952015-09-25 17:38:44 -04001745void trace_buffer_unlock_commit_regs(struct trace_array *tr,
1746 struct ring_buffer *buffer,
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001747 struct ring_buffer_event *event,
1748 unsigned long flags, int pc,
1749 struct pt_regs *regs)
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001750{
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001751 __buffer_unlock_commit(buffer, event);
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001752
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -04001753 ftrace_trace_stack(tr, buffer, flags, 6, pc, regs);
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001754 ftrace_trace_userstack(buffer, flags, pc);
1755}
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001756EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001757
Steven Rostedte77405a2009-09-02 14:17:06 -04001758void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1759 struct ring_buffer_event *event)
Steven Rostedt77d9f462009-04-02 01:16:59 -04001760{
Steven Rostedte77405a2009-09-02 14:17:06 -04001761 ring_buffer_discard_commit(buffer, event);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001762}
Steven Rostedt12acd472009-04-17 16:01:56 -04001763EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001764
Ingo Molnare309b412008-05-12 21:20:51 +02001765void
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001766trace_function(struct trace_array *tr,
Steven Rostedt38697052008-10-01 13:14:09 -04001767 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1768 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001769{
Steven Rostedt (Red Hat)2425bcb2015-05-05 11:45:27 -04001770 struct trace_event_call *call = &event_function;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001771 struct ring_buffer *buffer = tr->trace_buffer.buffer;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001772 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001773 struct ftrace_entry *entry;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001774
Steven Rostedtd7690412008-10-01 00:29:53 -04001775 /* If we are reading the ring buffer, don't trace */
Rusty Russelldd17c8f2009-10-29 22:34:15 +09001776 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
Steven Rostedtd7690412008-10-01 00:29:53 -04001777 return;
1778
Steven Rostedte77405a2009-09-02 14:17:06 -04001779 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001780 flags, pc);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001781 if (!event)
1782 return;
1783 entry = ring_buffer_event_data(event);
Steven Rostedt777e2082008-09-29 23:02:42 -04001784 entry->ip = ip;
1785 entry->parent_ip = parent_ip;
Tom Zanussie1112b42009-03-31 00:48:49 -05001786
Tom Zanussif306cc82013-10-24 08:34:17 -05001787 if (!call_filter_check_discard(call, entry, buffer, event))
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001788 __buffer_unlock_commit(buffer, event);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001789}
1790
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001791#ifdef CONFIG_STACKTRACE
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001792
1793#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1794struct ftrace_stack {
1795 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1796};
1797
1798static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1799static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1800
Steven Rostedte77405a2009-09-02 14:17:06 -04001801static void __ftrace_trace_stack(struct ring_buffer *buffer,
Steven Rostedt53614992009-01-15 19:12:40 -05001802 unsigned long flags,
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001803 int skip, int pc, struct pt_regs *regs)
Ingo Molnar86387f72008-05-12 21:20:51 +02001804{
Steven Rostedt (Red Hat)2425bcb2015-05-05 11:45:27 -04001805 struct trace_event_call *call = &event_kernel_stack;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001806 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001807 struct stack_entry *entry;
Ingo Molnar86387f72008-05-12 21:20:51 +02001808 struct stack_trace trace;
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001809 int use_stack;
1810 int size = FTRACE_STACK_ENTRIES;
Ingo Molnar86387f72008-05-12 21:20:51 +02001811
1812 trace.nr_entries = 0;
Ingo Molnar86387f72008-05-12 21:20:51 +02001813 trace.skip = skip;
Ingo Molnar86387f72008-05-12 21:20:51 +02001814
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001815 /*
1816 * Since events can happen in NMIs there's no safe way to
1817 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1818 * or NMI comes in, it will just have to use the default
1819 * FTRACE_STACK_SIZE.
1820 */
1821 preempt_disable_notrace();
1822
Shan Wei82146522012-11-19 13:21:01 +08001823 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001824 /*
1825 * We don't need any atomic variables, just a barrier.
1826 * If an interrupt comes in, we don't care, because it would
1827 * have exited and put the counter back to what we want.
1828 * We just need a barrier to keep gcc from moving things
1829 * around.
1830 */
1831 barrier();
1832 if (use_stack == 1) {
Christoph Lameterbdffd892014-04-29 14:17:40 -05001833 trace.entries = this_cpu_ptr(ftrace_stack.calls);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001834 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1835
1836 if (regs)
1837 save_stack_trace_regs(regs, &trace);
1838 else
1839 save_stack_trace(&trace);
1840
1841 if (trace.nr_entries > size)
1842 size = trace.nr_entries;
1843 } else
1844 /* From now on, use_stack is a boolean */
1845 use_stack = 0;
1846
1847 size *= sizeof(unsigned long);
1848
1849 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1850 sizeof(*entry) + size, flags, pc);
1851 if (!event)
1852 goto out;
1853 entry = ring_buffer_event_data(event);
1854
1855 memset(&entry->caller, 0, size);
1856
1857 if (use_stack)
1858 memcpy(&entry->caller, trace.entries,
1859 trace.nr_entries * sizeof(unsigned long));
1860 else {
1861 trace.max_entries = FTRACE_STACK_ENTRIES;
1862 trace.entries = entry->caller;
1863 if (regs)
1864 save_stack_trace_regs(regs, &trace);
1865 else
1866 save_stack_trace(&trace);
1867 }
1868
1869 entry->size = trace.nr_entries;
1870
Tom Zanussif306cc82013-10-24 08:34:17 -05001871 if (!call_filter_check_discard(call, entry, buffer, event))
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001872 __buffer_unlock_commit(buffer, event);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001873
1874 out:
1875 /* Again, don't let gcc optimize things here */
1876 barrier();
Shan Wei82146522012-11-19 13:21:01 +08001877 __this_cpu_dec(ftrace_stack_reserve);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001878 preempt_enable_notrace();
1879
Ingo Molnarf0a920d2008-05-12 21:20:47 +02001880}
1881
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -04001882static inline void ftrace_trace_stack(struct trace_array *tr,
1883 struct ring_buffer *buffer,
Steven Rostedt (Red Hat)73dddbb2015-09-29 15:38:55 -04001884 unsigned long flags,
1885 int skip, int pc, struct pt_regs *regs)
Steven Rostedt53614992009-01-15 19:12:40 -05001886{
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -04001887 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
Steven Rostedt53614992009-01-15 19:12:40 -05001888 return;
1889
Steven Rostedt (Red Hat)73dddbb2015-09-29 15:38:55 -04001890 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
Steven Rostedt53614992009-01-15 19:12:40 -05001891}
1892
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001893void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1894 int pc)
Steven Rostedt38697052008-10-01 13:14:09 -04001895{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001896 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
Steven Rostedt38697052008-10-01 13:14:09 -04001897}
1898
Steven Rostedt03889382009-12-11 09:48:22 -05001899/**
1900 * trace_dump_stack - record a stack back trace in the trace buffer
Steven Rostedt (Red Hat)c142be82013-03-13 09:55:57 -04001901 * @skip: Number of functions to skip (helper handlers)
Steven Rostedt03889382009-12-11 09:48:22 -05001902 */
Steven Rostedt (Red Hat)c142be82013-03-13 09:55:57 -04001903void trace_dump_stack(int skip)
Steven Rostedt03889382009-12-11 09:48:22 -05001904{
1905 unsigned long flags;
1906
1907 if (tracing_disabled || tracing_selftest_running)
Steven Rostedte36c5452009-12-14 15:58:33 -05001908 return;
Steven Rostedt03889382009-12-11 09:48:22 -05001909
1910 local_save_flags(flags);
1911
Steven Rostedt (Red Hat)c142be82013-03-13 09:55:57 -04001912 /*
1913 * Skip 3 more, seems to get us at the caller of
1914 * this function.
1915 */
1916 skip += 3;
1917 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1918 flags, skip, preempt_count(), NULL);
Steven Rostedt03889382009-12-11 09:48:22 -05001919}
1920
Steven Rostedt91e86e52010-11-10 12:56:12 +01001921static DEFINE_PER_CPU(int, user_stack_count);
1922
Steven Rostedte77405a2009-09-02 14:17:06 -04001923void
1924ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
Török Edwin02b67512008-11-22 13:28:47 +02001925{
Steven Rostedt (Red Hat)2425bcb2015-05-05 11:45:27 -04001926 struct trace_event_call *call = &event_user_stack;
Török Edwin8d7c6a92008-11-23 12:39:06 +02001927 struct ring_buffer_event *event;
Török Edwin02b67512008-11-22 13:28:47 +02001928 struct userstack_entry *entry;
1929 struct stack_trace trace;
Török Edwin02b67512008-11-22 13:28:47 +02001930
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04001931 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
Török Edwin02b67512008-11-22 13:28:47 +02001932 return;
1933
Steven Rostedtb6345872010-03-12 20:03:30 -05001934 /*
1935 * NMIs can not handle page faults, even with fix ups.
1936 * The save user stack can (and often does) fault.
1937 */
1938 if (unlikely(in_nmi()))
1939 return;
1940
Steven Rostedt91e86e52010-11-10 12:56:12 +01001941 /*
1942 * prevent recursion, since the user stack tracing may
1943 * trigger other kernel events.
1944 */
1945 preempt_disable();
1946 if (__this_cpu_read(user_stack_count))
1947 goto out;
1948
1949 __this_cpu_inc(user_stack_count);
1950
Steven Rostedte77405a2009-09-02 14:17:06 -04001951 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001952 sizeof(*entry), flags, pc);
Török Edwin02b67512008-11-22 13:28:47 +02001953 if (!event)
Li Zefan1dbd1952010-12-09 15:47:56 +08001954 goto out_drop_count;
Török Edwin02b67512008-11-22 13:28:47 +02001955 entry = ring_buffer_event_data(event);
Török Edwin02b67512008-11-22 13:28:47 +02001956
Steven Rostedt48659d32009-09-11 11:36:23 -04001957 entry->tgid = current->tgid;
Török Edwin02b67512008-11-22 13:28:47 +02001958 memset(&entry->caller, 0, sizeof(entry->caller));
1959
1960 trace.nr_entries = 0;
1961 trace.max_entries = FTRACE_STACK_ENTRIES;
1962 trace.skip = 0;
1963 trace.entries = entry->caller;
1964
1965 save_stack_trace_user(&trace);
Tom Zanussif306cc82013-10-24 08:34:17 -05001966 if (!call_filter_check_discard(call, entry, buffer, event))
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001967 __buffer_unlock_commit(buffer, event);
Steven Rostedt91e86e52010-11-10 12:56:12 +01001968
Li Zefan1dbd1952010-12-09 15:47:56 +08001969 out_drop_count:
Steven Rostedt91e86e52010-11-10 12:56:12 +01001970 __this_cpu_dec(user_stack_count);
Steven Rostedt91e86e52010-11-10 12:56:12 +01001971 out:
1972 preempt_enable();
Török Edwin02b67512008-11-22 13:28:47 +02001973}
1974
Hannes Eder4fd27352009-02-10 19:44:12 +01001975#ifdef UNUSED
1976static void __trace_userstack(struct trace_array *tr, unsigned long flags)
Török Edwin02b67512008-11-22 13:28:47 +02001977{
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001978 ftrace_trace_userstack(tr, flags, preempt_count());
Török Edwin02b67512008-11-22 13:28:47 +02001979}
Hannes Eder4fd27352009-02-10 19:44:12 +01001980#endif /* UNUSED */
Török Edwin02b67512008-11-22 13:28:47 +02001981
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001982#endif /* CONFIG_STACKTRACE */
1983
Steven Rostedt07d777f2011-09-22 14:01:55 -04001984/* created for use with alloc_percpu */
1985struct trace_buffer_struct {
1986 char buffer[TRACE_BUF_SIZE];
1987};
1988
1989static struct trace_buffer_struct *trace_percpu_buffer;
1990static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1991static struct trace_buffer_struct *trace_percpu_irq_buffer;
1992static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1993
1994/*
1995 * The buffer used is dependent on the context. There is a per cpu
1996 * buffer for normal context, softirq contex, hard irq context and
1997 * for NMI context. Thise allows for lockless recording.
1998 *
1999 * Note, if the buffers failed to be allocated, then this returns NULL
2000 */
2001static char *get_trace_buf(void)
2002{
2003 struct trace_buffer_struct *percpu_buffer;
Steven Rostedt07d777f2011-09-22 14:01:55 -04002004
2005 /*
2006 * If we have allocated per cpu buffers, then we do not
2007 * need to do any locking.
2008 */
2009 if (in_nmi())
2010 percpu_buffer = trace_percpu_nmi_buffer;
2011 else if (in_irq())
2012 percpu_buffer = trace_percpu_irq_buffer;
2013 else if (in_softirq())
2014 percpu_buffer = trace_percpu_sirq_buffer;
2015 else
2016 percpu_buffer = trace_percpu_buffer;
2017
2018 if (!percpu_buffer)
2019 return NULL;
2020
Shan Weid8a03492012-11-13 09:53:04 +08002021 return this_cpu_ptr(&percpu_buffer->buffer[0]);
Steven Rostedt07d777f2011-09-22 14:01:55 -04002022}
2023
2024static int alloc_percpu_trace_buffer(void)
2025{
2026 struct trace_buffer_struct *buffers;
2027 struct trace_buffer_struct *sirq_buffers;
2028 struct trace_buffer_struct *irq_buffers;
2029 struct trace_buffer_struct *nmi_buffers;
2030
2031 buffers = alloc_percpu(struct trace_buffer_struct);
2032 if (!buffers)
2033 goto err_warn;
2034
2035 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
2036 if (!sirq_buffers)
2037 goto err_sirq;
2038
2039 irq_buffers = alloc_percpu(struct trace_buffer_struct);
2040 if (!irq_buffers)
2041 goto err_irq;
2042
2043 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
2044 if (!nmi_buffers)
2045 goto err_nmi;
2046
2047 trace_percpu_buffer = buffers;
2048 trace_percpu_sirq_buffer = sirq_buffers;
2049 trace_percpu_irq_buffer = irq_buffers;
2050 trace_percpu_nmi_buffer = nmi_buffers;
2051
2052 return 0;
2053
2054 err_nmi:
2055 free_percpu(irq_buffers);
2056 err_irq:
2057 free_percpu(sirq_buffers);
2058 err_sirq:
2059 free_percpu(buffers);
2060 err_warn:
2061 WARN(1, "Could not allocate percpu trace_printk buffer");
2062 return -ENOMEM;
2063}
2064
Steven Rostedt81698832012-10-11 10:15:05 -04002065static int buffers_allocated;
2066
Steven Rostedt07d777f2011-09-22 14:01:55 -04002067void trace_printk_init_buffers(void)
2068{
Steven Rostedt07d777f2011-09-22 14:01:55 -04002069 if (buffers_allocated)
2070 return;
2071
2072 if (alloc_percpu_trace_buffer())
2073 return;
2074
Steven Rostedt2184db42014-05-28 13:14:40 -04002075 /* trace_printk() is for debug use only. Don't use it in production. */
2076
Borislav Petkov69a1c992015-01-27 17:17:20 +01002077 pr_warning("\n");
2078 pr_warning("**********************************************************\n");
Steven Rostedt2184db42014-05-28 13:14:40 -04002079 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2080 pr_warning("** **\n");
2081 pr_warning("** trace_printk() being used. Allocating extra memory. **\n");
2082 pr_warning("** **\n");
2083 pr_warning("** This means that this is a DEBUG kernel and it is **\n");
Frans Klavereff264e2014-11-07 15:53:44 +01002084 pr_warning("** unsafe for production use. **\n");
Steven Rostedt2184db42014-05-28 13:14:40 -04002085 pr_warning("** **\n");
2086 pr_warning("** If you see this message and you are not debugging **\n");
2087 pr_warning("** the kernel, report this immediately to your vendor! **\n");
2088 pr_warning("** **\n");
2089 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2090 pr_warning("**********************************************************\n");
Steven Rostedt07d777f2011-09-22 14:01:55 -04002091
Steven Rostedtb382ede62012-10-10 21:44:34 -04002092 /* Expand the buffers to set size */
2093 tracing_update_buffers();
2094
Steven Rostedt07d777f2011-09-22 14:01:55 -04002095 buffers_allocated = 1;
Steven Rostedt81698832012-10-11 10:15:05 -04002096
2097 /*
2098 * trace_printk_init_buffers() can be called by modules.
2099 * If that happens, then we need to start cmdline recording
2100 * directly here. If the global_trace.buffer is already
2101 * allocated here, then this was called by module code.
2102 */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002103 if (global_trace.trace_buffer.buffer)
Steven Rostedt81698832012-10-11 10:15:05 -04002104 tracing_start_cmdline_record();
2105}
2106
2107void trace_printk_start_comm(void)
2108{
2109 /* Start tracing comms if trace printk is set */
2110 if (!buffers_allocated)
2111 return;
2112 tracing_start_cmdline_record();
2113}
2114
2115static void trace_printk_start_stop_comm(int enabled)
2116{
2117 if (!buffers_allocated)
2118 return;
2119
2120 if (enabled)
2121 tracing_start_cmdline_record();
2122 else
2123 tracing_stop_cmdline_record();
Steven Rostedt07d777f2011-09-22 14:01:55 -04002124}
2125
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002126/**
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002127 * trace_vbprintk - write binary msg to tracing buffer
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002128 *
2129 */
Steven Rostedt40ce74f2009-03-19 14:03:53 -04002130int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002131{
Steven Rostedt (Red Hat)2425bcb2015-05-05 11:45:27 -04002132 struct trace_event_call *call = &event_bprint;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002133 struct ring_buffer_event *event;
Steven Rostedte77405a2009-09-02 14:17:06 -04002134 struct ring_buffer *buffer;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002135 struct trace_array *tr = &global_trace;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002136 struct bprint_entry *entry;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002137 unsigned long flags;
Steven Rostedt07d777f2011-09-22 14:01:55 -04002138 char *tbuffer;
2139 int len = 0, size, pc;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002140
2141 if (unlikely(tracing_selftest_running || tracing_disabled))
2142 return 0;
2143
2144 /* Don't pollute graph traces with trace_vprintk internals */
2145 pause_graph_tracing();
2146
2147 pc = preempt_count();
Steven Rostedt5168ae52010-06-03 09:36:50 -04002148 preempt_disable_notrace();
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002149
Steven Rostedt07d777f2011-09-22 14:01:55 -04002150 tbuffer = get_trace_buf();
2151 if (!tbuffer) {
2152 len = 0;
2153 goto out;
2154 }
2155
2156 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2157
2158 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002159 goto out;
2160
Steven Rostedt07d777f2011-09-22 14:01:55 -04002161 local_save_flags(flags);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002162 size = sizeof(*entry) + sizeof(u32) * len;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002163 buffer = tr->trace_buffer.buffer;
Steven Rostedte77405a2009-09-02 14:17:06 -04002164 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2165 flags, pc);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002166 if (!event)
Steven Rostedt07d777f2011-09-22 14:01:55 -04002167 goto out;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002168 entry = ring_buffer_event_data(event);
2169 entry->ip = ip;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002170 entry->fmt = fmt;
2171
Steven Rostedt07d777f2011-09-22 14:01:55 -04002172 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
Tom Zanussif306cc82013-10-24 08:34:17 -05002173 if (!call_filter_check_discard(call, entry, buffer, event)) {
Steven Rostedt7ffbd482012-10-11 12:14:25 -04002174 __buffer_unlock_commit(buffer, event);
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -04002175 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
Steven Rostedtd9313692010-01-06 17:27:11 -05002176 }
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002177
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002178out:
Steven Rostedt5168ae52010-06-03 09:36:50 -04002179 preempt_enable_notrace();
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002180 unpause_graph_tracing();
2181
2182 return len;
2183}
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002184EXPORT_SYMBOL_GPL(trace_vbprintk);
2185
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002186static int
2187__trace_array_vprintk(struct ring_buffer *buffer,
2188 unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002189{
Steven Rostedt (Red Hat)2425bcb2015-05-05 11:45:27 -04002190 struct trace_event_call *call = &event_print;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002191 struct ring_buffer_event *event;
Steven Rostedt07d777f2011-09-22 14:01:55 -04002192 int len = 0, size, pc;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002193 struct print_entry *entry;
Steven Rostedt07d777f2011-09-22 14:01:55 -04002194 unsigned long flags;
2195 char *tbuffer;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002196
2197 if (tracing_disabled || tracing_selftest_running)
2198 return 0;
2199
Steven Rostedt07d777f2011-09-22 14:01:55 -04002200 /* Don't pollute graph traces with trace_vprintk internals */
2201 pause_graph_tracing();
2202
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002203 pc = preempt_count();
2204 preempt_disable_notrace();
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002205
Steven Rostedt07d777f2011-09-22 14:01:55 -04002206
2207 tbuffer = get_trace_buf();
2208 if (!tbuffer) {
2209 len = 0;
2210 goto out;
2211 }
2212
Dan Carpenter3558a5a2014-11-27 18:57:52 +03002213 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002214
Steven Rostedt07d777f2011-09-22 14:01:55 -04002215 local_save_flags(flags);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002216 size = sizeof(*entry) + len + 1;
Steven Rostedte77405a2009-09-02 14:17:06 -04002217 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
Steven Rostedt07d777f2011-09-22 14:01:55 -04002218 flags, pc);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002219 if (!event)
Steven Rostedt07d777f2011-09-22 14:01:55 -04002220 goto out;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002221 entry = ring_buffer_event_data(event);
Carsten Emdec13d2f72009-11-16 20:56:13 +01002222 entry->ip = ip;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002223
Dan Carpenter3558a5a2014-11-27 18:57:52 +03002224 memcpy(&entry->buf, tbuffer, len + 1);
Tom Zanussif306cc82013-10-24 08:34:17 -05002225 if (!call_filter_check_discard(call, entry, buffer, event)) {
Steven Rostedt7ffbd482012-10-11 12:14:25 -04002226 __buffer_unlock_commit(buffer, event);
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -04002227 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
Steven Rostedtd9313692010-01-06 17:27:11 -05002228 }
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002229 out:
2230 preempt_enable_notrace();
Steven Rostedt07d777f2011-09-22 14:01:55 -04002231 unpause_graph_tracing();
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002232
2233 return len;
2234}
Steven Rostedt659372d2009-09-03 19:11:07 -04002235
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002236int trace_array_vprintk(struct trace_array *tr,
2237 unsigned long ip, const char *fmt, va_list args)
2238{
2239 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2240}
2241
2242int trace_array_printk(struct trace_array *tr,
2243 unsigned long ip, const char *fmt, ...)
2244{
2245 int ret;
2246 va_list ap;
2247
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002248 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002249 return 0;
2250
2251 va_start(ap, fmt);
2252 ret = trace_array_vprintk(tr, ip, fmt, ap);
2253 va_end(ap);
2254 return ret;
2255}
2256
2257int trace_array_printk_buf(struct ring_buffer *buffer,
2258 unsigned long ip, const char *fmt, ...)
2259{
2260 int ret;
2261 va_list ap;
2262
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002263 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002264 return 0;
2265
2266 va_start(ap, fmt);
2267 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2268 va_end(ap);
2269 return ret;
2270}
2271
Steven Rostedt659372d2009-09-03 19:11:07 -04002272int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2273{
Steven Rostedta813a152009-10-09 01:41:35 -04002274 return trace_array_vprintk(&global_trace, ip, fmt, args);
Steven Rostedt659372d2009-09-03 19:11:07 -04002275}
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002276EXPORT_SYMBOL_GPL(trace_vprintk);
2277
Robert Richtere2ac8ef2008-11-12 12:59:32 +01002278static void trace_iterator_increment(struct trace_iterator *iter)
Steven Rostedt5a90f572008-09-03 17:42:51 -04002279{
Steven Rostedt6d158a82012-06-27 20:46:14 -04002280 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2281
Steven Rostedt5a90f572008-09-03 17:42:51 -04002282 iter->idx++;
Steven Rostedt6d158a82012-06-27 20:46:14 -04002283 if (buf_iter)
2284 ring_buffer_read(buf_iter, NULL);
Steven Rostedt5a90f572008-09-03 17:42:51 -04002285}
2286
Ingo Molnare309b412008-05-12 21:20:51 +02002287static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04002288peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2289 unsigned long *lost_events)
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002290{
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002291 struct ring_buffer_event *event;
Steven Rostedt6d158a82012-06-27 20:46:14 -04002292 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002293
Steven Rostedtd7690412008-10-01 00:29:53 -04002294 if (buf_iter)
2295 event = ring_buffer_iter_peek(buf_iter, ts);
2296 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002297 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
Steven Rostedtbc21b472010-03-31 19:49:26 -04002298 lost_events);
Steven Rostedtd7690412008-10-01 00:29:53 -04002299
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04002300 if (event) {
2301 iter->ent_size = ring_buffer_event_length(event);
2302 return ring_buffer_event_data(event);
2303 }
2304 iter->ent_size = 0;
2305 return NULL;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002306}
Steven Rostedtd7690412008-10-01 00:29:53 -04002307
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002308static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04002309__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2310 unsigned long *missing_events, u64 *ent_ts)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002311{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002312 struct ring_buffer *buffer = iter->trace_buffer->buffer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002313 struct trace_entry *ent, *next = NULL;
Lai Jiangshanaa274972010-04-05 17:11:05 +08002314 unsigned long lost_events = 0, next_lost = 0;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002315 int cpu_file = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002316 u64 next_ts = 0, ts;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002317 int next_cpu = -1;
Steven Rostedt12b5da32012-03-27 10:43:28 -04002318 int next_size = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002319 int cpu;
2320
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002321 /*
2322 * If we are in a per_cpu trace file, don't bother by iterating over
2323 * all cpu and peek directly.
2324 */
Steven Rostedtae3b5092013-01-23 15:22:59 -05002325 if (cpu_file > RING_BUFFER_ALL_CPUS) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002326 if (ring_buffer_empty_cpu(buffer, cpu_file))
2327 return NULL;
Steven Rostedtbc21b472010-03-31 19:49:26 -04002328 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002329 if (ent_cpu)
2330 *ent_cpu = cpu_file;
2331
2332 return ent;
2333 }
2334
Steven Rostedtab464282008-05-12 21:21:00 +02002335 for_each_tracing_cpu(cpu) {
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002336
2337 if (ring_buffer_empty_cpu(buffer, cpu))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002338 continue;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002339
Steven Rostedtbc21b472010-03-31 19:49:26 -04002340 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002341
Ingo Molnarcdd31cd2008-05-12 21:20:46 +02002342 /*
2343 * Pick the entry with the smallest timestamp:
2344 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002345 if (ent && (!next || ts < next_ts)) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002346 next = ent;
2347 next_cpu = cpu;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002348 next_ts = ts;
Steven Rostedtbc21b472010-03-31 19:49:26 -04002349 next_lost = lost_events;
Steven Rostedt12b5da32012-03-27 10:43:28 -04002350 next_size = iter->ent_size;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002351 }
2352 }
2353
Steven Rostedt12b5da32012-03-27 10:43:28 -04002354 iter->ent_size = next_size;
2355
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002356 if (ent_cpu)
2357 *ent_cpu = next_cpu;
2358
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002359 if (ent_ts)
2360 *ent_ts = next_ts;
2361
Steven Rostedtbc21b472010-03-31 19:49:26 -04002362 if (missing_events)
2363 *missing_events = next_lost;
2364
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002365 return next;
2366}
2367
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002368/* Find the next real entry, without updating the iterator itself */
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002369struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2370 int *ent_cpu, u64 *ent_ts)
Steven Rostedtb3806b42008-05-12 21:20:46 +02002371{
Steven Rostedtbc21b472010-03-31 19:49:26 -04002372 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002373}
Ingo Molnar8c523a92008-05-12 21:20:46 +02002374
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002375/* Find the next real entry, and increment the iterator to the next entry */
Jason Wessel955b61e2010-08-05 09:22:23 -05002376void *trace_find_next_entry_inc(struct trace_iterator *iter)
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002377{
Steven Rostedtbc21b472010-03-31 19:49:26 -04002378 iter->ent = __find_next_entry(iter, &iter->cpu,
2379 &iter->lost_events, &iter->ts);
Steven Rostedtb3806b42008-05-12 21:20:46 +02002380
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002381 if (iter->ent)
Robert Richtere2ac8ef2008-11-12 12:59:32 +01002382 trace_iterator_increment(iter);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002383
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002384 return iter->ent ? iter : NULL;
Steven Rostedtb3806b42008-05-12 21:20:46 +02002385}
2386
Ingo Molnare309b412008-05-12 21:20:51 +02002387static void trace_consume(struct trace_iterator *iter)
Steven Rostedtb3806b42008-05-12 21:20:46 +02002388{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002389 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
Steven Rostedtbc21b472010-03-31 19:49:26 -04002390 &iter->lost_events);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002391}
2392
Ingo Molnare309b412008-05-12 21:20:51 +02002393static void *s_next(struct seq_file *m, void *v, loff_t *pos)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002394{
2395 struct trace_iterator *iter = m->private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002396 int i = (int)*pos;
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002397 void *ent;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002398
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002399 WARN_ON_ONCE(iter->leftover);
2400
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002401 (*pos)++;
2402
2403 /* can't go backwards */
2404 if (iter->idx > i)
2405 return NULL;
2406
2407 if (iter->idx < 0)
Jason Wessel955b61e2010-08-05 09:22:23 -05002408 ent = trace_find_next_entry_inc(iter);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002409 else
2410 ent = iter;
2411
2412 while (ent && iter->idx < i)
Jason Wessel955b61e2010-08-05 09:22:23 -05002413 ent = trace_find_next_entry_inc(iter);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002414
2415 iter->pos = *pos;
2416
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002417 return ent;
2418}
2419
Jason Wessel955b61e2010-08-05 09:22:23 -05002420void tracing_iter_reset(struct trace_iterator *iter, int cpu)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002421{
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002422 struct ring_buffer_event *event;
2423 struct ring_buffer_iter *buf_iter;
2424 unsigned long entries = 0;
2425 u64 ts;
2426
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002427 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002428
Steven Rostedt6d158a82012-06-27 20:46:14 -04002429 buf_iter = trace_buffer_iter(iter, cpu);
2430 if (!buf_iter)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002431 return;
2432
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002433 ring_buffer_iter_reset(buf_iter);
2434
2435 /*
2436 * We could have the case with the max latency tracers
2437 * that a reset never took place on a cpu. This is evident
2438 * by the timestamp being before the start of the buffer.
2439 */
2440 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002441 if (ts >= iter->trace_buffer->time_start)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002442 break;
2443 entries++;
2444 ring_buffer_read(buf_iter, NULL);
2445 }
2446
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002447 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002448}
2449
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002450/*
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002451 * The current tracer is copied to avoid a global locking
2452 * all around.
2453 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002454static void *s_start(struct seq_file *m, loff_t *pos)
2455{
2456 struct trace_iterator *iter = m->private;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002457 struct trace_array *tr = iter->tr;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002458 int cpu_file = iter->cpu_file;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002459 void *p = NULL;
2460 loff_t l = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002461 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002462
Hiraku Toyooka2fd196e2012-12-26 11:52:52 +09002463 /*
2464 * copy the tracer to avoid using a global lock all around.
2465 * iter->trace is a copy of current_trace, the pointer to the
2466 * name may be used instead of a strcmp(), as iter->trace->name
2467 * will point to the same string as current_trace->name.
2468 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002469 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002470 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2471 *iter->trace = *tr->current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002472 mutex_unlock(&trace_types_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002473
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002474#ifdef CONFIG_TRACER_MAX_TRACE
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002475 if (iter->snapshot && iter->trace->use_max_tr)
2476 return ERR_PTR(-EBUSY);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002477#endif
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002478
2479 if (!iter->snapshot)
2480 atomic_inc(&trace_record_cmdline_disabled);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002481
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002482 if (*pos != iter->pos) {
2483 iter->ent = NULL;
2484 iter->cpu = 0;
2485 iter->idx = -1;
2486
Steven Rostedtae3b5092013-01-23 15:22:59 -05002487 if (cpu_file == RING_BUFFER_ALL_CPUS) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002488 for_each_tracing_cpu(cpu)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002489 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002490 } else
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002491 tracing_iter_reset(iter, cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002492
Lai Jiangshanac91d852010-03-02 17:54:50 +08002493 iter->leftover = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002494 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2495 ;
2496
2497 } else {
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002498 /*
2499 * If we overflowed the seq_file before, then we want
2500 * to just reuse the trace_seq buffer again.
2501 */
2502 if (iter->leftover)
2503 p = iter;
2504 else {
2505 l = *pos - 1;
2506 p = s_next(m, p, &l);
2507 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002508 }
2509
Lai Jiangshan4f535962009-05-18 19:35:34 +08002510 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08002511 trace_access_lock(cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002512 return p;
2513}
2514
2515static void s_stop(struct seq_file *m, void *p)
2516{
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08002517 struct trace_iterator *iter = m->private;
2518
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002519#ifdef CONFIG_TRACER_MAX_TRACE
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002520 if (iter->snapshot && iter->trace->use_max_tr)
2521 return;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002522#endif
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002523
2524 if (!iter->snapshot)
2525 atomic_dec(&trace_record_cmdline_disabled);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002526
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08002527 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08002528 trace_event_read_unlock();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002529}
2530
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002531static void
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002532get_total_entries(struct trace_buffer *buf,
2533 unsigned long *total, unsigned long *entries)
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002534{
2535 unsigned long count;
2536 int cpu;
2537
2538 *total = 0;
2539 *entries = 0;
2540
2541 for_each_tracing_cpu(cpu) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002542 count = ring_buffer_entries_cpu(buf->buffer, cpu);
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002543 /*
2544 * If this buffer has skipped entries, then we hold all
2545 * entries for the trace and we need to ignore the
2546 * ones before the time stamp.
2547 */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002548 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2549 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002550 /* total is the same as the entries */
2551 *total += count;
2552 } else
2553 *total += count +
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002554 ring_buffer_overrun_cpu(buf->buffer, cpu);
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002555 *entries += count;
2556 }
2557}
2558
Ingo Molnare309b412008-05-12 21:20:51 +02002559static void print_lat_help_header(struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002560{
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002561 seq_puts(m, "# _------=> CPU# \n"
2562 "# / _-----=> irqs-off \n"
2563 "# | / _----=> need-resched \n"
2564 "# || / _---=> hardirq/softirq \n"
2565 "# ||| / _--=> preempt-depth \n"
2566 "# |||| / delay \n"
2567 "# cmd pid ||||| time | caller \n"
2568 "# \\ / ||||| \\ | / \n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002569}
2570
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002571static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002572{
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002573 unsigned long total;
2574 unsigned long entries;
2575
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002576 get_total_entries(buf, &total, &entries);
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002577 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2578 entries, total, num_online_cpus());
2579 seq_puts(m, "#\n");
2580}
2581
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002582static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002583{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002584 print_event_info(buf, m);
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002585 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2586 "# | | | | |\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002587}
2588
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002589static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
Steven Rostedt77271ce2011-11-17 09:34:33 -05002590{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002591 print_event_info(buf, m);
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002592 seq_puts(m, "# _-----=> irqs-off\n"
2593 "# / _----=> need-resched\n"
2594 "# | / _---=> hardirq/softirq\n"
2595 "# || / _--=> preempt-depth\n"
2596 "# ||| / delay\n"
2597 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
2598 "# | | | |||| | |\n");
Steven Rostedt77271ce2011-11-17 09:34:33 -05002599}
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002600
Jiri Olsa62b915f2010-04-02 19:01:22 +02002601void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002602print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2603{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002604 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002605 struct trace_buffer *buf = iter->trace_buffer;
2606 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002607 struct tracer *type = iter->trace;
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002608 unsigned long entries;
2609 unsigned long total;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002610 const char *name = "preemption";
2611
Steven Rostedt (Red Hat)d840f712013-02-01 18:38:47 -05002612 name = type->name;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002613
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002614 get_total_entries(buf, &total, &entries);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002615
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002616 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002617 name, UTS_RELEASE);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002618 seq_puts(m, "# -----------------------------------"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002619 "---------------------------------\n");
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002620 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002621 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
Steven Rostedt57f50be2008-05-12 21:20:44 +02002622 nsecs_to_usecs(data->saved_latency),
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002623 entries,
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02002624 total,
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002625 buf->cpu,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002626#if defined(CONFIG_PREEMPT_NONE)
2627 "server",
2628#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2629 "desktop",
Steven Rostedtb5c21b42008-07-10 20:58:12 -04002630#elif defined(CONFIG_PREEMPT)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002631 "preempt",
2632#else
2633 "unknown",
2634#endif
2635 /* These are reserved for later use */
2636 0, 0, 0, 0);
2637#ifdef CONFIG_SMP
2638 seq_printf(m, " #P:%d)\n", num_online_cpus());
2639#else
2640 seq_puts(m, ")\n");
2641#endif
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002642 seq_puts(m, "# -----------------\n");
2643 seq_printf(m, "# | task: %.16s-%d "
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002644 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
Eric W. Biedermand20b92a2012-03-13 16:02:19 -07002645 data->comm, data->pid,
2646 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002647 data->policy, data->rt_priority);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002648 seq_puts(m, "# -----------------\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002649
2650 if (data->critical_start) {
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002651 seq_puts(m, "# => started at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02002652 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2653 trace_print_seq(m, &iter->seq);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002654 seq_puts(m, "\n# => ended at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02002655 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2656 trace_print_seq(m, &iter->seq);
Steven Rostedt8248ac02009-09-02 12:27:41 -04002657 seq_puts(m, "\n#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002658 }
2659
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002660 seq_puts(m, "#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002661}
2662
Steven Rostedta3097202008-11-07 22:36:02 -05002663static void test_cpu_buff_start(struct trace_iterator *iter)
2664{
2665 struct trace_seq *s = &iter->seq;
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002666 struct trace_array *tr = iter->tr;
Steven Rostedta3097202008-11-07 22:36:02 -05002667
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002668 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
Steven Rostedt12ef7d42008-11-12 17:52:38 -05002669 return;
2670
2671 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2672 return;
2673
Rusty Russell44623442009-01-01 10:12:23 +10302674 if (cpumask_test_cpu(iter->cpu, iter->started))
Steven Rostedta3097202008-11-07 22:36:02 -05002675 return;
2676
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002677 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002678 return;
2679
Rusty Russell44623442009-01-01 10:12:23 +10302680 cpumask_set_cpu(iter->cpu, iter->started);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002681
2682 /* Don't print started cpu buffer for the first entry of the trace */
2683 if (iter->idx > 1)
2684 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2685 iter->cpu);
Steven Rostedta3097202008-11-07 22:36:02 -05002686}
2687
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002688static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002689{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002690 struct trace_array *tr = iter->tr;
Steven Rostedt214023c2008-05-12 21:20:46 +02002691 struct trace_seq *s = &iter->seq;
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002692 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002693 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002694 struct trace_event *event;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002695
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002696 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002697
Steven Rostedta3097202008-11-07 22:36:02 -05002698 test_cpu_buff_start(iter);
2699
Steven Rostedtf633cef2008-12-23 23:24:13 -05002700 event = ftrace_find_event(entry->type);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002701
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002702 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002703 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2704 trace_print_lat_context(iter);
2705 else
2706 trace_print_context(iter);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002707 }
2708
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002709 if (trace_seq_has_overflowed(s))
2710 return TRACE_TYPE_PARTIAL_LINE;
2711
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002712 if (event)
Steven Rostedta9a57762010-04-22 18:46:14 -04002713 return event->funcs->trace(iter, sym_flags, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002714
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002715 trace_seq_printf(s, "Unknown type %d\n", entry->type);
Steven Rostedt7104f302008-10-01 10:52:51 -04002716
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002717 return trace_handle_return(s);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002718}
2719
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002720static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002721{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002722 struct trace_array *tr = iter->tr;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002723 struct trace_seq *s = &iter->seq;
2724 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002725 struct trace_event *event;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002726
2727 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002728
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002729 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002730 trace_seq_printf(s, "%d %d %llu ",
2731 entry->pid, iter->cpu, iter->ts);
2732
2733 if (trace_seq_has_overflowed(s))
2734 return TRACE_TYPE_PARTIAL_LINE;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002735
Steven Rostedtf633cef2008-12-23 23:24:13 -05002736 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002737 if (event)
Steven Rostedta9a57762010-04-22 18:46:14 -04002738 return event->funcs->raw(iter, 0, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002739
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002740 trace_seq_printf(s, "%d ?\n", entry->type);
Steven Rostedt7104f302008-10-01 10:52:51 -04002741
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002742 return trace_handle_return(s);
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002743}
2744
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002745static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002746{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002747 struct trace_array *tr = iter->tr;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002748 struct trace_seq *s = &iter->seq;
2749 unsigned char newline = '\n';
2750 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002751 struct trace_event *event;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002752
2753 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002754
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002755 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002756 SEQ_PUT_HEX_FIELD(s, entry->pid);
2757 SEQ_PUT_HEX_FIELD(s, iter->cpu);
2758 SEQ_PUT_HEX_FIELD(s, iter->ts);
2759 if (trace_seq_has_overflowed(s))
2760 return TRACE_TYPE_PARTIAL_LINE;
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002761 }
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002762
Steven Rostedtf633cef2008-12-23 23:24:13 -05002763 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002764 if (event) {
Steven Rostedta9a57762010-04-22 18:46:14 -04002765 enum print_line_t ret = event->funcs->hex(iter, 0, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002766 if (ret != TRACE_TYPE_HANDLED)
2767 return ret;
2768 }
Steven Rostedt7104f302008-10-01 10:52:51 -04002769
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002770 SEQ_PUT_FIELD(s, newline);
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002771
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002772 return trace_handle_return(s);
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002773}
2774
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002775static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002776{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002777 struct trace_array *tr = iter->tr;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002778 struct trace_seq *s = &iter->seq;
2779 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002780 struct trace_event *event;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002781
2782 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002783
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002784 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002785 SEQ_PUT_FIELD(s, entry->pid);
2786 SEQ_PUT_FIELD(s, iter->cpu);
2787 SEQ_PUT_FIELD(s, iter->ts);
2788 if (trace_seq_has_overflowed(s))
2789 return TRACE_TYPE_PARTIAL_LINE;
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002790 }
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002791
Steven Rostedtf633cef2008-12-23 23:24:13 -05002792 event = ftrace_find_event(entry->type);
Steven Rostedta9a57762010-04-22 18:46:14 -04002793 return event ? event->funcs->binary(iter, 0, event) :
2794 TRACE_TYPE_HANDLED;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002795}
2796
Jiri Olsa62b915f2010-04-02 19:01:22 +02002797int trace_empty(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002798{
Steven Rostedt6d158a82012-06-27 20:46:14 -04002799 struct ring_buffer_iter *buf_iter;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002800 int cpu;
2801
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002802 /* If we are looking at one CPU buffer, only check that one */
Steven Rostedtae3b5092013-01-23 15:22:59 -05002803 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002804 cpu = iter->cpu_file;
Steven Rostedt6d158a82012-06-27 20:46:14 -04002805 buf_iter = trace_buffer_iter(iter, cpu);
2806 if (buf_iter) {
2807 if (!ring_buffer_iter_empty(buf_iter))
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002808 return 0;
2809 } else {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002810 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002811 return 0;
2812 }
2813 return 1;
2814 }
2815
Steven Rostedtab464282008-05-12 21:21:00 +02002816 for_each_tracing_cpu(cpu) {
Steven Rostedt6d158a82012-06-27 20:46:14 -04002817 buf_iter = trace_buffer_iter(iter, cpu);
2818 if (buf_iter) {
2819 if (!ring_buffer_iter_empty(buf_iter))
Steven Rostedtd7690412008-10-01 00:29:53 -04002820 return 0;
2821 } else {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002822 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
Steven Rostedtd7690412008-10-01 00:29:53 -04002823 return 0;
2824 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002825 }
Steven Rostedtd7690412008-10-01 00:29:53 -04002826
Frederic Weisbecker797d3712008-09-30 18:13:45 +02002827 return 1;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002828}
2829
Lai Jiangshan4f535962009-05-18 19:35:34 +08002830/* Called with trace_event_read_lock() held. */
Jason Wessel955b61e2010-08-05 09:22:23 -05002831enum print_line_t print_trace_line(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002832{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002833 struct trace_array *tr = iter->tr;
2834 unsigned long trace_flags = tr->trace_flags;
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002835 enum print_line_t ret;
2836
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002837 if (iter->lost_events) {
2838 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2839 iter->cpu, iter->lost_events);
2840 if (trace_seq_has_overflowed(&iter->seq))
2841 return TRACE_TYPE_PARTIAL_LINE;
2842 }
Steven Rostedtbc21b472010-03-31 19:49:26 -04002843
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002844 if (iter->trace && iter->trace->print_line) {
2845 ret = iter->trace->print_line(iter);
2846 if (ret != TRACE_TYPE_UNHANDLED)
2847 return ret;
2848 }
Thomas Gleixner72829bc2008-05-23 21:37:28 +02002849
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -05002850 if (iter->ent->type == TRACE_BPUTS &&
2851 trace_flags & TRACE_ITER_PRINTK &&
2852 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2853 return trace_print_bputs_msg_only(iter);
2854
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002855 if (iter->ent->type == TRACE_BPRINT &&
2856 trace_flags & TRACE_ITER_PRINTK &&
2857 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002858 return trace_print_bprintk_msg_only(iter);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002859
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002860 if (iter->ent->type == TRACE_PRINT &&
2861 trace_flags & TRACE_ITER_PRINTK &&
2862 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002863 return trace_print_printk_msg_only(iter);
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002864
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002865 if (trace_flags & TRACE_ITER_BIN)
2866 return print_bin_fmt(iter);
2867
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002868 if (trace_flags & TRACE_ITER_HEX)
2869 return print_hex_fmt(iter);
2870
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002871 if (trace_flags & TRACE_ITER_RAW)
2872 return print_raw_fmt(iter);
2873
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002874 return print_trace_fmt(iter);
2875}
2876
Jiri Olsa7e9a49e2011-11-07 16:08:49 +01002877void trace_latency_header(struct seq_file *m)
2878{
2879 struct trace_iterator *iter = m->private;
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002880 struct trace_array *tr = iter->tr;
Jiri Olsa7e9a49e2011-11-07 16:08:49 +01002881
2882 /* print nothing if the buffers are empty */
2883 if (trace_empty(iter))
2884 return;
2885
2886 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2887 print_trace_header(m, iter);
2888
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002889 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
Jiri Olsa7e9a49e2011-11-07 16:08:49 +01002890 print_lat_help_header(m);
2891}
2892
Jiri Olsa62b915f2010-04-02 19:01:22 +02002893void trace_default_header(struct seq_file *m)
2894{
2895 struct trace_iterator *iter = m->private;
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002896 struct trace_array *tr = iter->tr;
2897 unsigned long trace_flags = tr->trace_flags;
Jiri Olsa62b915f2010-04-02 19:01:22 +02002898
Jiri Olsaf56e7f82011-06-03 16:58:49 +02002899 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2900 return;
2901
Jiri Olsa62b915f2010-04-02 19:01:22 +02002902 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2903 /* print nothing if the buffers are empty */
2904 if (trace_empty(iter))
2905 return;
2906 print_trace_header(m, iter);
2907 if (!(trace_flags & TRACE_ITER_VERBOSE))
2908 print_lat_help_header(m);
2909 } else {
Steven Rostedt77271ce2011-11-17 09:34:33 -05002910 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2911 if (trace_flags & TRACE_ITER_IRQ_INFO)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002912 print_func_help_header_irq(iter->trace_buffer, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002913 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002914 print_func_help_header(iter->trace_buffer, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002915 }
Jiri Olsa62b915f2010-04-02 19:01:22 +02002916 }
2917}
2918
Steven Rostedte0a413f2011-09-29 21:26:16 -04002919static void test_ftrace_alive(struct seq_file *m)
2920{
2921 if (!ftrace_is_dead())
2922 return;
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002923 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
2924 "# MAY BE MISSING FUNCTION EVENTS\n");
Steven Rostedte0a413f2011-09-29 21:26:16 -04002925}
2926
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002927#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002928static void show_snapshot_main_help(struct seq_file *m)
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002929{
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002930 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
2931 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2932 "# Takes a snapshot of the main buffer.\n"
2933 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
2934 "# (Doesn't have to be '2' works with any number that\n"
2935 "# is not a '0' or '1')\n");
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002936}
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002937
2938static void show_snapshot_percpu_help(struct seq_file *m)
2939{
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01002940 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002941#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002942 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2943 "# Takes a snapshot of the main buffer for this cpu.\n");
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002944#else
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002945 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
2946 "# Must use main snapshot file to allocate.\n");
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002947#endif
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002948 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
2949 "# (Doesn't have to be '2' works with any number that\n"
2950 "# is not a '0' or '1')\n");
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002951}
2952
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002953static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2954{
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05002955 if (iter->tr->allocated_snapshot)
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01002956 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002957 else
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01002958 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002959
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01002960 seq_puts(m, "# Snapshot commands:\n");
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002961 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2962 show_snapshot_main_help(m);
2963 else
2964 show_snapshot_percpu_help(m);
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002965}
2966#else
2967/* Should never be called */
2968static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2969#endif
2970
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002971static int s_show(struct seq_file *m, void *v)
2972{
2973 struct trace_iterator *iter = v;
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002974 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002975
2976 if (iter->ent == NULL) {
2977 if (iter->tr) {
2978 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2979 seq_puts(m, "#\n");
Steven Rostedte0a413f2011-09-29 21:26:16 -04002980 test_ftrace_alive(m);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002981 }
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002982 if (iter->snapshot && trace_empty(iter))
2983 print_snapshot_help(m, iter);
2984 else if (iter->trace && iter->trace->print_header)
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002985 iter->trace->print_header(m);
Jiri Olsa62b915f2010-04-02 19:01:22 +02002986 else
2987 trace_default_header(m);
2988
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002989 } else if (iter->leftover) {
2990 /*
2991 * If we filled the seq_file buffer earlier, we
2992 * want to just show it now.
2993 */
2994 ret = trace_print_seq(m, &iter->seq);
2995
2996 /* ret should this time be zero, but you never know */
2997 iter->leftover = ret;
2998
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002999 } else {
Ingo Molnarf9896bf2008-05-12 21:20:47 +02003000 print_trace_line(iter);
Steven Rostedta63ce5b2009-12-07 09:11:39 -05003001 ret = trace_print_seq(m, &iter->seq);
3002 /*
3003 * If we overflow the seq_file buffer, then it will
3004 * ask us for this data again at start up.
3005 * Use that instead.
3006 * ret is 0 if seq_file write succeeded.
3007 * -1 otherwise.
3008 */
3009 iter->leftover = ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003010 }
3011
3012 return 0;
3013}
3014
Oleg Nesterov649e9c72013-07-23 17:25:54 +02003015/*
3016 * Should be used after trace_array_get(), trace_types_lock
3017 * ensures that i_cdev was already initialized.
3018 */
3019static inline int tracing_get_cpu(struct inode *inode)
3020{
3021 if (inode->i_cdev) /* See trace_create_cpu_file() */
3022 return (long)inode->i_cdev - 1;
3023 return RING_BUFFER_ALL_CPUS;
3024}
3025
James Morris88e9d342009-09-22 16:43:43 -07003026static const struct seq_operations tracer_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003027 .start = s_start,
3028 .next = s_next,
3029 .stop = s_stop,
3030 .show = s_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003031};
3032
Ingo Molnare309b412008-05-12 21:20:51 +02003033static struct trace_iterator *
Oleg Nesterov6484c712013-07-23 17:26:10 +02003034__tracing_open(struct inode *inode, struct file *file, bool snapshot)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003035{
Oleg Nesterov6484c712013-07-23 17:26:10 +02003036 struct trace_array *tr = inode->i_private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003037 struct trace_iterator *iter;
Jiri Olsa50e18b92012-04-25 10:23:39 +02003038 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003039
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05003040 if (tracing_disabled)
3041 return ERR_PTR(-ENODEV);
Steven Rostedt60a11772008-05-12 21:20:44 +02003042
Jiri Olsa50e18b92012-04-25 10:23:39 +02003043 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05003044 if (!iter)
3045 return ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003046
Gil Fruchter72917232015-06-09 10:32:35 +03003047 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
Steven Rostedt6d158a82012-06-27 20:46:14 -04003048 GFP_KERNEL);
Dan Carpenter93574fc2012-07-11 09:35:08 +03003049 if (!iter->buffer_iter)
3050 goto release;
3051
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003052 /*
3053 * We make a copy of the current tracer to avoid concurrent
3054 * changes on it while we are reading.
3055 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003056 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003057 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05003058 if (!iter->trace)
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003059 goto fail;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05003060
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003061 *iter->trace = *tr->current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003062
Li Zefan79f55992009-06-15 14:58:26 +08003063 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02003064 goto fail;
3065
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003066 iter->tr = tr;
3067
3068#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003069 /* Currently only the top directory has a snapshot */
3070 if (tr->current_trace->print_max || snapshot)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003071 iter->trace_buffer = &tr->max_buffer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003072 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003073#endif
3074 iter->trace_buffer = &tr->trace_buffer;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09003075 iter->snapshot = snapshot;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003076 iter->pos = -1;
Oleg Nesterov6484c712013-07-23 17:26:10 +02003077 iter->cpu_file = tracing_get_cpu(inode);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003078 mutex_init(&iter->mutex);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003079
Markus Metzger8bba1bf2008-11-25 09:12:31 +01003080 /* Notify the tracer early; before we stop tracing. */
3081 if (iter->trace && iter->trace->open)
Markus Metzgera93751c2008-12-11 13:53:26 +01003082 iter->trace->open(iter);
Markus Metzger8bba1bf2008-11-25 09:12:31 +01003083
Steven Rostedt12ef7d42008-11-12 17:52:38 -05003084 /* Annotate start of buffers if we had overruns */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003085 if (ring_buffer_overruns(iter->trace_buffer->buffer))
Steven Rostedt12ef7d42008-11-12 17:52:38 -05003086 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3087
David Sharp8be07092012-11-13 12:18:22 -08003088 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
Yoshihiro YUNOMAE58e8eed2013-04-23 10:32:39 +09003089 if (trace_clocks[tr->clock_id].in_ns)
David Sharp8be07092012-11-13 12:18:22 -08003090 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3091
Hiraku Toyookadebdd572012-12-26 11:53:00 +09003092 /* stop the trace while dumping if we are not opening "snapshot" */
3093 if (!iter->snapshot)
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003094 tracing_stop_tr(tr);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04003095
Steven Rostedtae3b5092013-01-23 15:22:59 -05003096 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003097 for_each_tracing_cpu(cpu) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003098 iter->buffer_iter[cpu] =
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003099 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
David Miller72c9ddf2010-04-20 15:47:11 -07003100 }
3101 ring_buffer_read_prepare_sync();
3102 for_each_tracing_cpu(cpu) {
3103 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04003104 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003105 }
3106 } else {
3107 cpu = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003108 iter->buffer_iter[cpu] =
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003109 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
David Miller72c9ddf2010-04-20 15:47:11 -07003110 ring_buffer_read_prepare_sync();
3111 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04003112 tracing_iter_reset(iter, cpu);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003113 }
3114
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003115 mutex_unlock(&trace_types_lock);
3116
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003117 return iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003118
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003119 fail:
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003120 mutex_unlock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003121 kfree(iter->trace);
Steven Rostedt6d158a82012-06-27 20:46:14 -04003122 kfree(iter->buffer_iter);
Dan Carpenter93574fc2012-07-11 09:35:08 +03003123release:
Jiri Olsa50e18b92012-04-25 10:23:39 +02003124 seq_release_private(inode, file);
3125 return ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003126}
3127
3128int tracing_open_generic(struct inode *inode, struct file *filp)
3129{
Steven Rostedt60a11772008-05-12 21:20:44 +02003130 if (tracing_disabled)
3131 return -ENODEV;
3132
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003133 filp->private_data = inode->i_private;
3134 return 0;
3135}
3136
Geyslan G. Bem2e864212013-10-18 21:15:54 -03003137bool tracing_is_disabled(void)
3138{
3139 return (tracing_disabled) ? true: false;
3140}
3141
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003142/*
3143 * Open and update trace_array ref count.
3144 * Must have the current trace_array passed to it.
3145 */
Steven Rostedt (Red Hat)dcc30222013-07-02 20:30:52 -04003146static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003147{
3148 struct trace_array *tr = inode->i_private;
3149
3150 if (tracing_disabled)
3151 return -ENODEV;
3152
3153 if (trace_array_get(tr) < 0)
3154 return -ENODEV;
3155
3156 filp->private_data = inode->i_private;
3157
3158 return 0;
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003159}
3160
Hannes Eder4fd27352009-02-10 19:44:12 +01003161static int tracing_release(struct inode *inode, struct file *file)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003162{
Oleg Nesterov6484c712013-07-23 17:26:10 +02003163 struct trace_array *tr = inode->i_private;
matt mooney907f2782010-09-27 19:04:53 -07003164 struct seq_file *m = file->private_data;
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003165 struct trace_iterator *iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003166 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003167
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003168 if (!(file->f_mode & FMODE_READ)) {
Oleg Nesterov6484c712013-07-23 17:26:10 +02003169 trace_array_put(tr);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003170 return 0;
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003171 }
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003172
Oleg Nesterov6484c712013-07-23 17:26:10 +02003173 /* Writes do not use seq_file */
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003174 iter = m->private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003175 mutex_lock(&trace_types_lock);
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05003176
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003177 for_each_tracing_cpu(cpu) {
3178 if (iter->buffer_iter[cpu])
3179 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3180 }
3181
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003182 if (iter->trace && iter->trace->close)
3183 iter->trace->close(iter);
3184
Hiraku Toyookadebdd572012-12-26 11:53:00 +09003185 if (!iter->snapshot)
3186 /* reenable tracing if it was previously enabled */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003187 tracing_start_tr(tr);
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07003188
3189 __trace_array_put(tr);
3190
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003191 mutex_unlock(&trace_types_lock);
3192
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003193 mutex_destroy(&iter->mutex);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02003194 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003195 kfree(iter->trace);
Steven Rostedt6d158a82012-06-27 20:46:14 -04003196 kfree(iter->buffer_iter);
Jiri Olsa50e18b92012-04-25 10:23:39 +02003197 seq_release_private(inode, file);
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003198
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003199 return 0;
3200}
3201
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003202static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3203{
3204 struct trace_array *tr = inode->i_private;
3205
3206 trace_array_put(tr);
3207 return 0;
3208}
3209
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003210static int tracing_single_release_tr(struct inode *inode, struct file *file)
3211{
3212 struct trace_array *tr = inode->i_private;
3213
3214 trace_array_put(tr);
3215
3216 return single_release(inode, file);
3217}
3218
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003219static int tracing_open(struct inode *inode, struct file *file)
3220{
Oleg Nesterov6484c712013-07-23 17:26:10 +02003221 struct trace_array *tr = inode->i_private;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05003222 struct trace_iterator *iter;
3223 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003224
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003225 if (trace_array_get(tr) < 0)
3226 return -ENODEV;
3227
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003228 /* If this file was open for write, then erase contents */
Oleg Nesterov6484c712013-07-23 17:26:10 +02003229 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3230 int cpu = tracing_get_cpu(inode);
3231
3232 if (cpu == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003233 tracing_reset_online_cpus(&tr->trace_buffer);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003234 else
Oleg Nesterov6484c712013-07-23 17:26:10 +02003235 tracing_reset(&tr->trace_buffer, cpu);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003236 }
3237
3238 if (file->f_mode & FMODE_READ) {
Oleg Nesterov6484c712013-07-23 17:26:10 +02003239 iter = __tracing_open(inode, file, false);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003240 if (IS_ERR(iter))
3241 ret = PTR_ERR(iter);
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04003242 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003243 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3244 }
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003245
3246 if (ret < 0)
3247 trace_array_put(tr);
3248
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003249 return ret;
3250}
3251
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003252/*
3253 * Some tracers are not suitable for instance buffers.
3254 * A tracer is always available for the global array (toplevel)
3255 * or if it explicitly states that it is.
3256 */
3257static bool
3258trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3259{
3260 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3261}
3262
3263/* Find the next tracer that this trace array may use */
3264static struct tracer *
3265get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3266{
3267 while (t && !trace_ok_for_array(t, tr))
3268 t = t->next;
3269
3270 return t;
3271}
3272
Ingo Molnare309b412008-05-12 21:20:51 +02003273static void *
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003274t_next(struct seq_file *m, void *v, loff_t *pos)
3275{
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003276 struct trace_array *tr = m->private;
Li Zefanf129e962009-06-24 09:53:44 +08003277 struct tracer *t = v;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003278
3279 (*pos)++;
3280
3281 if (t)
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003282 t = get_tracer_for_array(tr, t->next);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003283
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003284 return t;
3285}
3286
3287static void *t_start(struct seq_file *m, loff_t *pos)
3288{
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003289 struct trace_array *tr = m->private;
Li Zefanf129e962009-06-24 09:53:44 +08003290 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003291 loff_t l = 0;
3292
3293 mutex_lock(&trace_types_lock);
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003294
3295 t = get_tracer_for_array(tr, trace_types);
3296 for (; t && l < *pos; t = t_next(m, t, &l))
3297 ;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003298
3299 return t;
3300}
3301
3302static void t_stop(struct seq_file *m, void *p)
3303{
3304 mutex_unlock(&trace_types_lock);
3305}
3306
3307static int t_show(struct seq_file *m, void *v)
3308{
3309 struct tracer *t = v;
3310
3311 if (!t)
3312 return 0;
3313
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01003314 seq_puts(m, t->name);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003315 if (t->next)
3316 seq_putc(m, ' ');
3317 else
3318 seq_putc(m, '\n');
3319
3320 return 0;
3321}
3322
James Morris88e9d342009-09-22 16:43:43 -07003323static const struct seq_operations show_traces_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003324 .start = t_start,
3325 .next = t_next,
3326 .stop = t_stop,
3327 .show = t_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003328};
3329
3330static int show_traces_open(struct inode *inode, struct file *file)
3331{
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003332 struct trace_array *tr = inode->i_private;
3333 struct seq_file *m;
3334 int ret;
3335
Steven Rostedt60a11772008-05-12 21:20:44 +02003336 if (tracing_disabled)
3337 return -ENODEV;
3338
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003339 ret = seq_open(file, &show_traces_seq_ops);
3340 if (ret)
3341 return ret;
3342
3343 m = file->private_data;
3344 m->private = tr;
3345
3346 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003347}
3348
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003349static ssize_t
3350tracing_write_stub(struct file *filp, const char __user *ubuf,
3351 size_t count, loff_t *ppos)
3352{
3353 return count;
3354}
3355
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05003356loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
Slava Pestov364829b2010-11-24 15:13:16 -08003357{
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05003358 int ret;
3359
Slava Pestov364829b2010-11-24 15:13:16 -08003360 if (file->f_mode & FMODE_READ)
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05003361 ret = seq_lseek(file, offset, whence);
Slava Pestov364829b2010-11-24 15:13:16 -08003362 else
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05003363 file->f_pos = ret = 0;
3364
3365 return ret;
Slava Pestov364829b2010-11-24 15:13:16 -08003366}
3367
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003368static const struct file_operations tracing_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003369 .open = tracing_open,
3370 .read = seq_read,
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003371 .write = tracing_write_stub,
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05003372 .llseek = tracing_lseek,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003373 .release = tracing_release,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003374};
3375
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003376static const struct file_operations show_traces_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02003377 .open = show_traces_open,
3378 .read = seq_read,
3379 .release = seq_release,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003380 .llseek = seq_lseek,
Ingo Molnarc7078de2008-05-12 21:20:52 +02003381};
3382
Ingo Molnar36dfe922008-05-12 21:20:52 +02003383/*
Ingo Molnar36dfe922008-05-12 21:20:52 +02003384 * The tracer itself will not take this lock, but still we want
3385 * to provide a consistent cpumask to user-space:
3386 */
3387static DEFINE_MUTEX(tracing_cpumask_update_lock);
3388
3389/*
3390 * Temporary storage for the character representation of the
3391 * CPU bitmask (and one more byte for the newline):
3392 */
3393static char mask_str[NR_CPUS + 1];
3394
Ingo Molnarc7078de2008-05-12 21:20:52 +02003395static ssize_t
3396tracing_cpumask_read(struct file *filp, char __user *ubuf,
3397 size_t count, loff_t *ppos)
3398{
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003399 struct trace_array *tr = file_inode(filp)->i_private;
Ingo Molnar36dfe922008-05-12 21:20:52 +02003400 int len;
Ingo Molnarc7078de2008-05-12 21:20:52 +02003401
3402 mutex_lock(&tracing_cpumask_update_lock);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003403
Tejun Heo1a402432015-02-13 14:37:39 -08003404 len = snprintf(mask_str, count, "%*pb\n",
3405 cpumask_pr_args(tr->tracing_cpumask));
3406 if (len >= count) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02003407 count = -EINVAL;
3408 goto out_err;
3409 }
Ingo Molnar36dfe922008-05-12 21:20:52 +02003410 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3411
3412out_err:
Ingo Molnarc7078de2008-05-12 21:20:52 +02003413 mutex_unlock(&tracing_cpumask_update_lock);
3414
3415 return count;
3416}
3417
3418static ssize_t
3419tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3420 size_t count, loff_t *ppos)
3421{
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003422 struct trace_array *tr = file_inode(filp)->i_private;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303423 cpumask_var_t tracing_cpumask_new;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003424 int err, cpu;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303425
3426 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3427 return -ENOMEM;
Ingo Molnarc7078de2008-05-12 21:20:52 +02003428
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303429 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003430 if (err)
3431 goto err_unlock;
3432
Li Zefan215368e2009-06-15 10:56:42 +08003433 mutex_lock(&tracing_cpumask_update_lock);
3434
Steven Rostedta5e25882008-12-02 15:34:05 -05003435 local_irq_disable();
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05003436 arch_spin_lock(&tr->max_lock);
Steven Rostedtab464282008-05-12 21:21:00 +02003437 for_each_tracing_cpu(cpu) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02003438 /*
3439 * Increase/decrease the disabled counter if we are
3440 * about to flip a bit in the cpumask:
3441 */
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003442 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303443 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003444 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3445 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003446 }
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003447 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303448 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003449 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3450 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003451 }
3452 }
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05003453 arch_spin_unlock(&tr->max_lock);
Steven Rostedta5e25882008-12-02 15:34:05 -05003454 local_irq_enable();
Ingo Molnar36dfe922008-05-12 21:20:52 +02003455
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003456 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003457
Ingo Molnarc7078de2008-05-12 21:20:52 +02003458 mutex_unlock(&tracing_cpumask_update_lock);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303459 free_cpumask_var(tracing_cpumask_new);
Ingo Molnarc7078de2008-05-12 21:20:52 +02003460
Ingo Molnarc7078de2008-05-12 21:20:52 +02003461 return count;
Ingo Molnar36dfe922008-05-12 21:20:52 +02003462
3463err_unlock:
Li Zefan215368e2009-06-15 10:56:42 +08003464 free_cpumask_var(tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003465
3466 return err;
Ingo Molnarc7078de2008-05-12 21:20:52 +02003467}
3468
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003469static const struct file_operations tracing_cpumask_fops = {
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003470 .open = tracing_open_generic_tr,
Ingo Molnarc7078de2008-05-12 21:20:52 +02003471 .read = tracing_cpumask_read,
3472 .write = tracing_cpumask_write,
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003473 .release = tracing_release_generic_tr,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003474 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003475};
3476
Li Zefanfdb372e2009-12-08 11:15:59 +08003477static int tracing_trace_options_show(struct seq_file *m, void *v)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003478{
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003479 struct tracer_opt *trace_opts;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003480 struct trace_array *tr = m->private;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003481 u32 tracer_flags;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003482 int i;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003483
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003484 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003485 tracer_flags = tr->current_trace->flags->val;
3486 trace_opts = tr->current_trace->flags->opts;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003487
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003488 for (i = 0; trace_options[i]; i++) {
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04003489 if (tr->trace_flags & (1 << i))
Li Zefanfdb372e2009-12-08 11:15:59 +08003490 seq_printf(m, "%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003491 else
Li Zefanfdb372e2009-12-08 11:15:59 +08003492 seq_printf(m, "no%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003493 }
3494
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003495 for (i = 0; trace_opts[i].name; i++) {
3496 if (tracer_flags & trace_opts[i].bit)
Li Zefanfdb372e2009-12-08 11:15:59 +08003497 seq_printf(m, "%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003498 else
Li Zefanfdb372e2009-12-08 11:15:59 +08003499 seq_printf(m, "no%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003500 }
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003501 mutex_unlock(&trace_types_lock);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003502
Li Zefanfdb372e2009-12-08 11:15:59 +08003503 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003504}
3505
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003506static int __set_tracer_option(struct trace_array *tr,
Li Zefan8d18eaa2009-12-08 11:17:06 +08003507 struct tracer_flags *tracer_flags,
3508 struct tracer_opt *opts, int neg)
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003509{
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003510 struct tracer *trace = tr->current_trace;
Li Zefan8d18eaa2009-12-08 11:17:06 +08003511 int ret;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003512
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003513 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003514 if (ret)
3515 return ret;
3516
3517 if (neg)
Zhaolei77708412009-08-07 18:53:21 +08003518 tracer_flags->val &= ~opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003519 else
Zhaolei77708412009-08-07 18:53:21 +08003520 tracer_flags->val |= opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003521 return 0;
3522}
3523
Li Zefan8d18eaa2009-12-08 11:17:06 +08003524/* Try to assign a tracer specific option */
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003525static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
Li Zefan8d18eaa2009-12-08 11:17:06 +08003526{
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003527 struct tracer *trace = tr->current_trace;
Li Zefan8d18eaa2009-12-08 11:17:06 +08003528 struct tracer_flags *tracer_flags = trace->flags;
3529 struct tracer_opt *opts = NULL;
3530 int i;
3531
3532 for (i = 0; tracer_flags->opts[i].name; i++) {
3533 opts = &tracer_flags->opts[i];
3534
3535 if (strcmp(cmp, opts->name) == 0)
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003536 return __set_tracer_option(tr, trace->flags, opts, neg);
Li Zefan8d18eaa2009-12-08 11:17:06 +08003537 }
3538
3539 return -EINVAL;
3540}
3541
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003542/* Some tracers require overwrite to stay enabled */
3543int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3544{
3545 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3546 return -1;
3547
3548 return 0;
3549}
3550
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003551int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
Steven Rostedtaf4617b2009-03-17 18:09:55 -04003552{
3553 /* do nothing if flag is already set */
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04003554 if (!!(tr->trace_flags & mask) == !!enabled)
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003555 return 0;
3556
3557 /* Give the tracer a chance to approve the change */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003558 if (tr->current_trace->flag_changed)
Steven Rostedt (Red Hat)bf6065b2014-01-10 17:51:01 -05003559 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003560 return -EINVAL;
Steven Rostedtaf4617b2009-03-17 18:09:55 -04003561
3562 if (enabled)
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04003563 tr->trace_flags |= mask;
Steven Rostedtaf4617b2009-03-17 18:09:55 -04003564 else
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04003565 tr->trace_flags &= ~mask;
Li Zefane870e9a2010-07-02 11:07:32 +08003566
3567 if (mask == TRACE_ITER_RECORD_CMD)
3568 trace_event_enable_cmd_record(enabled);
David Sharp750912f2010-12-08 13:46:47 -08003569
Steven Rostedt (Red Hat)80902822013-03-14 14:20:54 -04003570 if (mask == TRACE_ITER_OVERWRITE) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003571 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
Steven Rostedt (Red Hat)80902822013-03-14 14:20:54 -04003572#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003573 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
Steven Rostedt (Red Hat)80902822013-03-14 14:20:54 -04003574#endif
3575 }
Steven Rostedt81698832012-10-11 10:15:05 -04003576
Steven Rostedt (Red Hat)b9f91082015-09-29 18:21:35 -04003577 if (mask == TRACE_ITER_PRINTK) {
Steven Rostedt81698832012-10-11 10:15:05 -04003578 trace_printk_start_stop_comm(enabled);
Steven Rostedt (Red Hat)b9f91082015-09-29 18:21:35 -04003579 trace_printk_control(enabled);
3580 }
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003581
3582 return 0;
Steven Rostedtaf4617b2009-03-17 18:09:55 -04003583}
3584
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003585static int trace_set_options(struct trace_array *tr, char *option)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003586{
Li Zefan8d18eaa2009-12-08 11:17:06 +08003587 char *cmp;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003588 int neg = 0;
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003589 int ret = -ENODEV;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003590 int i;
3591
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003592 cmp = strstrip(option);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003593
Li Zefan8d18eaa2009-12-08 11:17:06 +08003594 if (strncmp(cmp, "no", 2) == 0) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003595 neg = 1;
3596 cmp += 2;
3597 }
3598
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04003599 mutex_lock(&trace_types_lock);
3600
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003601 for (i = 0; trace_options[i]; i++) {
Li Zefan8d18eaa2009-12-08 11:17:06 +08003602 if (strcmp(cmp, trace_options[i]) == 0) {
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003603 ret = set_tracer_flag(tr, 1 << i, !neg);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003604 break;
3605 }
3606 }
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003607
3608 /* If no option could be set, test the specific tracer options */
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04003609 if (!trace_options[i])
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003610 ret = set_tracer_option(tr, cmp, neg);
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04003611
3612 mutex_unlock(&trace_types_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003613
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003614 return ret;
3615}
3616
3617static ssize_t
3618tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3619 size_t cnt, loff_t *ppos)
3620{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003621 struct seq_file *m = filp->private_data;
3622 struct trace_array *tr = m->private;
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003623 char buf[64];
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003624 int ret;
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003625
3626 if (cnt >= sizeof(buf))
3627 return -EINVAL;
3628
3629 if (copy_from_user(&buf, ubuf, cnt))
3630 return -EFAULT;
3631
Steven Rostedta8dd2172013-01-09 20:54:17 -05003632 buf[cnt] = 0;
3633
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003634 ret = trace_set_options(tr, buf);
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003635 if (ret < 0)
3636 return ret;
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003637
Jiri Olsacf8517c2009-10-23 19:36:16 -04003638 *ppos += cnt;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003639
3640 return cnt;
3641}
3642
Li Zefanfdb372e2009-12-08 11:15:59 +08003643static int tracing_trace_options_open(struct inode *inode, struct file *file)
3644{
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003645 struct trace_array *tr = inode->i_private;
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07003646 int ret;
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003647
Li Zefanfdb372e2009-12-08 11:15:59 +08003648 if (tracing_disabled)
3649 return -ENODEV;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003650
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003651 if (trace_array_get(tr) < 0)
3652 return -ENODEV;
3653
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07003654 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3655 if (ret < 0)
3656 trace_array_put(tr);
3657
3658 return ret;
Li Zefanfdb372e2009-12-08 11:15:59 +08003659}
3660
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003661static const struct file_operations tracing_iter_fops = {
Li Zefanfdb372e2009-12-08 11:15:59 +08003662 .open = tracing_trace_options_open,
3663 .read = seq_read,
3664 .llseek = seq_lseek,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003665 .release = tracing_single_release_tr,
Steven Rostedtee6bce52008-11-12 17:52:37 -05003666 .write = tracing_trace_options_write,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003667};
3668
Ingo Molnar7bd2f242008-05-12 21:20:45 +02003669static const char readme_msg[] =
3670 "tracing mini-HOWTO:\n\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003671 "# echo 0 > tracing_on : quick way to disable tracing\n"
3672 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3673 " Important files:\n"
3674 " trace\t\t\t- The static contents of the buffer\n"
3675 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3676 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3677 " current_tracer\t- function and latency tracers\n"
3678 " available_tracers\t- list of configured tracers for current_tracer\n"
3679 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3680 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3681 " trace_clock\t\t-change the clock used to order events\n"
3682 " local: Per cpu clock but may not be synced across CPUs\n"
3683 " global: Synced across CPUs but slows tracing down.\n"
3684 " counter: Not a clock, but just an increment\n"
3685 " uptime: Jiffy counter from time of boot\n"
3686 " perf: Same clock that perf events use\n"
3687#ifdef CONFIG_X86_64
3688 " x86-tsc: TSC cycle counter\n"
3689#endif
3690 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3691 " tracing_cpumask\t- Limit which CPUs to trace\n"
3692 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3693 "\t\t\t Remove sub-buffer with rmdir\n"
3694 " trace_options\t\t- Set format or modify how tracing happens\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003695 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3696 "\t\t\t option name\n"
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09003697 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003698#ifdef CONFIG_DYNAMIC_FTRACE
3699 "\n available_filter_functions - list of functions that can be filtered on\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003700 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3701 "\t\t\t functions\n"
3702 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3703 "\t modules: Can select a group via module\n"
3704 "\t Format: :mod:<module-name>\n"
3705 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3706 "\t triggers: a command to perform when function is hit\n"
3707 "\t Format: <function>:<trigger>[:count]\n"
3708 "\t trigger: traceon, traceoff\n"
3709 "\t\t enable_event:<system>:<event>\n"
3710 "\t\t disable_event:<system>:<event>\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003711#ifdef CONFIG_STACKTRACE
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003712 "\t\t stacktrace\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003713#endif
3714#ifdef CONFIG_TRACER_SNAPSHOT
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003715 "\t\t snapshot\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003716#endif
Steven Rostedt (Red Hat)17a280e2014-04-10 22:43:37 -04003717 "\t\t dump\n"
3718 "\t\t cpudump\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003719 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3720 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3721 "\t The first one will disable tracing every time do_fault is hit\n"
3722 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3723 "\t The first time do trap is hit and it disables tracing, the\n"
3724 "\t counter will decrement to 2. If tracing is already disabled,\n"
3725 "\t the counter will not decrement. It only decrements when the\n"
3726 "\t trigger did work\n"
3727 "\t To remove trigger without count:\n"
3728 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3729 "\t To remove trigger with a count:\n"
3730 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003731 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003732 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3733 "\t modules: Can select a group via module command :mod:\n"
3734 "\t Does not accept triggers\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003735#endif /* CONFIG_DYNAMIC_FTRACE */
3736#ifdef CONFIG_FUNCTION_TRACER
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003737 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3738 "\t\t (function)\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003739#endif
3740#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3741 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
Namhyung Kimd048a8c72014-06-13 01:23:53 +09003742 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003743 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3744#endif
3745#ifdef CONFIG_TRACER_SNAPSHOT
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003746 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3747 "\t\t\t snapshot buffer. Read the contents for more\n"
3748 "\t\t\t information\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003749#endif
zhangwei(Jovi)991821c2013-07-15 16:32:34 +08003750#ifdef CONFIG_STACK_TRACER
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003751 " stack_trace\t\t- Shows the max stack trace when active\n"
3752 " stack_max_size\t- Shows current max stack size that was traced\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003753 "\t\t\t Write into this file to reset the max size (trigger a\n"
3754 "\t\t\t new trace)\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003755#ifdef CONFIG_DYNAMIC_FTRACE
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003756 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3757 "\t\t\t traces\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003758#endif
zhangwei(Jovi)991821c2013-07-15 16:32:34 +08003759#endif /* CONFIG_STACK_TRACER */
Tom Zanussi26f25562014-01-17 15:11:44 -06003760 " events/\t\t- Directory containing all trace event subsystems:\n"
3761 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3762 " events/<system>/\t- Directory containing all trace events for <system>:\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003763 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3764 "\t\t\t events\n"
Tom Zanussi26f25562014-01-17 15:11:44 -06003765 " filter\t\t- If set, only events passing filter are traced\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003766 " events/<system>/<event>/\t- Directory containing control files for\n"
3767 "\t\t\t <event>:\n"
Tom Zanussi26f25562014-01-17 15:11:44 -06003768 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3769 " filter\t\t- If set, only events passing filter are traced\n"
3770 " trigger\t\t- If set, a command to perform when event is hit\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003771 "\t Format: <trigger>[:count][if <filter>]\n"
3772 "\t trigger: traceon, traceoff\n"
3773 "\t enable_event:<system>:<event>\n"
3774 "\t disable_event:<system>:<event>\n"
Tom Zanussi26f25562014-01-17 15:11:44 -06003775#ifdef CONFIG_STACKTRACE
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003776 "\t\t stacktrace\n"
Tom Zanussi26f25562014-01-17 15:11:44 -06003777#endif
3778#ifdef CONFIG_TRACER_SNAPSHOT
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003779 "\t\t snapshot\n"
Tom Zanussi26f25562014-01-17 15:11:44 -06003780#endif
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003781 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3782 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3783 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3784 "\t events/block/block_unplug/trigger\n"
3785 "\t The first disables tracing every time block_unplug is hit.\n"
3786 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3787 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3788 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3789 "\t Like function triggers, the counter is only decremented if it\n"
3790 "\t enabled or disabled tracing.\n"
3791 "\t To remove a trigger without a count:\n"
3792 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3793 "\t To remove a trigger with a count:\n"
3794 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3795 "\t Filters can be ignored when removing a trigger.\n"
Ingo Molnar7bd2f242008-05-12 21:20:45 +02003796;
3797
3798static ssize_t
3799tracing_readme_read(struct file *filp, char __user *ubuf,
3800 size_t cnt, loff_t *ppos)
3801{
3802 return simple_read_from_buffer(ubuf, cnt, ppos,
3803 readme_msg, strlen(readme_msg));
3804}
3805
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003806static const struct file_operations tracing_readme_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02003807 .open = tracing_open_generic,
3808 .read = tracing_readme_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003809 .llseek = generic_file_llseek,
Ingo Molnar7bd2f242008-05-12 21:20:45 +02003810};
3811
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003812static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
Avadh Patel69abe6a2009-04-10 16:04:48 -04003813{
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003814 unsigned int *ptr = v;
Avadh Patel69abe6a2009-04-10 16:04:48 -04003815
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003816 if (*pos || m->count)
3817 ptr++;
Avadh Patel69abe6a2009-04-10 16:04:48 -04003818
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003819 (*pos)++;
Avadh Patel69abe6a2009-04-10 16:04:48 -04003820
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09003821 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
3822 ptr++) {
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003823 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
Avadh Patel69abe6a2009-04-10 16:04:48 -04003824 continue;
3825
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003826 return ptr;
Avadh Patel69abe6a2009-04-10 16:04:48 -04003827 }
3828
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003829 return NULL;
3830}
Avadh Patel69abe6a2009-04-10 16:04:48 -04003831
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003832static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
3833{
3834 void *v;
3835 loff_t l = 0;
Avadh Patel69abe6a2009-04-10 16:04:48 -04003836
Steven Rostedt (Red Hat)4c27e752014-05-30 10:49:46 -04003837 preempt_disable();
3838 arch_spin_lock(&trace_cmdline_lock);
3839
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09003840 v = &savedcmd->map_cmdline_to_pid[0];
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003841 while (l <= *pos) {
3842 v = saved_cmdlines_next(m, v, &l);
3843 if (!v)
3844 return NULL;
3845 }
3846
3847 return v;
3848}
3849
3850static void saved_cmdlines_stop(struct seq_file *m, void *v)
3851{
Steven Rostedt (Red Hat)4c27e752014-05-30 10:49:46 -04003852 arch_spin_unlock(&trace_cmdline_lock);
3853 preempt_enable();
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003854}
3855
3856static int saved_cmdlines_show(struct seq_file *m, void *v)
3857{
3858 char buf[TASK_COMM_LEN];
3859 unsigned int *pid = v;
3860
Steven Rostedt (Red Hat)4c27e752014-05-30 10:49:46 -04003861 __trace_find_cmdline(*pid, buf);
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003862 seq_printf(m, "%d %s\n", *pid, buf);
3863 return 0;
3864}
3865
3866static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
3867 .start = saved_cmdlines_start,
3868 .next = saved_cmdlines_next,
3869 .stop = saved_cmdlines_stop,
3870 .show = saved_cmdlines_show,
3871};
3872
3873static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
3874{
3875 if (tracing_disabled)
3876 return -ENODEV;
3877
3878 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
Avadh Patel69abe6a2009-04-10 16:04:48 -04003879}
3880
3881static const struct file_operations tracing_saved_cmdlines_fops = {
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003882 .open = tracing_saved_cmdlines_open,
3883 .read = seq_read,
3884 .llseek = seq_lseek,
3885 .release = seq_release,
Avadh Patel69abe6a2009-04-10 16:04:48 -04003886};
3887
3888static ssize_t
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09003889tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
3890 size_t cnt, loff_t *ppos)
3891{
3892 char buf[64];
3893 int r;
3894
3895 arch_spin_lock(&trace_cmdline_lock);
Namhyung Kima6af8fb2014-06-10 16:11:35 +09003896 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09003897 arch_spin_unlock(&trace_cmdline_lock);
3898
3899 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3900}
3901
3902static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
3903{
3904 kfree(s->saved_cmdlines);
3905 kfree(s->map_cmdline_to_pid);
3906 kfree(s);
3907}
3908
3909static int tracing_resize_saved_cmdlines(unsigned int val)
3910{
3911 struct saved_cmdlines_buffer *s, *savedcmd_temp;
3912
Namhyung Kima6af8fb2014-06-10 16:11:35 +09003913 s = kmalloc(sizeof(*s), GFP_KERNEL);
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09003914 if (!s)
3915 return -ENOMEM;
3916
3917 if (allocate_cmdlines_buffer(val, s) < 0) {
3918 kfree(s);
3919 return -ENOMEM;
3920 }
3921
3922 arch_spin_lock(&trace_cmdline_lock);
3923 savedcmd_temp = savedcmd;
3924 savedcmd = s;
3925 arch_spin_unlock(&trace_cmdline_lock);
3926 free_saved_cmdlines_buffer(savedcmd_temp);
3927
3928 return 0;
3929}
3930
3931static ssize_t
3932tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
3933 size_t cnt, loff_t *ppos)
3934{
3935 unsigned long val;
3936 int ret;
3937
3938 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3939 if (ret)
3940 return ret;
3941
3942 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
3943 if (!val || val > PID_MAX_DEFAULT)
3944 return -EINVAL;
3945
3946 ret = tracing_resize_saved_cmdlines((unsigned int)val);
3947 if (ret < 0)
3948 return ret;
3949
3950 *ppos += cnt;
3951
3952 return cnt;
3953}
3954
3955static const struct file_operations tracing_saved_cmdlines_size_fops = {
3956 .open = tracing_open_generic,
3957 .read = tracing_saved_cmdlines_size_read,
3958 .write = tracing_saved_cmdlines_size_write,
3959};
3960
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04003961#ifdef CONFIG_TRACE_ENUM_MAP_FILE
3962static union trace_enum_map_item *
3963update_enum_map(union trace_enum_map_item *ptr)
3964{
3965 if (!ptr->map.enum_string) {
3966 if (ptr->tail.next) {
3967 ptr = ptr->tail.next;
3968 /* Set ptr to the next real item (skip head) */
3969 ptr++;
3970 } else
3971 return NULL;
3972 }
3973 return ptr;
3974}
3975
3976static void *enum_map_next(struct seq_file *m, void *v, loff_t *pos)
3977{
3978 union trace_enum_map_item *ptr = v;
3979
3980 /*
3981 * Paranoid! If ptr points to end, we don't want to increment past it.
3982 * This really should never happen.
3983 */
3984 ptr = update_enum_map(ptr);
3985 if (WARN_ON_ONCE(!ptr))
3986 return NULL;
3987
3988 ptr++;
3989
3990 (*pos)++;
3991
3992 ptr = update_enum_map(ptr);
3993
3994 return ptr;
3995}
3996
3997static void *enum_map_start(struct seq_file *m, loff_t *pos)
3998{
3999 union trace_enum_map_item *v;
4000 loff_t l = 0;
4001
4002 mutex_lock(&trace_enum_mutex);
4003
4004 v = trace_enum_maps;
4005 if (v)
4006 v++;
4007
4008 while (v && l < *pos) {
4009 v = enum_map_next(m, v, &l);
4010 }
4011
4012 return v;
4013}
4014
4015static void enum_map_stop(struct seq_file *m, void *v)
4016{
4017 mutex_unlock(&trace_enum_mutex);
4018}
4019
4020static int enum_map_show(struct seq_file *m, void *v)
4021{
4022 union trace_enum_map_item *ptr = v;
4023
4024 seq_printf(m, "%s %ld (%s)\n",
4025 ptr->map.enum_string, ptr->map.enum_value,
4026 ptr->map.system);
4027
4028 return 0;
4029}
4030
4031static const struct seq_operations tracing_enum_map_seq_ops = {
4032 .start = enum_map_start,
4033 .next = enum_map_next,
4034 .stop = enum_map_stop,
4035 .show = enum_map_show,
4036};
4037
4038static int tracing_enum_map_open(struct inode *inode, struct file *filp)
4039{
4040 if (tracing_disabled)
4041 return -ENODEV;
4042
4043 return seq_open(filp, &tracing_enum_map_seq_ops);
4044}
4045
4046static const struct file_operations tracing_enum_map_fops = {
4047 .open = tracing_enum_map_open,
4048 .read = seq_read,
4049 .llseek = seq_lseek,
4050 .release = seq_release,
4051};
4052
4053static inline union trace_enum_map_item *
4054trace_enum_jmp_to_tail(union trace_enum_map_item *ptr)
4055{
4056 /* Return tail of array given the head */
4057 return ptr + ptr->head.length + 1;
4058}
4059
4060static void
4061trace_insert_enum_map_file(struct module *mod, struct trace_enum_map **start,
4062 int len)
4063{
4064 struct trace_enum_map **stop;
4065 struct trace_enum_map **map;
4066 union trace_enum_map_item *map_array;
4067 union trace_enum_map_item *ptr;
4068
4069 stop = start + len;
4070
4071 /*
4072 * The trace_enum_maps contains the map plus a head and tail item,
4073 * where the head holds the module and length of array, and the
4074 * tail holds a pointer to the next list.
4075 */
4076 map_array = kmalloc(sizeof(*map_array) * (len + 2), GFP_KERNEL);
4077 if (!map_array) {
4078 pr_warning("Unable to allocate trace enum mapping\n");
4079 return;
4080 }
4081
4082 mutex_lock(&trace_enum_mutex);
4083
4084 if (!trace_enum_maps)
4085 trace_enum_maps = map_array;
4086 else {
4087 ptr = trace_enum_maps;
4088 for (;;) {
4089 ptr = trace_enum_jmp_to_tail(ptr);
4090 if (!ptr->tail.next)
4091 break;
4092 ptr = ptr->tail.next;
4093
4094 }
4095 ptr->tail.next = map_array;
4096 }
4097 map_array->head.mod = mod;
4098 map_array->head.length = len;
4099 map_array++;
4100
4101 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
4102 map_array->map = **map;
4103 map_array++;
4104 }
4105 memset(map_array, 0, sizeof(*map_array));
4106
4107 mutex_unlock(&trace_enum_mutex);
4108}
4109
4110static void trace_create_enum_file(struct dentry *d_tracer)
4111{
4112 trace_create_file("enum_map", 0444, d_tracer,
4113 NULL, &tracing_enum_map_fops);
4114}
4115
4116#else /* CONFIG_TRACE_ENUM_MAP_FILE */
4117static inline void trace_create_enum_file(struct dentry *d_tracer) { }
4118static inline void trace_insert_enum_map_file(struct module *mod,
4119 struct trace_enum_map **start, int len) { }
4120#endif /* !CONFIG_TRACE_ENUM_MAP_FILE */
4121
4122static void trace_insert_enum_map(struct module *mod,
4123 struct trace_enum_map **start, int len)
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04004124{
4125 struct trace_enum_map **map;
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04004126
4127 if (len <= 0)
4128 return;
4129
4130 map = start;
4131
4132 trace_event_enum_update(map, len);
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04004133
4134 trace_insert_enum_map_file(mod, start, len);
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04004135}
4136
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09004137static ssize_t
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004138tracing_set_trace_read(struct file *filp, char __user *ubuf,
4139 size_t cnt, loff_t *ppos)
4140{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004141 struct trace_array *tr = filp->private_data;
Li Zefanee6c2c12009-09-18 14:06:47 +08004142 char buf[MAX_TRACER_SIZE+2];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004143 int r;
4144
4145 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004146 r = sprintf(buf, "%s\n", tr->current_trace->name);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004147 mutex_unlock(&trace_types_lock);
4148
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004149 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004150}
4151
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02004152int tracer_init(struct tracer *t, struct trace_array *tr)
4153{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004154 tracing_reset_online_cpus(&tr->trace_buffer);
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02004155 return t->init(tr);
4156}
4157
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004158static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004159{
4160 int cpu;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05004161
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004162 for_each_tracing_cpu(cpu)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004163 per_cpu_ptr(buf->data, cpu)->entries = val;
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004164}
4165
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004166#ifdef CONFIG_TRACER_MAX_TRACE
Hiraku Toyookad60da502012-10-17 11:56:16 +09004167/* resize @tr's buffer to the size of @size_tr's entries */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004168static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
4169 struct trace_buffer *size_buf, int cpu_id)
Hiraku Toyookad60da502012-10-17 11:56:16 +09004170{
4171 int cpu, ret = 0;
4172
4173 if (cpu_id == RING_BUFFER_ALL_CPUS) {
4174 for_each_tracing_cpu(cpu) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004175 ret = ring_buffer_resize(trace_buf->buffer,
4176 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
Hiraku Toyookad60da502012-10-17 11:56:16 +09004177 if (ret < 0)
4178 break;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004179 per_cpu_ptr(trace_buf->data, cpu)->entries =
4180 per_cpu_ptr(size_buf->data, cpu)->entries;
Hiraku Toyookad60da502012-10-17 11:56:16 +09004181 }
4182 } else {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004183 ret = ring_buffer_resize(trace_buf->buffer,
4184 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
Hiraku Toyookad60da502012-10-17 11:56:16 +09004185 if (ret == 0)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004186 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
4187 per_cpu_ptr(size_buf->data, cpu_id)->entries;
Hiraku Toyookad60da502012-10-17 11:56:16 +09004188 }
4189
4190 return ret;
4191}
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004192#endif /* CONFIG_TRACER_MAX_TRACE */
Hiraku Toyookad60da502012-10-17 11:56:16 +09004193
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004194static int __tracing_resize_ring_buffer(struct trace_array *tr,
4195 unsigned long size, int cpu)
Steven Rostedt73c51622009-03-11 13:42:01 -04004196{
4197 int ret;
4198
4199 /*
4200 * If kernel or user changes the size of the ring buffer
Steven Rostedta123c522009-03-12 11:21:08 -04004201 * we use the size that was given, and we can forget about
4202 * expanding it later.
Steven Rostedt73c51622009-03-11 13:42:01 -04004203 */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05004204 ring_buffer_expanded = true;
Steven Rostedt73c51622009-03-11 13:42:01 -04004205
Steven Rostedtb382ede62012-10-10 21:44:34 -04004206 /* May be called before buffers are initialized */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004207 if (!tr->trace_buffer.buffer)
Steven Rostedtb382ede62012-10-10 21:44:34 -04004208 return 0;
4209
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004210 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04004211 if (ret < 0)
4212 return ret;
4213
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004214#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004215 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
4216 !tr->current_trace->use_max_tr)
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09004217 goto out;
4218
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004219 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04004220 if (ret < 0) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004221 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
4222 &tr->trace_buffer, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04004223 if (r < 0) {
Steven Rostedta123c522009-03-12 11:21:08 -04004224 /*
4225 * AARGH! We are left with different
4226 * size max buffer!!!!
4227 * The max buffer is our "snapshot" buffer.
4228 * When a tracer needs a snapshot (one of the
4229 * latency tracers), it swaps the max buffer
4230 * with the saved snap shot. We succeeded to
4231 * update the size of the main buffer, but failed to
4232 * update the size of the max buffer. But when we tried
4233 * to reset the main buffer to the original size, we
4234 * failed there too. This is very unlikely to
4235 * happen, but if it does, warn and kill all
4236 * tracing.
4237 */
Steven Rostedt73c51622009-03-11 13:42:01 -04004238 WARN_ON(1);
4239 tracing_disabled = 1;
4240 }
4241 return ret;
4242 }
4243
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004244 if (cpu == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004245 set_buffer_entries(&tr->max_buffer, size);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004246 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004247 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004248
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09004249 out:
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004250#endif /* CONFIG_TRACER_MAX_TRACE */
4251
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004252 if (cpu == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004253 set_buffer_entries(&tr->trace_buffer, size);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004254 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004255 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
Steven Rostedt73c51622009-03-11 13:42:01 -04004256
4257 return ret;
4258}
4259
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004260static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4261 unsigned long size, int cpu_id)
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004262{
Vaibhav Nagarnaik83f40312012-05-03 18:59:50 -07004263 int ret = size;
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004264
4265 mutex_lock(&trace_types_lock);
4266
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004267 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4268 /* make sure, this cpu is enabled in the mask */
4269 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4270 ret = -EINVAL;
4271 goto out;
4272 }
4273 }
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004274
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004275 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004276 if (ret < 0)
4277 ret = -ENOMEM;
4278
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004279out:
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004280 mutex_unlock(&trace_types_lock);
4281
4282 return ret;
4283}
4284
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09004285
Steven Rostedt1852fcc2009-03-11 14:33:00 -04004286/**
4287 * tracing_update_buffers - used by tracing facility to expand ring buffers
4288 *
4289 * To save on memory when the tracing is never used on a system with it
4290 * configured in. The ring buffers are set to a minimum size. But once
4291 * a user starts to use the tracing facility, then they need to grow
4292 * to their default size.
4293 *
4294 * This function is to be called when a tracer is about to be used.
4295 */
4296int tracing_update_buffers(void)
4297{
4298 int ret = 0;
4299
Steven Rostedt1027fcb2009-03-12 11:33:20 -04004300 mutex_lock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04004301 if (!ring_buffer_expanded)
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004302 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004303 RING_BUFFER_ALL_CPUS);
Steven Rostedt1027fcb2009-03-12 11:33:20 -04004304 mutex_unlock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04004305
4306 return ret;
4307}
4308
Steven Rostedt577b7852009-02-26 23:43:05 -05004309struct trace_option_dentry;
4310
4311static struct trace_option_dentry *
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004312create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
Steven Rostedt577b7852009-02-26 23:43:05 -05004313
Steven Rostedt (Red Hat)6b450d22014-01-14 08:43:01 -05004314/*
4315 * Used to clear out the tracer before deletion of an instance.
4316 * Must have trace_types_lock held.
4317 */
4318static void tracing_set_nop(struct trace_array *tr)
4319{
4320 if (tr->current_trace == &nop_trace)
4321 return;
4322
Steven Rostedt (Red Hat)50512ab2014-01-14 08:52:35 -05004323 tr->current_trace->enabled--;
Steven Rostedt (Red Hat)6b450d22014-01-14 08:43:01 -05004324
4325 if (tr->current_trace->reset)
4326 tr->current_trace->reset(tr);
4327
4328 tr->current_trace = &nop_trace;
4329}
4330
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04004331static void add_tracer_options(struct trace_array *tr, struct tracer *t)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004332{
Steven Rostedt (Red Hat)09d23a12015-02-03 12:45:53 -05004333 /* Only enable if the directory has been created already. */
4334 if (!tr->dir)
4335 return;
4336
4337 /* Currently, only the top instance has options */
4338 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL))
4339 return;
4340
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04004341 /* Ignore if they were already created */
4342 if (t->topts)
4343 return;
4344
4345 t->topts = create_trace_option_files(tr, t);
Steven Rostedt (Red Hat)09d23a12015-02-03 12:45:53 -05004346}
4347
4348static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4349{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004350 struct tracer *t;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004351#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt34600f02013-01-22 13:35:11 -05004352 bool had_max_tr;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004353#endif
Peter Zijlstrad9e54072008-11-01 19:57:37 +01004354 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004355
Steven Rostedt1027fcb2009-03-12 11:33:20 -04004356 mutex_lock(&trace_types_lock);
4357
Steven Rostedt73c51622009-03-11 13:42:01 -04004358 if (!ring_buffer_expanded) {
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004359 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004360 RING_BUFFER_ALL_CPUS);
Steven Rostedt73c51622009-03-11 13:42:01 -04004361 if (ret < 0)
Frederic Weisbecker59f586d2009-03-15 22:10:39 +01004362 goto out;
Steven Rostedt73c51622009-03-11 13:42:01 -04004363 ret = 0;
4364 }
4365
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004366 for (t = trace_types; t; t = t->next) {
4367 if (strcmp(t->name, buf) == 0)
4368 break;
4369 }
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02004370 if (!t) {
4371 ret = -EINVAL;
4372 goto out;
4373 }
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004374 if (t == tr->current_trace)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004375 goto out;
4376
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05004377 /* Some tracers are only allowed for the top level buffer */
4378 if (!trace_ok_for_array(t, tr)) {
4379 ret = -EINVAL;
4380 goto out;
4381 }
4382
Steven Rostedt (Red Hat)cf6ab6d2014-12-15 20:13:31 -05004383 /* If trace pipe files are being read, we can't change the tracer */
4384 if (tr->current_trace->ref) {
4385 ret = -EBUSY;
4386 goto out;
4387 }
4388
Steven Rostedt9f029e82008-11-12 15:24:24 -05004389 trace_branch_disable();
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04004390
Steven Rostedt (Red Hat)50512ab2014-01-14 08:52:35 -05004391 tr->current_trace->enabled--;
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04004392
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004393 if (tr->current_trace->reset)
4394 tr->current_trace->reset(tr);
Steven Rostedt34600f02013-01-22 13:35:11 -05004395
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004396 /* Current trace needs to be nop_trace before synchronize_sched */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004397 tr->current_trace = &nop_trace;
Steven Rostedt34600f02013-01-22 13:35:11 -05004398
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05004399#ifdef CONFIG_TRACER_MAX_TRACE
4400 had_max_tr = tr->allocated_snapshot;
Steven Rostedt34600f02013-01-22 13:35:11 -05004401
4402 if (had_max_tr && !t->use_max_tr) {
4403 /*
4404 * We need to make sure that the update_max_tr sees that
4405 * current_trace changed to nop_trace to keep it from
4406 * swapping the buffers after we resize it.
4407 * The update_max_tr is called from interrupts disabled
4408 * so a synchronized_sched() is sufficient.
4409 */
4410 synchronize_sched();
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04004411 free_snapshot(tr);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09004412 }
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004413#endif
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004414
4415#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt34600f02013-01-22 13:35:11 -05004416 if (t->use_max_tr && !had_max_tr) {
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04004417 ret = alloc_snapshot(tr);
Hiraku Toyookad60da502012-10-17 11:56:16 +09004418 if (ret < 0)
4419 goto out;
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09004420 }
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004421#endif
Steven Rostedt577b7852009-02-26 23:43:05 -05004422
Frederic Weisbecker1c800252008-11-16 05:57:26 +01004423 if (t->init) {
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02004424 ret = tracer_init(t, tr);
Frederic Weisbecker1c800252008-11-16 05:57:26 +01004425 if (ret)
4426 goto out;
4427 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004428
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004429 tr->current_trace = t;
Steven Rostedt (Red Hat)50512ab2014-01-14 08:52:35 -05004430 tr->current_trace->enabled++;
Steven Rostedt9f029e82008-11-12 15:24:24 -05004431 trace_branch_enable(tr);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004432 out:
4433 mutex_unlock(&trace_types_lock);
4434
Peter Zijlstrad9e54072008-11-01 19:57:37 +01004435 return ret;
4436}
4437
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004438static ssize_t
4439tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4440 size_t cnt, loff_t *ppos)
4441{
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05004442 struct trace_array *tr = filp->private_data;
Li Zefanee6c2c12009-09-18 14:06:47 +08004443 char buf[MAX_TRACER_SIZE+1];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004444 int i;
4445 size_t ret;
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01004446 int err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004447
Steven Rostedt60063a62008-10-28 10:44:24 -04004448 ret = cnt;
4449
Li Zefanee6c2c12009-09-18 14:06:47 +08004450 if (cnt > MAX_TRACER_SIZE)
4451 cnt = MAX_TRACER_SIZE;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004452
4453 if (copy_from_user(&buf, ubuf, cnt))
4454 return -EFAULT;
4455
4456 buf[cnt] = 0;
4457
4458 /* strip ending whitespace. */
4459 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4460 buf[i] = 0;
4461
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05004462 err = tracing_set_tracer(tr, buf);
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01004463 if (err)
4464 return err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004465
Jiri Olsacf8517c2009-10-23 19:36:16 -04004466 *ppos += ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004467
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02004468 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004469}
4470
4471static ssize_t
Stanislav Fomichev6508fa72014-07-18 15:17:27 +04004472tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
4473 size_t cnt, loff_t *ppos)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004474{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004475 char buf[64];
4476 int r;
4477
Steven Rostedtcffae432008-05-12 21:21:00 +02004478 r = snprintf(buf, sizeof(buf), "%ld\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004479 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
Steven Rostedtcffae432008-05-12 21:21:00 +02004480 if (r > sizeof(buf))
4481 r = sizeof(buf);
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004482 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004483}
4484
4485static ssize_t
Stanislav Fomichev6508fa72014-07-18 15:17:27 +04004486tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
4487 size_t cnt, loff_t *ppos)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004488{
Hannes Eder5e398412009-02-10 19:44:34 +01004489 unsigned long val;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02004490 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004491
Peter Huewe22fe9b52011-06-07 21:58:27 +02004492 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4493 if (ret)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02004494 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004495
4496 *ptr = val * 1000;
4497
4498 return cnt;
4499}
4500
Stanislav Fomichev6508fa72014-07-18 15:17:27 +04004501static ssize_t
4502tracing_thresh_read(struct file *filp, char __user *ubuf,
4503 size_t cnt, loff_t *ppos)
4504{
4505 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
4506}
4507
4508static ssize_t
4509tracing_thresh_write(struct file *filp, const char __user *ubuf,
4510 size_t cnt, loff_t *ppos)
4511{
4512 struct trace_array *tr = filp->private_data;
4513 int ret;
4514
4515 mutex_lock(&trace_types_lock);
4516 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
4517 if (ret < 0)
4518 goto out;
4519
4520 if (tr->current_trace->update_thresh) {
4521 ret = tr->current_trace->update_thresh(tr);
4522 if (ret < 0)
4523 goto out;
4524 }
4525
4526 ret = cnt;
4527out:
4528 mutex_unlock(&trace_types_lock);
4529
4530 return ret;
4531}
4532
4533static ssize_t
4534tracing_max_lat_read(struct file *filp, char __user *ubuf,
4535 size_t cnt, loff_t *ppos)
4536{
4537 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
4538}
4539
4540static ssize_t
4541tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4542 size_t cnt, loff_t *ppos)
4543{
4544 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
4545}
4546
Steven Rostedtb3806b42008-05-12 21:20:46 +02004547static int tracing_open_pipe(struct inode *inode, struct file *filp)
4548{
Oleg Nesterov15544202013-07-23 17:25:57 +02004549 struct trace_array *tr = inode->i_private;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004550 struct trace_iterator *iter;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004551 int ret = 0;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004552
4553 if (tracing_disabled)
4554 return -ENODEV;
4555
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004556 if (trace_array_get(tr) < 0)
4557 return -ENODEV;
4558
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004559 mutex_lock(&trace_types_lock);
4560
Steven Rostedtb3806b42008-05-12 21:20:46 +02004561 /* create a buffer to store the information to pass to userspace */
4562 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004563 if (!iter) {
4564 ret = -ENOMEM;
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07004565 __trace_array_put(tr);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004566 goto out;
4567 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02004568
Steven Rostedt (Red Hat)3a161d92014-06-25 15:54:42 -04004569 trace_seq_init(&iter->seq);
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05004570 iter->trace = tr->current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004571
4572 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4573 ret = -ENOMEM;
4574 goto fail;
Rusty Russell44623442009-01-01 10:12:23 +10304575 }
4576
Steven Rostedta3097202008-11-07 22:36:02 -05004577 /* trace pipe does not show start of buffer */
Rusty Russell44623442009-01-01 10:12:23 +10304578 cpumask_setall(iter->started);
Steven Rostedta3097202008-11-07 22:36:02 -05004579
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04004580 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
Steven Rostedt112f38a72009-06-01 15:16:05 -04004581 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4582
David Sharp8be07092012-11-13 12:18:22 -08004583 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
Yoshihiro YUNOMAE58e8eed2013-04-23 10:32:39 +09004584 if (trace_clocks[tr->clock_id].in_ns)
David Sharp8be07092012-11-13 12:18:22 -08004585 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4586
Oleg Nesterov15544202013-07-23 17:25:57 +02004587 iter->tr = tr;
4588 iter->trace_buffer = &tr->trace_buffer;
4589 iter->cpu_file = tracing_get_cpu(inode);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004590 mutex_init(&iter->mutex);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004591 filp->private_data = iter;
4592
Steven Rostedt107bad82008-05-12 21:21:01 +02004593 if (iter->trace->pipe_open)
4594 iter->trace->pipe_open(iter);
Steven Rostedt107bad82008-05-12 21:21:01 +02004595
Arnd Bergmannb4447862010-07-07 23:40:11 +02004596 nonseekable_open(inode, filp);
Steven Rostedt (Red Hat)cf6ab6d2014-12-15 20:13:31 -05004597
4598 tr->current_trace->ref++;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004599out:
4600 mutex_unlock(&trace_types_lock);
4601 return ret;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004602
4603fail:
4604 kfree(iter->trace);
4605 kfree(iter);
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004606 __trace_array_put(tr);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004607 mutex_unlock(&trace_types_lock);
4608 return ret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004609}
4610
4611static int tracing_release_pipe(struct inode *inode, struct file *file)
4612{
4613 struct trace_iterator *iter = file->private_data;
Oleg Nesterov15544202013-07-23 17:25:57 +02004614 struct trace_array *tr = inode->i_private;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004615
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004616 mutex_lock(&trace_types_lock);
4617
Steven Rostedt (Red Hat)cf6ab6d2014-12-15 20:13:31 -05004618 tr->current_trace->ref--;
4619
Steven Rostedt29bf4a52009-12-09 12:37:43 -05004620 if (iter->trace->pipe_close)
Steven Rostedtc521efd2009-12-07 09:06:24 -05004621 iter->trace->pipe_close(iter);
4622
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004623 mutex_unlock(&trace_types_lock);
4624
Rusty Russell44623442009-01-01 10:12:23 +10304625 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004626 mutex_destroy(&iter->mutex);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004627 kfree(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004628
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004629 trace_array_put(tr);
4630
Steven Rostedtb3806b42008-05-12 21:20:46 +02004631 return 0;
4632}
4633
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004634static unsigned int
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004635trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004636{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04004637 struct trace_array *tr = iter->tr;
4638
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05004639 /* Iterators are static, they should be filled or empty */
4640 if (trace_buffer_iter(iter, iter->cpu_file))
4641 return POLLIN | POLLRDNORM;
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004642
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04004643 if (tr->trace_flags & TRACE_ITER_BLOCK)
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004644 /*
4645 * Always select as readable when in blocking mode
4646 */
4647 return POLLIN | POLLRDNORM;
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05004648 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004649 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05004650 filp, poll_table);
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004651}
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004652
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004653static unsigned int
4654tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4655{
4656 struct trace_iterator *iter = filp->private_data;
4657
4658 return trace_poll(iter, filp, poll_table);
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004659}
4660
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05004661/* Must be called with iter->mutex held. */
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004662static int tracing_wait_pipe(struct file *filp)
4663{
4664 struct trace_iterator *iter = filp->private_data;
Steven Rostedt (Red Hat)8b8b3682014-06-10 09:46:00 -04004665 int ret;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004666
4667 while (trace_empty(iter)) {
4668
4669 if ((filp->f_flags & O_NONBLOCK)) {
4670 return -EAGAIN;
4671 }
4672
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004673 /*
Liu Bo250bfd32013-01-14 10:54:11 +08004674 * We block until we read something and tracing is disabled.
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004675 * We still block if tracing is disabled, but we have never
4676 * read anything. This allows a user to cat this file, and
4677 * then enable tracing. But after we have read something,
4678 * we give an EOF when tracing is again disabled.
4679 *
4680 * iter->pos will be 0 if we haven't read anything.
4681 */
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -04004682 if (!tracing_is_on() && iter->pos)
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004683 break;
Steven Rostedt (Red Hat)f4874262014-04-29 16:07:28 -04004684
4685 mutex_unlock(&iter->mutex);
4686
Rabin Vincente30f53a2014-11-10 19:46:34 +01004687 ret = wait_on_pipe(iter, false);
Steven Rostedt (Red Hat)f4874262014-04-29 16:07:28 -04004688
4689 mutex_lock(&iter->mutex);
4690
Steven Rostedt (Red Hat)8b8b3682014-06-10 09:46:00 -04004691 if (ret)
4692 return ret;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004693 }
4694
4695 return 1;
4696}
4697
Steven Rostedtb3806b42008-05-12 21:20:46 +02004698/*
4699 * Consumer reader.
4700 */
4701static ssize_t
4702tracing_read_pipe(struct file *filp, char __user *ubuf,
4703 size_t cnt, loff_t *ppos)
4704{
4705 struct trace_iterator *iter = filp->private_data;
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004706 ssize_t sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004707
4708 /* return any leftover data */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004709 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4710 if (sret != -EBUSY)
4711 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004712
Steven Rostedtf9520752009-03-02 14:04:40 -05004713 trace_seq_init(&iter->seq);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004714
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004715 /*
4716 * Avoid more than one consumer on a single file descriptor
4717 * This is just a matter of traces coherency, the ring buffer itself
4718 * is protected.
4719 */
4720 mutex_lock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02004721 if (iter->trace->read) {
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004722 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4723 if (sret)
Steven Rostedt107bad82008-05-12 21:21:01 +02004724 goto out;
Steven Rostedt107bad82008-05-12 21:21:01 +02004725 }
4726
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02004727waitagain:
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004728 sret = tracing_wait_pipe(filp);
4729 if (sret <= 0)
4730 goto out;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004731
4732 /* stop when tracing is finished */
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004733 if (trace_empty(iter)) {
4734 sret = 0;
Steven Rostedt107bad82008-05-12 21:21:01 +02004735 goto out;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004736 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02004737
4738 if (cnt >= PAGE_SIZE)
4739 cnt = PAGE_SIZE - 1;
4740
Steven Rostedt53d0aa72008-05-12 21:21:01 +02004741 /* reset all but tr, trace, and overruns */
Steven Rostedt53d0aa72008-05-12 21:21:01 +02004742 memset(&iter->seq, 0,
4743 sizeof(struct trace_iterator) -
4744 offsetof(struct trace_iterator, seq));
Andrew Vagined5467d2013-08-02 21:16:43 +04004745 cpumask_clear(iter->started);
Steven Rostedt4823ed72008-05-12 21:21:01 +02004746 iter->pos = -1;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004747
Lai Jiangshan4f535962009-05-18 19:35:34 +08004748 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004749 trace_access_lock(iter->cpu_file);
Jason Wessel955b61e2010-08-05 09:22:23 -05004750 while (trace_find_next_entry_inc(iter) != NULL) {
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02004751 enum print_line_t ret;
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004752 int save_len = iter->seq.seq.len;
Steven Rostedt088b1e422008-05-12 21:20:48 +02004753
Ingo Molnarf9896bf2008-05-12 21:20:47 +02004754 ret = print_trace_line(iter);
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02004755 if (ret == TRACE_TYPE_PARTIAL_LINE) {
Steven Rostedt088b1e422008-05-12 21:20:48 +02004756 /* don't print partial lines */
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004757 iter->seq.seq.len = save_len;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004758 break;
Steven Rostedt088b1e422008-05-12 21:20:48 +02004759 }
Frederic Weisbeckerb91facc2009-02-06 18:30:44 +01004760 if (ret != TRACE_TYPE_NO_CONSUME)
4761 trace_consume(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004762
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004763 if (trace_seq_used(&iter->seq) >= cnt)
Steven Rostedtb3806b42008-05-12 21:20:46 +02004764 break;
Jiri Olsaee5e51f2011-03-25 12:05:18 +01004765
4766 /*
4767 * Setting the full flag means we reached the trace_seq buffer
4768 * size and we should leave by partial output condition above.
4769 * One of the trace_seq_* functions is not used properly.
4770 */
4771 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4772 iter->ent->type);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004773 }
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004774 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08004775 trace_event_read_unlock();
Steven Rostedtb3806b42008-05-12 21:20:46 +02004776
Steven Rostedtb3806b42008-05-12 21:20:46 +02004777 /* Now copy what we have to the user */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004778 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004779 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
Steven Rostedtf9520752009-03-02 14:04:40 -05004780 trace_seq_init(&iter->seq);
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02004781
4782 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03004783 * If there was nothing to send to user, in spite of consuming trace
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02004784 * entries, go back to wait for more entries.
4785 */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004786 if (sret == -EBUSY)
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02004787 goto waitagain;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004788
Steven Rostedt107bad82008-05-12 21:21:01 +02004789out:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004790 mutex_unlock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02004791
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004792 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004793}
4794
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004795static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4796 unsigned int idx)
4797{
4798 __free_page(spd->pages[idx]);
4799}
4800
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08004801static const struct pipe_buf_operations tracing_pipe_buf_ops = {
Steven Rostedt34cd4992009-02-09 12:06:29 -05004802 .can_merge = 0,
Steven Rostedt34cd4992009-02-09 12:06:29 -05004803 .confirm = generic_pipe_buf_confirm,
Al Viro92fdd982014-01-17 07:53:39 -05004804 .release = generic_pipe_buf_release,
Steven Rostedt34cd4992009-02-09 12:06:29 -05004805 .steal = generic_pipe_buf_steal,
4806 .get = generic_pipe_buf_get,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004807};
4808
Steven Rostedt34cd4992009-02-09 12:06:29 -05004809static size_t
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01004810tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
Steven Rostedt34cd4992009-02-09 12:06:29 -05004811{
4812 size_t count;
Steven Rostedt (Red Hat)74f06bb2014-11-17 13:12:22 -05004813 int save_len;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004814 int ret;
4815
4816 /* Seq buffer is page-sized, exactly what we need. */
4817 for (;;) {
Steven Rostedt (Red Hat)74f06bb2014-11-17 13:12:22 -05004818 save_len = iter->seq.seq.len;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004819 ret = print_trace_line(iter);
Steven Rostedt (Red Hat)74f06bb2014-11-17 13:12:22 -05004820
4821 if (trace_seq_has_overflowed(&iter->seq)) {
4822 iter->seq.seq.len = save_len;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004823 break;
4824 }
Steven Rostedt (Red Hat)74f06bb2014-11-17 13:12:22 -05004825
4826 /*
4827 * This should not be hit, because it should only
4828 * be set if the iter->seq overflowed. But check it
4829 * anyway to be safe.
4830 */
Steven Rostedt34cd4992009-02-09 12:06:29 -05004831 if (ret == TRACE_TYPE_PARTIAL_LINE) {
Steven Rostedt (Red Hat)74f06bb2014-11-17 13:12:22 -05004832 iter->seq.seq.len = save_len;
4833 break;
4834 }
4835
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004836 count = trace_seq_used(&iter->seq) - save_len;
Steven Rostedt (Red Hat)74f06bb2014-11-17 13:12:22 -05004837 if (rem < count) {
4838 rem = 0;
4839 iter->seq.seq.len = save_len;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004840 break;
4841 }
4842
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08004843 if (ret != TRACE_TYPE_NO_CONSUME)
4844 trace_consume(iter);
Steven Rostedt34cd4992009-02-09 12:06:29 -05004845 rem -= count;
Jason Wessel955b61e2010-08-05 09:22:23 -05004846 if (!trace_find_next_entry_inc(iter)) {
Steven Rostedt34cd4992009-02-09 12:06:29 -05004847 rem = 0;
4848 iter->ent = NULL;
4849 break;
4850 }
4851 }
4852
4853 return rem;
4854}
4855
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004856static ssize_t tracing_splice_read_pipe(struct file *filp,
4857 loff_t *ppos,
4858 struct pipe_inode_info *pipe,
4859 size_t len,
4860 unsigned int flags)
4861{
Jens Axboe35f3d142010-05-20 10:43:18 +02004862 struct page *pages_def[PIPE_DEF_BUFFERS];
4863 struct partial_page partial_def[PIPE_DEF_BUFFERS];
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004864 struct trace_iterator *iter = filp->private_data;
4865 struct splice_pipe_desc spd = {
Jens Axboe35f3d142010-05-20 10:43:18 +02004866 .pages = pages_def,
4867 .partial = partial_def,
Steven Rostedt34cd4992009-02-09 12:06:29 -05004868 .nr_pages = 0, /* This gets updated below. */
Eric Dumazet047fe362012-06-12 15:24:40 +02004869 .nr_pages_max = PIPE_DEF_BUFFERS,
Steven Rostedt34cd4992009-02-09 12:06:29 -05004870 .flags = flags,
4871 .ops = &tracing_pipe_buf_ops,
4872 .spd_release = tracing_spd_release_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004873 };
4874 ssize_t ret;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004875 size_t rem;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004876 unsigned int i;
4877
Jens Axboe35f3d142010-05-20 10:43:18 +02004878 if (splice_grow_spd(pipe, &spd))
4879 return -ENOMEM;
4880
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004881 mutex_lock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004882
4883 if (iter->trace->splice_read) {
4884 ret = iter->trace->splice_read(iter, filp,
4885 ppos, pipe, len, flags);
4886 if (ret)
Steven Rostedt34cd4992009-02-09 12:06:29 -05004887 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004888 }
4889
4890 ret = tracing_wait_pipe(filp);
4891 if (ret <= 0)
Steven Rostedt34cd4992009-02-09 12:06:29 -05004892 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004893
Jason Wessel955b61e2010-08-05 09:22:23 -05004894 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004895 ret = -EFAULT;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004896 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004897 }
4898
Lai Jiangshan4f535962009-05-18 19:35:34 +08004899 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004900 trace_access_lock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08004901
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004902 /* Fill as many pages as possible. */
Al Viroa786c062014-04-11 12:01:03 -04004903 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
Jens Axboe35f3d142010-05-20 10:43:18 +02004904 spd.pages[i] = alloc_page(GFP_KERNEL);
4905 if (!spd.pages[i])
Steven Rostedt34cd4992009-02-09 12:06:29 -05004906 break;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004907
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01004908 rem = tracing_fill_pipe_page(rem, iter);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004909
4910 /* Copy the data into the page, so we can start over. */
4911 ret = trace_seq_to_buffer(&iter->seq,
Jens Axboe35f3d142010-05-20 10:43:18 +02004912 page_address(spd.pages[i]),
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004913 trace_seq_used(&iter->seq));
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004914 if (ret < 0) {
Jens Axboe35f3d142010-05-20 10:43:18 +02004915 __free_page(spd.pages[i]);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004916 break;
4917 }
Jens Axboe35f3d142010-05-20 10:43:18 +02004918 spd.partial[i].offset = 0;
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004919 spd.partial[i].len = trace_seq_used(&iter->seq);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004920
Steven Rostedtf9520752009-03-02 14:04:40 -05004921 trace_seq_init(&iter->seq);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004922 }
4923
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004924 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08004925 trace_event_read_unlock();
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004926 mutex_unlock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004927
4928 spd.nr_pages = i;
4929
Jens Axboe35f3d142010-05-20 10:43:18 +02004930 ret = splice_to_pipe(pipe, &spd);
4931out:
Eric Dumazet047fe362012-06-12 15:24:40 +02004932 splice_shrink_spd(&spd);
Jens Axboe35f3d142010-05-20 10:43:18 +02004933 return ret;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004934
Steven Rostedt34cd4992009-02-09 12:06:29 -05004935out_err:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004936 mutex_unlock(&iter->mutex);
Jens Axboe35f3d142010-05-20 10:43:18 +02004937 goto out;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004938}
4939
Steven Rostedta98a3c32008-05-12 21:20:59 +02004940static ssize_t
4941tracing_entries_read(struct file *filp, char __user *ubuf,
4942 size_t cnt, loff_t *ppos)
4943{
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02004944 struct inode *inode = file_inode(filp);
4945 struct trace_array *tr = inode->i_private;
4946 int cpu = tracing_get_cpu(inode);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004947 char buf[64];
4948 int r = 0;
4949 ssize_t ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004950
Steven Rostedtdb526ca2009-03-12 13:53:25 -04004951 mutex_lock(&trace_types_lock);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004952
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02004953 if (cpu == RING_BUFFER_ALL_CPUS) {
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004954 int cpu, buf_size_same;
4955 unsigned long size;
4956
4957 size = 0;
4958 buf_size_same = 1;
4959 /* check if all cpu sizes are same */
4960 for_each_tracing_cpu(cpu) {
4961 /* fill in the size from first enabled cpu */
4962 if (size == 0)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004963 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4964 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004965 buf_size_same = 0;
4966 break;
4967 }
4968 }
4969
4970 if (buf_size_same) {
4971 if (!ring_buffer_expanded)
4972 r = sprintf(buf, "%lu (expanded: %lu)\n",
4973 size >> 10,
4974 trace_buf_size >> 10);
4975 else
4976 r = sprintf(buf, "%lu\n", size >> 10);
4977 } else
4978 r = sprintf(buf, "X\n");
4979 } else
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02004980 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004981
Steven Rostedtdb526ca2009-03-12 13:53:25 -04004982 mutex_unlock(&trace_types_lock);
4983
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004984 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4985 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004986}
4987
4988static ssize_t
4989tracing_entries_write(struct file *filp, const char __user *ubuf,
4990 size_t cnt, loff_t *ppos)
4991{
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02004992 struct inode *inode = file_inode(filp);
4993 struct trace_array *tr = inode->i_private;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004994 unsigned long val;
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004995 int ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004996
Peter Huewe22fe9b52011-06-07 21:58:27 +02004997 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4998 if (ret)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02004999 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02005000
5001 /* must have at least 1 entry */
5002 if (!val)
5003 return -EINVAL;
5004
Steven Rostedt1696b2b2008-11-13 00:09:35 -05005005 /* value is in KB */
5006 val <<= 10;
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02005007 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005008 if (ret < 0)
5009 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02005010
Jiri Olsacf8517c2009-10-23 19:36:16 -04005011 *ppos += cnt;
Steven Rostedta98a3c32008-05-12 21:20:59 +02005012
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005013 return cnt;
5014}
Steven Rostedtbf5e6512008-11-10 21:46:00 -05005015
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005016static ssize_t
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07005017tracing_total_entries_read(struct file *filp, char __user *ubuf,
5018 size_t cnt, loff_t *ppos)
5019{
5020 struct trace_array *tr = filp->private_data;
5021 char buf[64];
5022 int r, cpu;
5023 unsigned long size = 0, expanded_size = 0;
5024
5025 mutex_lock(&trace_types_lock);
5026 for_each_tracing_cpu(cpu) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005027 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07005028 if (!ring_buffer_expanded)
5029 expanded_size += trace_buf_size >> 10;
5030 }
5031 if (ring_buffer_expanded)
5032 r = sprintf(buf, "%lu\n", size);
5033 else
5034 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
5035 mutex_unlock(&trace_types_lock);
5036
5037 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5038}
5039
5040static ssize_t
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005041tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
5042 size_t cnt, loff_t *ppos)
5043{
5044 /*
5045 * There is no need to read what the user has written, this function
5046 * is just to make sure that there is no error when "echo" is used
5047 */
5048
5049 *ppos += cnt;
Steven Rostedta98a3c32008-05-12 21:20:59 +02005050
5051 return cnt;
5052}
5053
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005054static int
5055tracing_free_buffer_release(struct inode *inode, struct file *filp)
5056{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005057 struct trace_array *tr = inode->i_private;
5058
Steven Rostedtcf30cf62011-06-14 22:44:07 -04005059 /* disable tracing ? */
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04005060 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
Alexander Z Lam711e1242013-08-02 18:36:15 -07005061 tracer_tracing_off(tr);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005062 /* resize the ring buffer to 0 */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005063 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005064
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005065 trace_array_put(tr);
5066
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005067 return 0;
5068}
5069
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005070static ssize_t
5071tracing_mark_write(struct file *filp, const char __user *ubuf,
5072 size_t cnt, loff_t *fpos)
5073{
Steven Rostedtd696b582011-09-22 11:50:27 -04005074 unsigned long addr = (unsigned long)ubuf;
Alexander Z Lam2d716192013-07-01 15:31:24 -07005075 struct trace_array *tr = filp->private_data;
Steven Rostedtd696b582011-09-22 11:50:27 -04005076 struct ring_buffer_event *event;
5077 struct ring_buffer *buffer;
5078 struct print_entry *entry;
5079 unsigned long irq_flags;
5080 struct page *pages[2];
Steven Rostedt6edb2a82012-05-11 23:28:49 -04005081 void *map_page[2];
Steven Rostedtd696b582011-09-22 11:50:27 -04005082 int nr_pages = 1;
5083 ssize_t written;
Steven Rostedtd696b582011-09-22 11:50:27 -04005084 int offset;
5085 int size;
5086 int len;
5087 int ret;
Steven Rostedt6edb2a82012-05-11 23:28:49 -04005088 int i;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005089
Steven Rostedtc76f0692008-11-07 22:36:02 -05005090 if (tracing_disabled)
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005091 return -EINVAL;
5092
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04005093 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
Mandeep Singh Baines5224c3a2012-09-07 18:12:19 -07005094 return -EINVAL;
5095
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005096 if (cnt > TRACE_BUF_SIZE)
5097 cnt = TRACE_BUF_SIZE;
5098
Steven Rostedtd696b582011-09-22 11:50:27 -04005099 /*
5100 * Userspace is injecting traces into the kernel trace buffer.
5101 * We want to be as non intrusive as possible.
5102 * To do so, we do not want to allocate any special buffers
5103 * or take any locks, but instead write the userspace data
5104 * straight into the ring buffer.
5105 *
5106 * First we need to pin the userspace buffer into memory,
5107 * which, most likely it is, because it just referenced it.
5108 * But there's no guarantee that it is. By using get_user_pages_fast()
5109 * and kmap_atomic/kunmap_atomic() we can get access to the
5110 * pages directly. We then write the data directly into the
5111 * ring buffer.
5112 */
5113 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005114
Steven Rostedtd696b582011-09-22 11:50:27 -04005115 /* check if we cross pages */
5116 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
5117 nr_pages = 2;
5118
5119 offset = addr & (PAGE_SIZE - 1);
5120 addr &= PAGE_MASK;
5121
5122 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
5123 if (ret < nr_pages) {
5124 while (--ret >= 0)
5125 put_page(pages[ret]);
5126 written = -EFAULT;
5127 goto out;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005128 }
5129
Steven Rostedt6edb2a82012-05-11 23:28:49 -04005130 for (i = 0; i < nr_pages; i++)
5131 map_page[i] = kmap_atomic(pages[i]);
Steven Rostedtd696b582011-09-22 11:50:27 -04005132
5133 local_save_flags(irq_flags);
5134 size = sizeof(*entry) + cnt + 2; /* possible \n added */
Alexander Z Lam2d716192013-07-01 15:31:24 -07005135 buffer = tr->trace_buffer.buffer;
Steven Rostedtd696b582011-09-22 11:50:27 -04005136 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
5137 irq_flags, preempt_count());
5138 if (!event) {
5139 /* Ring buffer disabled, return as if not open for write */
5140 written = -EBADF;
5141 goto out_unlock;
5142 }
5143
5144 entry = ring_buffer_event_data(event);
5145 entry->ip = _THIS_IP_;
5146
5147 if (nr_pages == 2) {
5148 len = PAGE_SIZE - offset;
Steven Rostedt6edb2a82012-05-11 23:28:49 -04005149 memcpy(&entry->buf, map_page[0] + offset, len);
5150 memcpy(&entry->buf[len], map_page[1], cnt - len);
Steven Rostedtd696b582011-09-22 11:50:27 -04005151 } else
Steven Rostedt6edb2a82012-05-11 23:28:49 -04005152 memcpy(&entry->buf, map_page[0] + offset, cnt);
Steven Rostedtd696b582011-09-22 11:50:27 -04005153
5154 if (entry->buf[cnt - 1] != '\n') {
5155 entry->buf[cnt] = '\n';
5156 entry->buf[cnt + 1] = '\0';
5157 } else
5158 entry->buf[cnt] = '\0';
5159
Steven Rostedt7ffbd482012-10-11 12:14:25 -04005160 __buffer_unlock_commit(buffer, event);
Steven Rostedtd696b582011-09-22 11:50:27 -04005161
5162 written = cnt;
5163
Marcin Slusarz1aa54bc2010-07-28 01:18:01 +02005164 *fpos += written;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005165
Steven Rostedtd696b582011-09-22 11:50:27 -04005166 out_unlock:
Vikram Mulukutla72158532014-12-17 18:50:56 -08005167 for (i = nr_pages - 1; i >= 0; i--) {
Steven Rostedt6edb2a82012-05-11 23:28:49 -04005168 kunmap_atomic(map_page[i]);
5169 put_page(pages[i]);
5170 }
Steven Rostedtd696b582011-09-22 11:50:27 -04005171 out:
Marcin Slusarz1aa54bc2010-07-28 01:18:01 +02005172 return written;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005173}
5174
Li Zefan13f16d22009-12-08 11:16:11 +08005175static int tracing_clock_show(struct seq_file *m, void *v)
Zhaolei5079f322009-08-25 16:12:56 +08005176{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005177 struct trace_array *tr = m->private;
Zhaolei5079f322009-08-25 16:12:56 +08005178 int i;
5179
5180 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
Li Zefan13f16d22009-12-08 11:16:11 +08005181 seq_printf(m,
Zhaolei5079f322009-08-25 16:12:56 +08005182 "%s%s%s%s", i ? " " : "",
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005183 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
5184 i == tr->clock_id ? "]" : "");
Li Zefan13f16d22009-12-08 11:16:11 +08005185 seq_putc(m, '\n');
Zhaolei5079f322009-08-25 16:12:56 +08005186
Li Zefan13f16d22009-12-08 11:16:11 +08005187 return 0;
Zhaolei5079f322009-08-25 16:12:56 +08005188}
5189
Steven Rostedte1e232c2014-02-10 23:38:46 -05005190static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
Zhaolei5079f322009-08-25 16:12:56 +08005191{
Zhaolei5079f322009-08-25 16:12:56 +08005192 int i;
5193
Zhaolei5079f322009-08-25 16:12:56 +08005194 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
5195 if (strcmp(trace_clocks[i].name, clockstr) == 0)
5196 break;
5197 }
5198 if (i == ARRAY_SIZE(trace_clocks))
5199 return -EINVAL;
5200
Zhaolei5079f322009-08-25 16:12:56 +08005201 mutex_lock(&trace_types_lock);
5202
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005203 tr->clock_id = i;
5204
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005205 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
Zhaolei5079f322009-08-25 16:12:56 +08005206
David Sharp60303ed2012-10-11 16:27:52 -07005207 /*
5208 * New clock may not be consistent with the previous clock.
5209 * Reset the buffer so that it doesn't have incomparable timestamps.
5210 */
Alexander Z Lam94571582013-08-02 18:36:16 -07005211 tracing_reset_online_cpus(&tr->trace_buffer);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005212
5213#ifdef CONFIG_TRACER_MAX_TRACE
5214 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
5215 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
Alexander Z Lam94571582013-08-02 18:36:16 -07005216 tracing_reset_online_cpus(&tr->max_buffer);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005217#endif
David Sharp60303ed2012-10-11 16:27:52 -07005218
Zhaolei5079f322009-08-25 16:12:56 +08005219 mutex_unlock(&trace_types_lock);
5220
Steven Rostedte1e232c2014-02-10 23:38:46 -05005221 return 0;
5222}
5223
5224static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
5225 size_t cnt, loff_t *fpos)
5226{
5227 struct seq_file *m = filp->private_data;
5228 struct trace_array *tr = m->private;
5229 char buf[64];
5230 const char *clockstr;
5231 int ret;
5232
5233 if (cnt >= sizeof(buf))
5234 return -EINVAL;
5235
5236 if (copy_from_user(&buf, ubuf, cnt))
5237 return -EFAULT;
5238
5239 buf[cnt] = 0;
5240
5241 clockstr = strstrip(buf);
5242
5243 ret = tracing_set_clock(tr, clockstr);
5244 if (ret)
5245 return ret;
5246
Zhaolei5079f322009-08-25 16:12:56 +08005247 *fpos += cnt;
5248
5249 return cnt;
5250}
5251
Li Zefan13f16d22009-12-08 11:16:11 +08005252static int tracing_clock_open(struct inode *inode, struct file *file)
5253{
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005254 struct trace_array *tr = inode->i_private;
5255 int ret;
5256
Li Zefan13f16d22009-12-08 11:16:11 +08005257 if (tracing_disabled)
5258 return -ENODEV;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005259
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005260 if (trace_array_get(tr))
5261 return -ENODEV;
5262
5263 ret = single_open(file, tracing_clock_show, inode->i_private);
5264 if (ret < 0)
5265 trace_array_put(tr);
5266
5267 return ret;
Li Zefan13f16d22009-12-08 11:16:11 +08005268}
5269
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005270struct ftrace_buffer_info {
5271 struct trace_iterator iter;
5272 void *spare;
5273 unsigned int read;
5274};
5275
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005276#ifdef CONFIG_TRACER_SNAPSHOT
5277static int tracing_snapshot_open(struct inode *inode, struct file *file)
5278{
Oleg Nesterov6484c712013-07-23 17:26:10 +02005279 struct trace_array *tr = inode->i_private;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005280 struct trace_iterator *iter;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005281 struct seq_file *m;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005282 int ret = 0;
5283
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04005284 if (trace_array_get(tr) < 0)
5285 return -ENODEV;
5286
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005287 if (file->f_mode & FMODE_READ) {
Oleg Nesterov6484c712013-07-23 17:26:10 +02005288 iter = __tracing_open(inode, file, true);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005289 if (IS_ERR(iter))
5290 ret = PTR_ERR(iter);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005291 } else {
5292 /* Writes still need the seq_file to hold the private data */
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07005293 ret = -ENOMEM;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005294 m = kzalloc(sizeof(*m), GFP_KERNEL);
5295 if (!m)
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07005296 goto out;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005297 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5298 if (!iter) {
5299 kfree(m);
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07005300 goto out;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005301 }
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07005302 ret = 0;
5303
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04005304 iter->tr = tr;
Oleg Nesterov6484c712013-07-23 17:26:10 +02005305 iter->trace_buffer = &tr->max_buffer;
5306 iter->cpu_file = tracing_get_cpu(inode);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005307 m->private = iter;
5308 file->private_data = m;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005309 }
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07005310out:
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04005311 if (ret < 0)
5312 trace_array_put(tr);
5313
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005314 return ret;
5315}
5316
5317static ssize_t
5318tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5319 loff_t *ppos)
5320{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005321 struct seq_file *m = filp->private_data;
5322 struct trace_iterator *iter = m->private;
5323 struct trace_array *tr = iter->tr;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005324 unsigned long val;
5325 int ret;
5326
5327 ret = tracing_update_buffers();
5328 if (ret < 0)
5329 return ret;
5330
5331 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5332 if (ret)
5333 return ret;
5334
5335 mutex_lock(&trace_types_lock);
5336
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005337 if (tr->current_trace->use_max_tr) {
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005338 ret = -EBUSY;
5339 goto out;
5340 }
5341
5342 switch (val) {
5343 case 0:
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05005344 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5345 ret = -EINVAL;
5346 break;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005347 }
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04005348 if (tr->allocated_snapshot)
5349 free_snapshot(tr);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005350 break;
5351 case 1:
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05005352/* Only allow per-cpu swap if the ring buffer supports it */
5353#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5354 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5355 ret = -EINVAL;
5356 break;
5357 }
5358#endif
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05005359 if (!tr->allocated_snapshot) {
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04005360 ret = alloc_snapshot(tr);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005361 if (ret < 0)
5362 break;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005363 }
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005364 local_irq_disable();
5365 /* Now, we're going to swap */
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05005366 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05005367 update_max_tr(tr, current, smp_processor_id());
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05005368 else
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05005369 update_max_tr_single(tr, current, iter->cpu_file);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005370 local_irq_enable();
5371 break;
5372 default:
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05005373 if (tr->allocated_snapshot) {
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05005374 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5375 tracing_reset_online_cpus(&tr->max_buffer);
5376 else
5377 tracing_reset(&tr->max_buffer, iter->cpu_file);
5378 }
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005379 break;
5380 }
5381
5382 if (ret >= 0) {
5383 *ppos += cnt;
5384 ret = cnt;
5385 }
5386out:
5387 mutex_unlock(&trace_types_lock);
5388 return ret;
5389}
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005390
5391static int tracing_snapshot_release(struct inode *inode, struct file *file)
5392{
5393 struct seq_file *m = file->private_data;
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04005394 int ret;
5395
5396 ret = tracing_release(inode, file);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005397
5398 if (file->f_mode & FMODE_READ)
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04005399 return ret;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005400
5401 /* If write only, the seq_file is just a stub */
5402 if (m)
5403 kfree(m->private);
5404 kfree(m);
5405
5406 return 0;
5407}
5408
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005409static int tracing_buffers_open(struct inode *inode, struct file *filp);
5410static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5411 size_t count, loff_t *ppos);
5412static int tracing_buffers_release(struct inode *inode, struct file *file);
5413static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5414 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5415
5416static int snapshot_raw_open(struct inode *inode, struct file *filp)
5417{
5418 struct ftrace_buffer_info *info;
5419 int ret;
5420
5421 ret = tracing_buffers_open(inode, filp);
5422 if (ret < 0)
5423 return ret;
5424
5425 info = filp->private_data;
5426
5427 if (info->iter.trace->use_max_tr) {
5428 tracing_buffers_release(inode, filp);
5429 return -EBUSY;
5430 }
5431
5432 info->iter.snapshot = true;
5433 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5434
5435 return ret;
5436}
5437
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005438#endif /* CONFIG_TRACER_SNAPSHOT */
5439
5440
Stanislav Fomichev6508fa72014-07-18 15:17:27 +04005441static const struct file_operations tracing_thresh_fops = {
5442 .open = tracing_open_generic,
5443 .read = tracing_thresh_read,
5444 .write = tracing_thresh_write,
5445 .llseek = generic_file_llseek,
5446};
5447
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005448static const struct file_operations tracing_max_lat_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005449 .open = tracing_open_generic,
5450 .read = tracing_max_lat_read,
5451 .write = tracing_max_lat_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005452 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005453};
5454
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005455static const struct file_operations set_tracer_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005456 .open = tracing_open_generic,
5457 .read = tracing_set_trace_read,
5458 .write = tracing_set_trace_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005459 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005460};
5461
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005462static const struct file_operations tracing_pipe_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005463 .open = tracing_open_pipe,
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02005464 .poll = tracing_poll_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005465 .read = tracing_read_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02005466 .splice_read = tracing_splice_read_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005467 .release = tracing_release_pipe,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005468 .llseek = no_llseek,
Steven Rostedtb3806b42008-05-12 21:20:46 +02005469};
5470
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005471static const struct file_operations tracing_entries_fops = {
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02005472 .open = tracing_open_generic_tr,
Steven Rostedta98a3c32008-05-12 21:20:59 +02005473 .read = tracing_entries_read,
5474 .write = tracing_entries_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005475 .llseek = generic_file_llseek,
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02005476 .release = tracing_release_generic_tr,
Steven Rostedta98a3c32008-05-12 21:20:59 +02005477};
5478
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07005479static const struct file_operations tracing_total_entries_fops = {
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005480 .open = tracing_open_generic_tr,
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07005481 .read = tracing_total_entries_read,
5482 .llseek = generic_file_llseek,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005483 .release = tracing_release_generic_tr,
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07005484};
5485
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005486static const struct file_operations tracing_free_buffer_fops = {
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005487 .open = tracing_open_generic_tr,
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005488 .write = tracing_free_buffer_write,
5489 .release = tracing_free_buffer_release,
5490};
5491
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005492static const struct file_operations tracing_mark_fops = {
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005493 .open = tracing_open_generic_tr,
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005494 .write = tracing_mark_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005495 .llseek = generic_file_llseek,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005496 .release = tracing_release_generic_tr,
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005497};
5498
Zhaolei5079f322009-08-25 16:12:56 +08005499static const struct file_operations trace_clock_fops = {
Li Zefan13f16d22009-12-08 11:16:11 +08005500 .open = tracing_clock_open,
5501 .read = seq_read,
5502 .llseek = seq_lseek,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005503 .release = tracing_single_release_tr,
Zhaolei5079f322009-08-25 16:12:56 +08005504 .write = tracing_clock_write,
5505};
5506
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005507#ifdef CONFIG_TRACER_SNAPSHOT
5508static const struct file_operations snapshot_fops = {
5509 .open = tracing_snapshot_open,
5510 .read = seq_read,
5511 .write = tracing_snapshot_write,
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05005512 .llseek = tracing_lseek,
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005513 .release = tracing_snapshot_release,
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005514};
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005515
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005516static const struct file_operations snapshot_raw_fops = {
5517 .open = snapshot_raw_open,
5518 .read = tracing_buffers_read,
5519 .release = tracing_buffers_release,
5520 .splice_read = tracing_buffers_splice_read,
5521 .llseek = no_llseek,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005522};
5523
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005524#endif /* CONFIG_TRACER_SNAPSHOT */
5525
Steven Rostedt2cadf912008-12-01 22:20:19 -05005526static int tracing_buffers_open(struct inode *inode, struct file *filp)
5527{
Oleg Nesterov46ef2be2013-07-23 17:26:00 +02005528 struct trace_array *tr = inode->i_private;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005529 struct ftrace_buffer_info *info;
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005530 int ret;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005531
5532 if (tracing_disabled)
5533 return -ENODEV;
5534
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005535 if (trace_array_get(tr) < 0)
5536 return -ENODEV;
5537
Steven Rostedt2cadf912008-12-01 22:20:19 -05005538 info = kzalloc(sizeof(*info), GFP_KERNEL);
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005539 if (!info) {
5540 trace_array_put(tr);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005541 return -ENOMEM;
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005542 }
Steven Rostedt2cadf912008-12-01 22:20:19 -05005543
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05005544 mutex_lock(&trace_types_lock);
5545
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005546 info->iter.tr = tr;
Oleg Nesterov46ef2be2013-07-23 17:26:00 +02005547 info->iter.cpu_file = tracing_get_cpu(inode);
Steven Rostedtb6273442013-02-28 13:44:11 -05005548 info->iter.trace = tr->current_trace;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005549 info->iter.trace_buffer = &tr->trace_buffer;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005550 info->spare = NULL;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005551 /* Force reading ring buffer for first read */
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005552 info->read = (unsigned int)-1;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005553
5554 filp->private_data = info;
5555
Steven Rostedt (Red Hat)cf6ab6d2014-12-15 20:13:31 -05005556 tr->current_trace->ref++;
5557
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05005558 mutex_unlock(&trace_types_lock);
5559
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005560 ret = nonseekable_open(inode, filp);
5561 if (ret < 0)
5562 trace_array_put(tr);
5563
5564 return ret;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005565}
5566
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005567static unsigned int
5568tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5569{
5570 struct ftrace_buffer_info *info = filp->private_data;
5571 struct trace_iterator *iter = &info->iter;
5572
5573 return trace_poll(iter, filp, poll_table);
5574}
5575
Steven Rostedt2cadf912008-12-01 22:20:19 -05005576static ssize_t
5577tracing_buffers_read(struct file *filp, char __user *ubuf,
5578 size_t count, loff_t *ppos)
5579{
5580 struct ftrace_buffer_info *info = filp->private_data;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005581 struct trace_iterator *iter = &info->iter;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005582 ssize_t ret;
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005583 ssize_t size;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005584
Steven Rostedt2dc5d122009-03-04 19:10:05 -05005585 if (!count)
5586 return 0;
5587
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005588#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005589 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5590 return -EBUSY;
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005591#endif
5592
Lai Jiangshanddd538f2009-04-02 15:16:59 +08005593 if (!info->spare)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005594 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5595 iter->cpu_file);
Lai Jiangshanddd538f2009-04-02 15:16:59 +08005596 if (!info->spare)
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005597 return -ENOMEM;
Lai Jiangshanddd538f2009-04-02 15:16:59 +08005598
Steven Rostedt2cadf912008-12-01 22:20:19 -05005599 /* Do we have previous read data to read? */
5600 if (info->read < PAGE_SIZE)
5601 goto read;
5602
Steven Rostedtb6273442013-02-28 13:44:11 -05005603 again:
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005604 trace_access_lock(iter->cpu_file);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005605 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005606 &info->spare,
5607 count,
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005608 iter->cpu_file, 0);
5609 trace_access_unlock(iter->cpu_file);
Steven Rostedtb6273442013-02-28 13:44:11 -05005610
5611 if (ret < 0) {
5612 if (trace_empty(iter)) {
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005613 if ((filp->f_flags & O_NONBLOCK))
5614 return -EAGAIN;
5615
Rabin Vincente30f53a2014-11-10 19:46:34 +01005616 ret = wait_on_pipe(iter, false);
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005617 if (ret)
5618 return ret;
5619
Steven Rostedtb6273442013-02-28 13:44:11 -05005620 goto again;
5621 }
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005622 return 0;
Steven Rostedtb6273442013-02-28 13:44:11 -05005623 }
Steven Rostedt2cadf912008-12-01 22:20:19 -05005624
Steven Rostedt436fc282011-10-14 10:44:25 -04005625 info->read = 0;
Steven Rostedtb6273442013-02-28 13:44:11 -05005626 read:
Steven Rostedt2cadf912008-12-01 22:20:19 -05005627 size = PAGE_SIZE - info->read;
5628 if (size > count)
5629 size = count;
5630
5631 ret = copy_to_user(ubuf, info->spare + info->read, size);
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005632 if (ret == size)
5633 return -EFAULT;
5634
Steven Rostedt2dc5d122009-03-04 19:10:05 -05005635 size -= ret;
5636
Steven Rostedt2cadf912008-12-01 22:20:19 -05005637 *ppos += size;
5638 info->read += size;
5639
5640 return size;
5641}
5642
5643static int tracing_buffers_release(struct inode *inode, struct file *file)
5644{
5645 struct ftrace_buffer_info *info = file->private_data;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005646 struct trace_iterator *iter = &info->iter;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005647
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05005648 mutex_lock(&trace_types_lock);
5649
Steven Rostedt (Red Hat)cf6ab6d2014-12-15 20:13:31 -05005650 iter->tr->current_trace->ref--;
5651
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04005652 __trace_array_put(iter->tr);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005653
Lai Jiangshanddd538f2009-04-02 15:16:59 +08005654 if (info->spare)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005655 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005656 kfree(info);
5657
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05005658 mutex_unlock(&trace_types_lock);
5659
Steven Rostedt2cadf912008-12-01 22:20:19 -05005660 return 0;
5661}
5662
5663struct buffer_ref {
5664 struct ring_buffer *buffer;
5665 void *page;
5666 int ref;
5667};
5668
5669static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5670 struct pipe_buffer *buf)
5671{
5672 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5673
5674 if (--ref->ref)
5675 return;
5676
5677 ring_buffer_free_read_page(ref->buffer, ref->page);
5678 kfree(ref);
5679 buf->private = 0;
5680}
5681
Steven Rostedt2cadf912008-12-01 22:20:19 -05005682static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5683 struct pipe_buffer *buf)
5684{
5685 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5686
5687 ref->ref++;
5688}
5689
5690/* Pipe buffer operations for a buffer. */
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08005691static const struct pipe_buf_operations buffer_pipe_buf_ops = {
Steven Rostedt2cadf912008-12-01 22:20:19 -05005692 .can_merge = 0,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005693 .confirm = generic_pipe_buf_confirm,
5694 .release = buffer_pipe_buf_release,
Masami Hiramatsud55cb6c2012-08-09 21:31:10 +09005695 .steal = generic_pipe_buf_steal,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005696 .get = buffer_pipe_buf_get,
5697};
5698
5699/*
5700 * Callback from splice_to_pipe(), if we need to release some pages
5701 * at the end of the spd in case we error'ed out in filling the pipe.
5702 */
5703static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5704{
5705 struct buffer_ref *ref =
5706 (struct buffer_ref *)spd->partial[i].private;
5707
5708 if (--ref->ref)
5709 return;
5710
5711 ring_buffer_free_read_page(ref->buffer, ref->page);
5712 kfree(ref);
5713 spd->partial[i].private = 0;
5714}
5715
5716static ssize_t
5717tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5718 struct pipe_inode_info *pipe, size_t len,
5719 unsigned int flags)
5720{
5721 struct ftrace_buffer_info *info = file->private_data;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005722 struct trace_iterator *iter = &info->iter;
Jens Axboe35f3d142010-05-20 10:43:18 +02005723 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5724 struct page *pages_def[PIPE_DEF_BUFFERS];
Steven Rostedt2cadf912008-12-01 22:20:19 -05005725 struct splice_pipe_desc spd = {
Jens Axboe35f3d142010-05-20 10:43:18 +02005726 .pages = pages_def,
5727 .partial = partial_def,
Eric Dumazet047fe362012-06-12 15:24:40 +02005728 .nr_pages_max = PIPE_DEF_BUFFERS,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005729 .flags = flags,
5730 .ops = &buffer_pipe_buf_ops,
5731 .spd_release = buffer_spd_release,
5732 };
5733 struct buffer_ref *ref;
Steven Rostedt93459c62009-04-29 00:23:13 -04005734 int entries, size, i;
Rabin Vincent07906da2014-11-06 22:26:07 +01005735 ssize_t ret = 0;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005736
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005737#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005738 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5739 return -EBUSY;
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005740#endif
5741
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005742 if (splice_grow_spd(pipe, &spd))
5743 return -ENOMEM;
Jens Axboe35f3d142010-05-20 10:43:18 +02005744
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005745 if (*ppos & (PAGE_SIZE - 1))
5746 return -EINVAL;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08005747
5748 if (len & (PAGE_SIZE - 1)) {
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005749 if (len < PAGE_SIZE)
5750 return -EINVAL;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08005751 len &= PAGE_MASK;
5752 }
5753
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005754 again:
5755 trace_access_lock(iter->cpu_file);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005756 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
Steven Rostedt93459c62009-04-29 00:23:13 -04005757
Al Viroa786c062014-04-11 12:01:03 -04005758 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
Steven Rostedt2cadf912008-12-01 22:20:19 -05005759 struct page *page;
5760 int r;
5761
5762 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
Rabin Vincent07906da2014-11-06 22:26:07 +01005763 if (!ref) {
5764 ret = -ENOMEM;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005765 break;
Rabin Vincent07906da2014-11-06 22:26:07 +01005766 }
Steven Rostedt2cadf912008-12-01 22:20:19 -05005767
Steven Rostedt7267fa62009-04-29 00:16:21 -04005768 ref->ref = 1;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005769 ref->buffer = iter->trace_buffer->buffer;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005770 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005771 if (!ref->page) {
Rabin Vincent07906da2014-11-06 22:26:07 +01005772 ret = -ENOMEM;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005773 kfree(ref);
5774 break;
5775 }
5776
5777 r = ring_buffer_read_page(ref->buffer, &ref->page,
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005778 len, iter->cpu_file, 1);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005779 if (r < 0) {
Vaibhav Nagarnaik7ea59062011-05-03 17:56:42 -07005780 ring_buffer_free_read_page(ref->buffer, ref->page);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005781 kfree(ref);
5782 break;
5783 }
5784
5785 /*
5786 * zero out any left over data, this is going to
5787 * user land.
5788 */
5789 size = ring_buffer_page_len(ref->page);
5790 if (size < PAGE_SIZE)
5791 memset(ref->page + size, 0, PAGE_SIZE - size);
5792
5793 page = virt_to_page(ref->page);
5794
5795 spd.pages[i] = page;
5796 spd.partial[i].len = PAGE_SIZE;
5797 spd.partial[i].offset = 0;
5798 spd.partial[i].private = (unsigned long)ref;
5799 spd.nr_pages++;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08005800 *ppos += PAGE_SIZE;
Steven Rostedt93459c62009-04-29 00:23:13 -04005801
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005802 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005803 }
5804
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005805 trace_access_unlock(iter->cpu_file);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005806 spd.nr_pages = i;
5807
5808 /* did we read anything? */
5809 if (!spd.nr_pages) {
Rabin Vincent07906da2014-11-06 22:26:07 +01005810 if (ret)
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005811 return ret;
Rabin Vincent07906da2014-11-06 22:26:07 +01005812
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005813 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
5814 return -EAGAIN;
5815
Rabin Vincente30f53a2014-11-10 19:46:34 +01005816 ret = wait_on_pipe(iter, true);
Steven Rostedt (Red Hat)8b8b3682014-06-10 09:46:00 -04005817 if (ret)
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005818 return ret;
Rabin Vincente30f53a2014-11-10 19:46:34 +01005819
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005820 goto again;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005821 }
5822
5823 ret = splice_to_pipe(pipe, &spd);
Eric Dumazet047fe362012-06-12 15:24:40 +02005824 splice_shrink_spd(&spd);
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005825
Steven Rostedt2cadf912008-12-01 22:20:19 -05005826 return ret;
5827}
5828
5829static const struct file_operations tracing_buffers_fops = {
5830 .open = tracing_buffers_open,
5831 .read = tracing_buffers_read,
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005832 .poll = tracing_buffers_poll,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005833 .release = tracing_buffers_release,
5834 .splice_read = tracing_buffers_splice_read,
5835 .llseek = no_llseek,
5836};
5837
Steven Rostedtc8d77182009-04-29 18:03:45 -04005838static ssize_t
5839tracing_stats_read(struct file *filp, char __user *ubuf,
5840 size_t count, loff_t *ppos)
5841{
Oleg Nesterov4d3435b2013-07-23 17:26:03 +02005842 struct inode *inode = file_inode(filp);
5843 struct trace_array *tr = inode->i_private;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005844 struct trace_buffer *trace_buf = &tr->trace_buffer;
Oleg Nesterov4d3435b2013-07-23 17:26:03 +02005845 int cpu = tracing_get_cpu(inode);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005846 struct trace_seq *s;
5847 unsigned long cnt;
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005848 unsigned long long t;
5849 unsigned long usec_rem;
Steven Rostedtc8d77182009-04-29 18:03:45 -04005850
Li Zefane4f2d102009-06-15 10:57:28 +08005851 s = kmalloc(sizeof(*s), GFP_KERNEL);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005852 if (!s)
Roel Kluina6463652009-11-11 22:26:35 +01005853 return -ENOMEM;
Steven Rostedtc8d77182009-04-29 18:03:45 -04005854
5855 trace_seq_init(s);
5856
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005857 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005858 trace_seq_printf(s, "entries: %ld\n", cnt);
5859
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005860 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005861 trace_seq_printf(s, "overrun: %ld\n", cnt);
5862
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005863 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005864 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5865
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005866 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005867 trace_seq_printf(s, "bytes: %ld\n", cnt);
5868
Yoshihiro YUNOMAE58e8eed2013-04-23 10:32:39 +09005869 if (trace_clocks[tr->clock_id].in_ns) {
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005870 /* local or global for trace_clock */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005871 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005872 usec_rem = do_div(t, USEC_PER_SEC);
5873 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5874 t, usec_rem);
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005875
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005876 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005877 usec_rem = do_div(t, USEC_PER_SEC);
5878 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5879 } else {
5880 /* counter or tsc mode for trace_clock */
5881 trace_seq_printf(s, "oldest event ts: %llu\n",
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005882 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005883
5884 trace_seq_printf(s, "now ts: %llu\n",
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005885 ring_buffer_time_stamp(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005886 }
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005887
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005888 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
Slava Pestov884bfe82011-07-15 14:23:58 -07005889 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5890
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005891 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
Steven Rostedt (Red Hat)ad964702013-01-29 17:45:49 -05005892 trace_seq_printf(s, "read events: %ld\n", cnt);
5893
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05005894 count = simple_read_from_buffer(ubuf, count, ppos,
5895 s->buffer, trace_seq_used(s));
Steven Rostedtc8d77182009-04-29 18:03:45 -04005896
5897 kfree(s);
5898
5899 return count;
5900}
5901
5902static const struct file_operations tracing_stats_fops = {
Oleg Nesterov4d3435b2013-07-23 17:26:03 +02005903 .open = tracing_open_generic_tr,
Steven Rostedtc8d77182009-04-29 18:03:45 -04005904 .read = tracing_stats_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005905 .llseek = generic_file_llseek,
Oleg Nesterov4d3435b2013-07-23 17:26:03 +02005906 .release = tracing_release_generic_tr,
Steven Rostedtc8d77182009-04-29 18:03:45 -04005907};
5908
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005909#ifdef CONFIG_DYNAMIC_FTRACE
5910
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005911int __weak ftrace_arch_read_dyn_info(char *buf, int size)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005912{
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005913 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005914}
5915
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005916static ssize_t
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005917tracing_read_dyn_info(struct file *filp, char __user *ubuf,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005918 size_t cnt, loff_t *ppos)
5919{
Steven Rostedta26a2a22008-10-31 00:03:22 -04005920 static char ftrace_dyn_info_buffer[1024];
5921 static DEFINE_MUTEX(dyn_info_mutex);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005922 unsigned long *p = filp->private_data;
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005923 char *buf = ftrace_dyn_info_buffer;
Steven Rostedta26a2a22008-10-31 00:03:22 -04005924 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005925 int r;
5926
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005927 mutex_lock(&dyn_info_mutex);
5928 r = sprintf(buf, "%ld ", *p);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005929
Steven Rostedta26a2a22008-10-31 00:03:22 -04005930 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005931 buf[r++] = '\n';
5932
5933 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5934
5935 mutex_unlock(&dyn_info_mutex);
5936
5937 return r;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005938}
5939
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005940static const struct file_operations tracing_dyn_info_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005941 .open = tracing_open_generic,
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005942 .read = tracing_read_dyn_info,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005943 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005944};
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005945#endif /* CONFIG_DYNAMIC_FTRACE */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005946
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005947#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5948static void
5949ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005950{
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005951 tracing_snapshot();
5952}
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005953
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005954static void
5955ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5956{
5957 unsigned long *count = (long *)data;
5958
5959 if (!*count)
5960 return;
5961
5962 if (*count != -1)
5963 (*count)--;
5964
5965 tracing_snapshot();
5966}
5967
5968static int
5969ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5970 struct ftrace_probe_ops *ops, void *data)
5971{
5972 long count = (long)data;
5973
5974 seq_printf(m, "%ps:", (void *)ip);
5975
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01005976 seq_puts(m, "snapshot");
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005977
5978 if (count == -1)
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01005979 seq_puts(m, ":unlimited\n");
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005980 else
5981 seq_printf(m, ":count=%ld\n", count);
5982
5983 return 0;
5984}
5985
5986static struct ftrace_probe_ops snapshot_probe_ops = {
5987 .func = ftrace_snapshot,
5988 .print = ftrace_snapshot_print,
5989};
5990
5991static struct ftrace_probe_ops snapshot_count_probe_ops = {
5992 .func = ftrace_count_snapshot,
5993 .print = ftrace_snapshot_print,
5994};
5995
5996static int
5997ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5998 char *glob, char *cmd, char *param, int enable)
5999{
6000 struct ftrace_probe_ops *ops;
6001 void *count = (void *)-1;
6002 char *number;
6003 int ret;
6004
6005 /* hash funcs only work with set_ftrace_filter */
6006 if (!enable)
6007 return -EINVAL;
6008
6009 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
6010
6011 if (glob[0] == '!') {
6012 unregister_ftrace_function_probe_func(glob+1, ops);
6013 return 0;
6014 }
6015
6016 if (!param)
6017 goto out_reg;
6018
6019 number = strsep(&param, ":");
6020
6021 if (!strlen(number))
6022 goto out_reg;
6023
6024 /*
6025 * We use the callback data field (which is a pointer)
6026 * as our counter.
6027 */
6028 ret = kstrtoul(number, 0, (unsigned long *)&count);
6029 if (ret)
6030 return ret;
6031
6032 out_reg:
6033 ret = register_ftrace_function_probe(glob, ops, count);
6034
6035 if (ret >= 0)
6036 alloc_snapshot(&global_trace);
6037
6038 return ret < 0 ? ret : 0;
6039}
6040
6041static struct ftrace_func_command ftrace_snapshot_cmd = {
6042 .name = "snapshot",
6043 .func = ftrace_trace_snapshot_callback,
6044};
6045
Tom Zanussi38de93a2013-10-24 08:34:18 -05006046static __init int register_snapshot_cmd(void)
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04006047{
6048 return register_ftrace_command(&ftrace_snapshot_cmd);
6049}
6050#else
Tom Zanussi38de93a2013-10-24 08:34:18 -05006051static inline __init int register_snapshot_cmd(void) { return 0; }
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04006052#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006053
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006054static struct dentry *tracing_get_dentry(struct trace_array *tr)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006055{
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006056 if (WARN_ON(!tr->dir))
6057 return ERR_PTR(-ENODEV);
6058
6059 /* Top directory uses NULL as the parent */
6060 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
6061 return NULL;
6062
6063 /* All sub buffers have a descriptor */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006064 return tr->dir;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006065}
6066
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006067static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
6068{
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006069 struct dentry *d_tracer;
6070
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006071 if (tr->percpu_dir)
6072 return tr->percpu_dir;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006073
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006074 d_tracer = tracing_get_dentry(tr);
Steven Rostedt (Red Hat)14a5ae42015-01-20 11:14:16 -05006075 if (IS_ERR(d_tracer))
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006076 return NULL;
6077
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006078 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006079
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006080 WARN_ONCE(!tr->percpu_dir,
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006081 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006082
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006083 return tr->percpu_dir;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006084}
6085
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006086static struct dentry *
6087trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
6088 void *data, long cpu, const struct file_operations *fops)
6089{
6090 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
6091
6092 if (ret) /* See tracing_get_cpu() */
David Howells7682c912015-03-17 22:26:16 +00006093 d_inode(ret)->i_cdev = (void *)(cpu + 1);
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006094 return ret;
6095}
6096
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006097static void
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006098tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006099{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006100 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006101 struct dentry *d_cpu;
Steven Rostedtdd49a382010-10-20 21:51:26 -04006102 char cpu_dir[30]; /* 30 characters should be more than enough */
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006103
Namhyung Kim0a3d7ce2012-04-23 10:11:57 +09006104 if (!d_percpu)
6105 return;
6106
Steven Rostedtdd49a382010-10-20 21:51:26 -04006107 snprintf(cpu_dir, 30, "cpu%ld", cpu);
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006108 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01006109 if (!d_cpu) {
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006110 pr_warning("Could not create tracefs '%s' entry\n", cpu_dir);
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01006111 return;
6112 }
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006113
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01006114 /* per cpu trace_pipe */
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006115 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
Oleg Nesterov15544202013-07-23 17:25:57 +02006116 tr, cpu, &tracing_pipe_fops);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006117
6118 /* per cpu trace */
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006119 trace_create_cpu_file("trace", 0644, d_cpu,
Oleg Nesterov6484c712013-07-23 17:26:10 +02006120 tr, cpu, &tracing_fops);
Steven Rostedt7f96f932009-03-13 00:37:42 -04006121
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006122 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
Oleg Nesterov46ef2be2013-07-23 17:26:00 +02006123 tr, cpu, &tracing_buffers_fops);
Steven Rostedtc8d77182009-04-29 18:03:45 -04006124
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006125 trace_create_cpu_file("stats", 0444, d_cpu,
Oleg Nesterov4d3435b2013-07-23 17:26:03 +02006126 tr, cpu, &tracing_stats_fops);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08006127
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006128 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02006129 tr, cpu, &tracing_entries_fops);
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05006130
6131#ifdef CONFIG_TRACER_SNAPSHOT
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006132 trace_create_cpu_file("snapshot", 0644, d_cpu,
Oleg Nesterov6484c712013-07-23 17:26:10 +02006133 tr, cpu, &snapshot_fops);
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05006134
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006135 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
Oleg Nesterov46ef2be2013-07-23 17:26:00 +02006136 tr, cpu, &snapshot_raw_fops);
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05006137#endif
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006138}
6139
Steven Rostedt60a11772008-05-12 21:20:44 +02006140#ifdef CONFIG_FTRACE_SELFTEST
6141/* Let selftest have access to static functions in this file */
6142#include "trace_selftest.c"
6143#endif
6144
Steven Rostedt577b7852009-02-26 23:43:05 -05006145static ssize_t
6146trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
6147 loff_t *ppos)
6148{
6149 struct trace_option_dentry *topt = filp->private_data;
6150 char *buf;
6151
6152 if (topt->flags->val & topt->opt->bit)
6153 buf = "1\n";
6154 else
6155 buf = "0\n";
6156
6157 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6158}
6159
6160static ssize_t
6161trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
6162 loff_t *ppos)
6163{
6164 struct trace_option_dentry *topt = filp->private_data;
6165 unsigned long val;
Steven Rostedt577b7852009-02-26 23:43:05 -05006166 int ret;
6167
Peter Huewe22fe9b52011-06-07 21:58:27 +02006168 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6169 if (ret)
Steven Rostedt577b7852009-02-26 23:43:05 -05006170 return ret;
6171
Li Zefan8d18eaa2009-12-08 11:17:06 +08006172 if (val != 0 && val != 1)
Steven Rostedt577b7852009-02-26 23:43:05 -05006173 return -EINVAL;
Li Zefan8d18eaa2009-12-08 11:17:06 +08006174
6175 if (!!(topt->flags->val & topt->opt->bit) != val) {
6176 mutex_lock(&trace_types_lock);
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05006177 ret = __set_tracer_option(topt->tr, topt->flags,
Steven Rostedtc757bea2009-12-21 22:35:16 -05006178 topt->opt, !val);
Li Zefan8d18eaa2009-12-08 11:17:06 +08006179 mutex_unlock(&trace_types_lock);
6180 if (ret)
6181 return ret;
Steven Rostedt577b7852009-02-26 23:43:05 -05006182 }
6183
6184 *ppos += cnt;
6185
6186 return cnt;
6187}
6188
6189
6190static const struct file_operations trace_options_fops = {
6191 .open = tracing_open_generic,
6192 .read = trace_options_read,
6193 .write = trace_options_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02006194 .llseek = generic_file_llseek,
Steven Rostedt577b7852009-02-26 23:43:05 -05006195};
6196
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006197/*
6198 * In order to pass in both the trace_array descriptor as well as the index
6199 * to the flag that the trace option file represents, the trace_array
6200 * has a character array of trace_flags_index[], which holds the index
6201 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
6202 * The address of this character array is passed to the flag option file
6203 * read/write callbacks.
6204 *
6205 * In order to extract both the index and the trace_array descriptor,
6206 * get_tr_index() uses the following algorithm.
6207 *
6208 * idx = *ptr;
6209 *
6210 * As the pointer itself contains the address of the index (remember
6211 * index[1] == 1).
6212 *
6213 * Then to get the trace_array descriptor, by subtracting that index
6214 * from the ptr, we get to the start of the index itself.
6215 *
6216 * ptr - idx == &index[0]
6217 *
6218 * Then a simple container_of() from that pointer gets us to the
6219 * trace_array descriptor.
6220 */
6221static void get_tr_index(void *data, struct trace_array **ptr,
6222 unsigned int *pindex)
6223{
6224 *pindex = *(unsigned char *)data;
6225
6226 *ptr = container_of(data - *pindex, struct trace_array,
6227 trace_flags_index);
6228}
6229
Steven Rostedta8259072009-02-26 22:19:12 -05006230static ssize_t
6231trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
6232 loff_t *ppos)
6233{
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006234 void *tr_index = filp->private_data;
6235 struct trace_array *tr;
6236 unsigned int index;
Steven Rostedta8259072009-02-26 22:19:12 -05006237 char *buf;
6238
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006239 get_tr_index(tr_index, &tr, &index);
6240
6241 if (tr->trace_flags & (1 << index))
Steven Rostedta8259072009-02-26 22:19:12 -05006242 buf = "1\n";
6243 else
6244 buf = "0\n";
6245
6246 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6247}
6248
6249static ssize_t
6250trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
6251 loff_t *ppos)
6252{
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006253 void *tr_index = filp->private_data;
6254 struct trace_array *tr;
6255 unsigned int index;
Steven Rostedta8259072009-02-26 22:19:12 -05006256 unsigned long val;
6257 int ret;
6258
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006259 get_tr_index(tr_index, &tr, &index);
6260
Peter Huewe22fe9b52011-06-07 21:58:27 +02006261 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6262 if (ret)
Steven Rostedta8259072009-02-26 22:19:12 -05006263 return ret;
6264
Zhaoleif2d84b62009-08-07 18:55:48 +08006265 if (val != 0 && val != 1)
Steven Rostedta8259072009-02-26 22:19:12 -05006266 return -EINVAL;
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04006267
6268 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006269 ret = set_tracer_flag(tr, 1 << index, val);
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04006270 mutex_unlock(&trace_types_lock);
Steven Rostedta8259072009-02-26 22:19:12 -05006271
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04006272 if (ret < 0)
6273 return ret;
6274
Steven Rostedta8259072009-02-26 22:19:12 -05006275 *ppos += cnt;
6276
6277 return cnt;
6278}
6279
Steven Rostedta8259072009-02-26 22:19:12 -05006280static const struct file_operations trace_options_core_fops = {
6281 .open = tracing_open_generic,
6282 .read = trace_options_core_read,
6283 .write = trace_options_core_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02006284 .llseek = generic_file_llseek,
Steven Rostedta8259072009-02-26 22:19:12 -05006285};
6286
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006287struct dentry *trace_create_file(const char *name,
Al Virof4ae40a2011-07-24 04:33:43 -04006288 umode_t mode,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006289 struct dentry *parent,
6290 void *data,
6291 const struct file_operations *fops)
6292{
6293 struct dentry *ret;
6294
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006295 ret = tracefs_create_file(name, mode, parent, data, fops);
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006296 if (!ret)
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006297 pr_warning("Could not create tracefs '%s' entry\n", name);
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006298
6299 return ret;
6300}
6301
6302
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006303static struct dentry *trace_options_init_dentry(struct trace_array *tr)
Steven Rostedta8259072009-02-26 22:19:12 -05006304{
6305 struct dentry *d_tracer;
Steven Rostedta8259072009-02-26 22:19:12 -05006306
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006307 if (tr->options)
6308 return tr->options;
Steven Rostedta8259072009-02-26 22:19:12 -05006309
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006310 d_tracer = tracing_get_dentry(tr);
Steven Rostedt (Red Hat)14a5ae42015-01-20 11:14:16 -05006311 if (IS_ERR(d_tracer))
Steven Rostedta8259072009-02-26 22:19:12 -05006312 return NULL;
6313
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006314 tr->options = tracefs_create_dir("options", d_tracer);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006315 if (!tr->options) {
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006316 pr_warning("Could not create tracefs directory 'options'\n");
Steven Rostedta8259072009-02-26 22:19:12 -05006317 return NULL;
6318 }
6319
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006320 return tr->options;
Steven Rostedta8259072009-02-26 22:19:12 -05006321}
6322
Steven Rostedt577b7852009-02-26 23:43:05 -05006323static void
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006324create_trace_option_file(struct trace_array *tr,
6325 struct trace_option_dentry *topt,
Steven Rostedt577b7852009-02-26 23:43:05 -05006326 struct tracer_flags *flags,
6327 struct tracer_opt *opt)
6328{
6329 struct dentry *t_options;
Steven Rostedt577b7852009-02-26 23:43:05 -05006330
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006331 t_options = trace_options_init_dentry(tr);
Steven Rostedt577b7852009-02-26 23:43:05 -05006332 if (!t_options)
6333 return;
6334
6335 topt->flags = flags;
6336 topt->opt = opt;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006337 topt->tr = tr;
Steven Rostedt577b7852009-02-26 23:43:05 -05006338
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006339 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
Steven Rostedt577b7852009-02-26 23:43:05 -05006340 &trace_options_fops);
6341
Steven Rostedt577b7852009-02-26 23:43:05 -05006342}
6343
6344static struct trace_option_dentry *
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006345create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
Steven Rostedt577b7852009-02-26 23:43:05 -05006346{
6347 struct trace_option_dentry *topts;
6348 struct tracer_flags *flags;
6349 struct tracer_opt *opts;
6350 int cnt;
6351
6352 if (!tracer)
6353 return NULL;
6354
6355 flags = tracer->flags;
6356
6357 if (!flags || !flags->opts)
6358 return NULL;
6359
6360 opts = flags->opts;
6361
6362 for (cnt = 0; opts[cnt].name; cnt++)
6363 ;
6364
Steven Rostedt0cfe8242009-02-27 10:51:10 -05006365 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
Steven Rostedt577b7852009-02-26 23:43:05 -05006366 if (!topts)
6367 return NULL;
6368
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04006369 for (cnt = 0; opts[cnt].name; cnt++) {
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006370 create_trace_option_file(tr, &topts[cnt], flags,
Steven Rostedt577b7852009-02-26 23:43:05 -05006371 &opts[cnt]);
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04006372 WARN_ONCE(topts[cnt].entry == NULL,
6373 "Failed to create trace option: %s",
6374 opts[cnt].name);
6375 }
Steven Rostedt577b7852009-02-26 23:43:05 -05006376
6377 return topts;
6378}
6379
Steven Rostedta8259072009-02-26 22:19:12 -05006380static struct dentry *
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006381create_trace_option_core_file(struct trace_array *tr,
6382 const char *option, long index)
Steven Rostedta8259072009-02-26 22:19:12 -05006383{
6384 struct dentry *t_options;
Steven Rostedta8259072009-02-26 22:19:12 -05006385
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006386 t_options = trace_options_init_dentry(tr);
Steven Rostedta8259072009-02-26 22:19:12 -05006387 if (!t_options)
6388 return NULL;
6389
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006390 return trace_create_file(option, 0644, t_options,
6391 (void *)&tr->trace_flags_index[index],
6392 &trace_options_core_fops);
Steven Rostedta8259072009-02-26 22:19:12 -05006393}
6394
Steven Rostedt (Red Hat)16270142015-09-30 12:30:06 -04006395static void create_trace_options_dir(struct trace_array *tr)
Steven Rostedta8259072009-02-26 22:19:12 -05006396{
6397 struct dentry *t_options;
Steven Rostedt (Red Hat)16270142015-09-30 12:30:06 -04006398 bool top_level = tr == &global_trace;
Steven Rostedta8259072009-02-26 22:19:12 -05006399 int i;
6400
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006401 t_options = trace_options_init_dentry(tr);
Steven Rostedta8259072009-02-26 22:19:12 -05006402 if (!t_options)
6403 return;
6404
Steven Rostedt (Red Hat)16270142015-09-30 12:30:06 -04006405 for (i = 0; trace_options[i]; i++) {
6406 if (top_level ||
6407 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
6408 create_trace_option_core_file(tr, trace_options[i], i);
6409 }
Steven Rostedta8259072009-02-26 22:19:12 -05006410}
6411
Steven Rostedt499e5472012-02-22 15:50:28 -05006412static ssize_t
6413rb_simple_read(struct file *filp, char __user *ubuf,
6414 size_t cnt, loff_t *ppos)
6415{
Steven Rostedt348f0fc2012-04-16 15:41:28 -04006416 struct trace_array *tr = filp->private_data;
Steven Rostedt499e5472012-02-22 15:50:28 -05006417 char buf[64];
6418 int r;
6419
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -04006420 r = tracer_tracing_is_on(tr);
Steven Rostedt499e5472012-02-22 15:50:28 -05006421 r = sprintf(buf, "%d\n", r);
6422
6423 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6424}
6425
6426static ssize_t
6427rb_simple_write(struct file *filp, const char __user *ubuf,
6428 size_t cnt, loff_t *ppos)
6429{
Steven Rostedt348f0fc2012-04-16 15:41:28 -04006430 struct trace_array *tr = filp->private_data;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05006431 struct ring_buffer *buffer = tr->trace_buffer.buffer;
Steven Rostedt499e5472012-02-22 15:50:28 -05006432 unsigned long val;
6433 int ret;
6434
6435 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6436 if (ret)
6437 return ret;
6438
6439 if (buffer) {
Steven Rostedt2df8f8a2013-01-11 16:14:10 -05006440 mutex_lock(&trace_types_lock);
6441 if (val) {
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -04006442 tracer_tracing_on(tr);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006443 if (tr->current_trace->start)
6444 tr->current_trace->start(tr);
Steven Rostedt2df8f8a2013-01-11 16:14:10 -05006445 } else {
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -04006446 tracer_tracing_off(tr);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006447 if (tr->current_trace->stop)
6448 tr->current_trace->stop(tr);
Steven Rostedt2df8f8a2013-01-11 16:14:10 -05006449 }
6450 mutex_unlock(&trace_types_lock);
Steven Rostedt499e5472012-02-22 15:50:28 -05006451 }
6452
6453 (*ppos)++;
6454
6455 return cnt;
6456}
6457
6458static const struct file_operations rb_simple_fops = {
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04006459 .open = tracing_open_generic_tr,
Steven Rostedt499e5472012-02-22 15:50:28 -05006460 .read = rb_simple_read,
6461 .write = rb_simple_write,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04006462 .release = tracing_release_generic_tr,
Steven Rostedt499e5472012-02-22 15:50:28 -05006463 .llseek = default_llseek,
6464};
6465
Steven Rostedt277ba042012-08-03 16:10:49 -04006466struct dentry *trace_instance_dir;
6467
6468static void
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006469init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
Steven Rostedt277ba042012-08-03 16:10:49 -04006470
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05006471static int
6472allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
Steven Rostedt277ba042012-08-03 16:10:49 -04006473{
6474 enum ring_buffer_flags rb_flags;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006475
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04006476 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006477
Steven Rostedt (Red Hat)dced3412014-01-14 10:19:46 -05006478 buf->tr = tr;
6479
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05006480 buf->buffer = ring_buffer_alloc(size, rb_flags);
6481 if (!buf->buffer)
6482 return -ENOMEM;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006483
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05006484 buf->data = alloc_percpu(struct trace_array_cpu);
6485 if (!buf->data) {
6486 ring_buffer_free(buf->buffer);
6487 return -ENOMEM;
6488 }
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006489
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006490 /* Allocate the first page for all buffers */
6491 set_buffer_entries(&tr->trace_buffer,
6492 ring_buffer_size(tr->trace_buffer.buffer, 0));
6493
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05006494 return 0;
6495}
6496
6497static int allocate_trace_buffers(struct trace_array *tr, int size)
6498{
6499 int ret;
6500
6501 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6502 if (ret)
6503 return ret;
6504
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006505#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05006506 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6507 allocate_snapshot ? size : 1);
6508 if (WARN_ON(ret)) {
6509 ring_buffer_free(tr->trace_buffer.buffer);
6510 free_percpu(tr->trace_buffer.data);
6511 return -ENOMEM;
6512 }
6513 tr->allocated_snapshot = allocate_snapshot;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006514
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05006515 /*
6516 * Only the top level trace array gets its snapshot allocated
6517 * from the kernel command line.
6518 */
6519 allocate_snapshot = false;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006520#endif
6521 return 0;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006522}
6523
Steven Rostedt (Red Hat)f0b70cc2014-06-10 12:06:30 -04006524static void free_trace_buffer(struct trace_buffer *buf)
6525{
6526 if (buf->buffer) {
6527 ring_buffer_free(buf->buffer);
6528 buf->buffer = NULL;
6529 free_percpu(buf->data);
6530 buf->data = NULL;
6531 }
6532}
6533
Steven Rostedt (Red Hat)23aaa3c2014-06-06 00:01:46 -04006534static void free_trace_buffers(struct trace_array *tr)
6535{
6536 if (!tr)
6537 return;
6538
Steven Rostedt (Red Hat)f0b70cc2014-06-10 12:06:30 -04006539 free_trace_buffer(&tr->trace_buffer);
Steven Rostedt (Red Hat)23aaa3c2014-06-06 00:01:46 -04006540
6541#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)f0b70cc2014-06-10 12:06:30 -04006542 free_trace_buffer(&tr->max_buffer);
Steven Rostedt (Red Hat)23aaa3c2014-06-06 00:01:46 -04006543#endif
6544}
6545
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006546static void init_trace_flags_index(struct trace_array *tr)
6547{
6548 int i;
6549
6550 /* Used by the trace options files */
6551 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
6552 tr->trace_flags_index[i] = i;
6553}
6554
Steven Rostedt (Red Hat)eae47352015-01-21 10:01:39 -05006555static int instance_mkdir(const char *name)
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006556{
Steven Rostedt277ba042012-08-03 16:10:49 -04006557 struct trace_array *tr;
6558 int ret;
Steven Rostedt277ba042012-08-03 16:10:49 -04006559
6560 mutex_lock(&trace_types_lock);
6561
6562 ret = -EEXIST;
6563 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6564 if (tr->name && strcmp(tr->name, name) == 0)
6565 goto out_unlock;
6566 }
6567
6568 ret = -ENOMEM;
6569 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6570 if (!tr)
6571 goto out_unlock;
6572
6573 tr->name = kstrdup(name, GFP_KERNEL);
6574 if (!tr->name)
6575 goto out_free_tr;
6576
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07006577 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6578 goto out_free_tr;
6579
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04006580 tr->trace_flags = global_trace.trace_flags;
6581
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07006582 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6583
Steven Rostedt277ba042012-08-03 16:10:49 -04006584 raw_spin_lock_init(&tr->start_lock);
6585
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05006586 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6587
Steven Rostedt277ba042012-08-03 16:10:49 -04006588 tr->current_trace = &nop_trace;
6589
6590 INIT_LIST_HEAD(&tr->systems);
6591 INIT_LIST_HEAD(&tr->events);
6592
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006593 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
Steven Rostedt277ba042012-08-03 16:10:49 -04006594 goto out_free_tr;
6595
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006596 tr->dir = tracefs_create_dir(name, trace_instance_dir);
Steven Rostedt277ba042012-08-03 16:10:49 -04006597 if (!tr->dir)
6598 goto out_free_tr;
6599
6600 ret = event_trace_add_tracer(tr->dir, tr);
Alexander Z Lam609e85a2013-07-10 17:34:34 -07006601 if (ret) {
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006602 tracefs_remove_recursive(tr->dir);
Steven Rostedt277ba042012-08-03 16:10:49 -04006603 goto out_free_tr;
Alexander Z Lam609e85a2013-07-10 17:34:34 -07006604 }
Steven Rostedt277ba042012-08-03 16:10:49 -04006605
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006606 init_tracer_tracefs(tr, tr->dir);
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006607 init_trace_flags_index(tr);
Steven Rostedt277ba042012-08-03 16:10:49 -04006608
6609 list_add(&tr->list, &ftrace_trace_arrays);
6610
6611 mutex_unlock(&trace_types_lock);
6612
6613 return 0;
6614
6615 out_free_tr:
Steven Rostedt (Red Hat)23aaa3c2014-06-06 00:01:46 -04006616 free_trace_buffers(tr);
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07006617 free_cpumask_var(tr->tracing_cpumask);
Steven Rostedt277ba042012-08-03 16:10:49 -04006618 kfree(tr->name);
6619 kfree(tr);
6620
6621 out_unlock:
6622 mutex_unlock(&trace_types_lock);
6623
6624 return ret;
6625
6626}
6627
Steven Rostedt (Red Hat)eae47352015-01-21 10:01:39 -05006628static int instance_rmdir(const char *name)
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006629{
6630 struct trace_array *tr;
6631 int found = 0;
6632 int ret;
6633
6634 mutex_lock(&trace_types_lock);
6635
6636 ret = -ENODEV;
6637 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6638 if (tr->name && strcmp(tr->name, name) == 0) {
6639 found = 1;
6640 break;
6641 }
6642 }
6643 if (!found)
6644 goto out_unlock;
6645
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05006646 ret = -EBUSY;
Steven Rostedt (Red Hat)cf6ab6d2014-12-15 20:13:31 -05006647 if (tr->ref || (tr->current_trace && tr->current_trace->ref))
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05006648 goto out_unlock;
6649
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006650 list_del(&tr->list);
6651
Steven Rostedt (Red Hat)6b450d22014-01-14 08:43:01 -05006652 tracing_set_nop(tr);
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006653 event_trace_del_tracer(tr);
Steven Rostedt (Red Hat)591dffd2014-01-10 16:17:45 -05006654 ftrace_destroy_function_files(tr);
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006655 debugfs_remove_recursive(tr->dir);
Steven Rostedt (Red Hat)a9fcaaa2014-06-06 23:17:28 -04006656 free_trace_buffers(tr);
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006657
6658 kfree(tr->name);
6659 kfree(tr);
6660
6661 ret = 0;
6662
6663 out_unlock:
6664 mutex_unlock(&trace_types_lock);
6665
6666 return ret;
6667}
6668
Steven Rostedt277ba042012-08-03 16:10:49 -04006669static __init void create_trace_instances(struct dentry *d_tracer)
6670{
Steven Rostedt (Red Hat)eae47352015-01-21 10:01:39 -05006671 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
6672 instance_mkdir,
6673 instance_rmdir);
Steven Rostedt277ba042012-08-03 16:10:49 -04006674 if (WARN_ON(!trace_instance_dir))
6675 return;
Steven Rostedt277ba042012-08-03 16:10:49 -04006676}
6677
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006678static void
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006679init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006680{
Steven Rostedt (Red Hat)121aaee2013-03-05 21:52:25 -05006681 int cpu;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006682
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05006683 trace_create_file("available_tracers", 0444, d_tracer,
6684 tr, &show_traces_fops);
6685
6686 trace_create_file("current_tracer", 0644, d_tracer,
6687 tr, &set_tracer_fops);
6688
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07006689 trace_create_file("tracing_cpumask", 0644, d_tracer,
6690 tr, &tracing_cpumask_fops);
6691
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006692 trace_create_file("trace_options", 0644, d_tracer,
6693 tr, &tracing_iter_fops);
6694
6695 trace_create_file("trace", 0644, d_tracer,
Oleg Nesterov6484c712013-07-23 17:26:10 +02006696 tr, &tracing_fops);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006697
6698 trace_create_file("trace_pipe", 0444, d_tracer,
Oleg Nesterov15544202013-07-23 17:25:57 +02006699 tr, &tracing_pipe_fops);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006700
6701 trace_create_file("buffer_size_kb", 0644, d_tracer,
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02006702 tr, &tracing_entries_fops);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006703
6704 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6705 tr, &tracing_total_entries_fops);
6706
Wang YanQing238ae932013-05-26 16:52:01 +08006707 trace_create_file("free_buffer", 0200, d_tracer,
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006708 tr, &tracing_free_buffer_fops);
6709
6710 trace_create_file("trace_marker", 0220, d_tracer,
6711 tr, &tracing_mark_fops);
6712
6713 trace_create_file("trace_clock", 0644, d_tracer, tr,
6714 &trace_clock_fops);
6715
6716 trace_create_file("tracing_on", 0644, d_tracer,
Oleg Nesterov6484c712013-07-23 17:26:10 +02006717 tr, &rb_simple_fops);
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05006718
Steven Rostedt (Red Hat)16270142015-09-30 12:30:06 -04006719 create_trace_options_dir(tr);
6720
Steven Rostedt (Red Hat)6d9b3fa2014-01-14 11:28:38 -05006721#ifdef CONFIG_TRACER_MAX_TRACE
6722 trace_create_file("tracing_max_latency", 0644, d_tracer,
6723 &tr->max_latency, &tracing_max_lat_fops);
6724#endif
6725
Steven Rostedt (Red Hat)591dffd2014-01-10 16:17:45 -05006726 if (ftrace_create_function_files(tr, d_tracer))
6727 WARN(1, "Could not allocate function filter files");
6728
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05006729#ifdef CONFIG_TRACER_SNAPSHOT
6730 trace_create_file("snapshot", 0644, d_tracer,
Oleg Nesterov6484c712013-07-23 17:26:10 +02006731 tr, &snapshot_fops);
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05006732#endif
Steven Rostedt (Red Hat)121aaee2013-03-05 21:52:25 -05006733
6734 for_each_tracing_cpu(cpu)
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006735 tracing_init_tracefs_percpu(tr, cpu);
Steven Rostedt (Red Hat)121aaee2013-03-05 21:52:25 -05006736
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006737}
6738
Steven Rostedt (Red Hat)f76180b2015-01-20 15:48:46 -05006739static struct vfsmount *trace_automount(void *ingore)
6740{
6741 struct vfsmount *mnt;
6742 struct file_system_type *type;
6743
6744 /*
6745 * To maintain backward compatibility for tools that mount
6746 * debugfs to get to the tracing facility, tracefs is automatically
6747 * mounted to the debugfs/tracing directory.
6748 */
6749 type = get_fs_type("tracefs");
6750 if (!type)
6751 return NULL;
6752 mnt = vfs_kern_mount(type, 0, "tracefs", NULL);
6753 put_filesystem(type);
6754 if (IS_ERR(mnt))
6755 return NULL;
6756 mntget(mnt);
6757
6758 return mnt;
6759}
6760
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006761/**
6762 * tracing_init_dentry - initialize top level trace array
6763 *
6764 * This is called when creating files or directories in the tracing
6765 * directory. It is called via fs_initcall() by any of the boot up code
6766 * and expects to return the dentry of the top level tracing directory.
6767 */
6768struct dentry *tracing_init_dentry(void)
6769{
6770 struct trace_array *tr = &global_trace;
6771
Steven Rostedt (Red Hat)f76180b2015-01-20 15:48:46 -05006772 /* The top level trace array uses NULL as parent */
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006773 if (tr->dir)
Steven Rostedt (Red Hat)f76180b2015-01-20 15:48:46 -05006774 return NULL;
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006775
6776 if (WARN_ON(!debugfs_initialized()))
6777 return ERR_PTR(-ENODEV);
6778
Steven Rostedt (Red Hat)f76180b2015-01-20 15:48:46 -05006779 /*
6780 * As there may still be users that expect the tracing
6781 * files to exist in debugfs/tracing, we must automount
6782 * the tracefs file system there, so older tools still
6783 * work with the newer kerenl.
6784 */
6785 tr->dir = debugfs_create_automount("tracing", NULL,
6786 trace_automount, NULL);
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006787 if (!tr->dir) {
6788 pr_warn_once("Could not create debugfs directory 'tracing'\n");
6789 return ERR_PTR(-ENOMEM);
6790 }
6791
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006792 return NULL;
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006793}
6794
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04006795extern struct trace_enum_map *__start_ftrace_enum_maps[];
6796extern struct trace_enum_map *__stop_ftrace_enum_maps[];
6797
6798static void __init trace_enum_init(void)
6799{
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006800 int len;
6801
6802 len = __stop_ftrace_enum_maps - __start_ftrace_enum_maps;
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04006803 trace_insert_enum_map(NULL, __start_ftrace_enum_maps, len);
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04006804}
6805
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006806#ifdef CONFIG_MODULES
6807static void trace_module_add_enums(struct module *mod)
6808{
6809 if (!mod->num_trace_enums)
6810 return;
6811
6812 /*
6813 * Modules with bad taint do not have events created, do
6814 * not bother with enums either.
6815 */
6816 if (trace_module_has_bad_taint(mod))
6817 return;
6818
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04006819 trace_insert_enum_map(mod, mod->trace_enums, mod->num_trace_enums);
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006820}
6821
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04006822#ifdef CONFIG_TRACE_ENUM_MAP_FILE
6823static void trace_module_remove_enums(struct module *mod)
6824{
6825 union trace_enum_map_item *map;
6826 union trace_enum_map_item **last = &trace_enum_maps;
6827
6828 if (!mod->num_trace_enums)
6829 return;
6830
6831 mutex_lock(&trace_enum_mutex);
6832
6833 map = trace_enum_maps;
6834
6835 while (map) {
6836 if (map->head.mod == mod)
6837 break;
6838 map = trace_enum_jmp_to_tail(map);
6839 last = &map->tail.next;
6840 map = map->tail.next;
6841 }
6842 if (!map)
6843 goto out;
6844
6845 *last = trace_enum_jmp_to_tail(map)->tail.next;
6846 kfree(map);
6847 out:
6848 mutex_unlock(&trace_enum_mutex);
6849}
6850#else
6851static inline void trace_module_remove_enums(struct module *mod) { }
6852#endif /* CONFIG_TRACE_ENUM_MAP_FILE */
6853
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006854static int trace_module_notify(struct notifier_block *self,
6855 unsigned long val, void *data)
6856{
6857 struct module *mod = data;
6858
6859 switch (val) {
6860 case MODULE_STATE_COMING:
6861 trace_module_add_enums(mod);
6862 break;
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04006863 case MODULE_STATE_GOING:
6864 trace_module_remove_enums(mod);
6865 break;
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006866 }
6867
6868 return 0;
6869}
6870
6871static struct notifier_block trace_module_nb = {
6872 .notifier_call = trace_module_notify,
6873 .priority = 0,
6874};
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04006875#endif /* CONFIG_MODULES */
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006876
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006877static __init int tracer_init_tracefs(void)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006878{
6879 struct dentry *d_tracer;
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04006880 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006881
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08006882 trace_access_lock_init();
6883
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006884 d_tracer = tracing_init_dentry();
Steven Rostedt (Red Hat)14a5ae42015-01-20 11:14:16 -05006885 if (IS_ERR(d_tracer))
Namhyung Kimed6f1c92013-04-10 09:18:12 +09006886 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006887
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006888 init_tracer_tracefs(&global_trace, d_tracer);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006889
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006890 trace_create_file("tracing_thresh", 0644, d_tracer,
Stanislav Fomichev6508fa72014-07-18 15:17:27 +04006891 &global_trace, &tracing_thresh_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006892
Li Zefan339ae5d2009-04-17 10:34:30 +08006893 trace_create_file("README", 0444, d_tracer,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006894 NULL, &tracing_readme_fops);
Ingo Molnar7bd2f242008-05-12 21:20:45 +02006895
Avadh Patel69abe6a2009-04-10 16:04:48 -04006896 trace_create_file("saved_cmdlines", 0444, d_tracer,
6897 NULL, &tracing_saved_cmdlines_fops);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03006898
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09006899 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
6900 NULL, &tracing_saved_cmdlines_size_fops);
6901
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04006902 trace_enum_init();
6903
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04006904 trace_create_enum_file(d_tracer);
6905
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006906#ifdef CONFIG_MODULES
6907 register_module_notifier(&trace_module_nb);
6908#endif
6909
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006910#ifdef CONFIG_DYNAMIC_FTRACE
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006911 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6912 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006913#endif
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006914
Steven Rostedt277ba042012-08-03 16:10:49 -04006915 create_trace_instances(d_tracer);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09006916
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04006917 mutex_lock(&trace_types_lock);
6918 for (t = trace_types; t; t = t->next)
6919 add_tracer_options(&global_trace, t);
6920 mutex_unlock(&trace_types_lock);
Steven Rostedt (Red Hat)09d23a12015-02-03 12:45:53 -05006921
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01006922 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006923}
6924
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006925static int trace_panic_handler(struct notifier_block *this,
6926 unsigned long event, void *unused)
6927{
Steven Rostedt944ac422008-10-23 19:26:08 -04006928 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02006929 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006930 return NOTIFY_OK;
6931}
6932
6933static struct notifier_block trace_panic_notifier = {
6934 .notifier_call = trace_panic_handler,
6935 .next = NULL,
6936 .priority = 150 /* priority: INT_MAX >= x >= 0 */
6937};
6938
6939static int trace_die_handler(struct notifier_block *self,
6940 unsigned long val,
6941 void *data)
6942{
6943 switch (val) {
6944 case DIE_OOPS:
Steven Rostedt944ac422008-10-23 19:26:08 -04006945 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02006946 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006947 break;
6948 default:
6949 break;
6950 }
6951 return NOTIFY_OK;
6952}
6953
6954static struct notifier_block trace_die_notifier = {
6955 .notifier_call = trace_die_handler,
6956 .priority = 200
6957};
6958
6959/*
6960 * printk is set to max of 1024, we really don't need it that big.
6961 * Nothing should be printing 1000 characters anyway.
6962 */
6963#define TRACE_MAX_PRINT 1000
6964
6965/*
6966 * Define here KERN_TRACE so that we have one place to modify
6967 * it if we decide to change what log level the ftrace dump
6968 * should be at.
6969 */
Steven Rostedt428aee12009-01-14 12:24:42 -05006970#define KERN_TRACE KERN_EMERG
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006971
Jason Wessel955b61e2010-08-05 09:22:23 -05006972void
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006973trace_printk_seq(struct trace_seq *s)
6974{
6975 /* Probably should print a warning here. */
Steven Rostedt (Red Hat)3a161d92014-06-25 15:54:42 -04006976 if (s->seq.len >= TRACE_MAX_PRINT)
6977 s->seq.len = TRACE_MAX_PRINT;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006978
Steven Rostedt (Red Hat)820b75f2014-11-19 10:56:41 -05006979 /*
6980 * More paranoid code. Although the buffer size is set to
6981 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
6982 * an extra layer of protection.
6983 */
6984 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
6985 s->seq.len = s->seq.size - 1;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006986
6987 /* should be zero ended, but we are paranoid. */
Steven Rostedt (Red Hat)3a161d92014-06-25 15:54:42 -04006988 s->buffer[s->seq.len] = 0;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006989
6990 printk(KERN_TRACE "%s", s->buffer);
6991
Steven Rostedtf9520752009-03-02 14:04:40 -05006992 trace_seq_init(s);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006993}
6994
Jason Wessel955b61e2010-08-05 09:22:23 -05006995void trace_init_global_iter(struct trace_iterator *iter)
6996{
6997 iter->tr = &global_trace;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006998 iter->trace = iter->tr->current_trace;
Steven Rostedtae3b5092013-01-23 15:22:59 -05006999 iter->cpu_file = RING_BUFFER_ALL_CPUS;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05007000 iter->trace_buffer = &global_trace.trace_buffer;
Cody P Schaferb2f974d2013-10-23 11:49:57 -07007001
7002 if (iter->trace && iter->trace->open)
7003 iter->trace->open(iter);
7004
7005 /* Annotate start of buffers if we had overruns */
7006 if (ring_buffer_overruns(iter->trace_buffer->buffer))
7007 iter->iter_flags |= TRACE_FILE_ANNOTATE;
7008
7009 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
7010 if (trace_clocks[iter->tr->clock_id].in_ns)
7011 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
Jason Wessel955b61e2010-08-05 09:22:23 -05007012}
7013
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007014void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007015{
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007016 /* use static because iter can be a bit big for the stack */
7017 static struct trace_iterator iter;
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007018 static atomic_t dump_running;
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04007019 struct trace_array *tr = &global_trace;
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01007020 unsigned int old_userobj;
Steven Rostedtd7690412008-10-01 00:29:53 -04007021 unsigned long flags;
7022 int cnt = 0, cpu;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007023
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007024 /* Only allow one dump user at a time. */
7025 if (atomic_inc_return(&dump_running) != 1) {
7026 atomic_dec(&dump_running);
7027 return;
Steven Rostedte0a413f2011-09-29 21:26:16 -04007028 }
7029
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007030 /*
7031 * Always turn off tracing when we dump.
7032 * We don't need to show trace output of what happens
7033 * between multiple crashes.
7034 *
7035 * If the user does a sysrq-z, then they can re-enable
7036 * tracing with echo 1 > tracing_on.
7037 */
7038 tracing_off();
7039
7040 local_irq_save(flags);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007041
Jovi Zhang38dbe0b2013-01-25 18:03:07 +08007042 /* Simulate the iterator */
Jason Wessel955b61e2010-08-05 09:22:23 -05007043 trace_init_global_iter(&iter);
7044
Steven Rostedtd7690412008-10-01 00:29:53 -04007045 for_each_tracing_cpu(cpu) {
Umesh Tiwari5e2d5ef2015-06-22 16:55:06 +05307046 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -04007047 }
7048
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04007049 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01007050
Török Edwinb54d3de2008-11-22 13:28:48 +02007051 /* don't look at user memory in panic mode */
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04007052 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
Török Edwinb54d3de2008-11-22 13:28:48 +02007053
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02007054 switch (oops_dump_mode) {
7055 case DUMP_ALL:
Steven Rostedtae3b5092013-01-23 15:22:59 -05007056 iter.cpu_file = RING_BUFFER_ALL_CPUS;
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02007057 break;
7058 case DUMP_ORIG:
7059 iter.cpu_file = raw_smp_processor_id();
7060 break;
7061 case DUMP_NONE:
7062 goto out_enable;
7063 default:
7064 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
Steven Rostedtae3b5092013-01-23 15:22:59 -05007065 iter.cpu_file = RING_BUFFER_ALL_CPUS;
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02007066 }
7067
7068 printk(KERN_TRACE "Dumping ftrace buffer:\n");
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007069
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007070 /* Did function tracer already get disabled? */
7071 if (ftrace_is_dead()) {
7072 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
7073 printk("# MAY BE MISSING FUNCTION EVENTS\n");
7074 }
7075
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007076 /*
7077 * We need to stop all tracing on all CPUS to read the
7078 * the next buffer. This is a bit expensive, but is
7079 * not done often. We fill all what we can read,
7080 * and then release the locks again.
7081 */
7082
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007083 while (!trace_empty(&iter)) {
7084
7085 if (!cnt)
7086 printk(KERN_TRACE "---------------------------------\n");
7087
7088 cnt++;
7089
7090 /* reset all but tr, trace, and overruns */
7091 memset(&iter.seq, 0,
7092 sizeof(struct trace_iterator) -
7093 offsetof(struct trace_iterator, seq));
7094 iter.iter_flags |= TRACE_FILE_LAT_FMT;
7095 iter.pos = -1;
7096
Jason Wessel955b61e2010-08-05 09:22:23 -05007097 if (trace_find_next_entry_inc(&iter) != NULL) {
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08007098 int ret;
7099
7100 ret = print_trace_line(&iter);
7101 if (ret != TRACE_TYPE_NO_CONSUME)
7102 trace_consume(&iter);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007103 }
Steven Rostedtb892e5c2012-03-01 22:06:48 -05007104 touch_nmi_watchdog();
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007105
7106 trace_printk_seq(&iter.seq);
7107 }
7108
7109 if (!cnt)
7110 printk(KERN_TRACE " (ftrace buffer empty)\n");
7111 else
7112 printk(KERN_TRACE "---------------------------------\n");
7113
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02007114 out_enable:
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04007115 tr->trace_flags |= old_userobj;
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01007116
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007117 for_each_tracing_cpu(cpu) {
7118 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01007119 }
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007120 atomic_dec(&dump_running);
Steven Rostedtcd891ae2009-04-28 11:39:34 -04007121 local_irq_restore(flags);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007122}
Paul E. McKenneya8eecf22011-10-02 11:01:15 -07007123EXPORT_SYMBOL_GPL(ftrace_dump);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01007124
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02007125__init static int tracer_alloc_buffers(void)
7126{
Steven Rostedt73c51622009-03-11 13:42:01 -04007127 int ring_buf_size;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10307128 int ret = -ENOMEM;
7129
Steven Rostedt (Red Hat)b5e87c02015-09-29 18:13:33 -04007130 /*
7131 * Make sure we don't accidently add more trace options
7132 * than we have bits for.
7133 */
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04007134 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
Steven Rostedt (Red Hat)b5e87c02015-09-29 18:13:33 -04007135
Rusty Russell9e01c1b2009-01-01 10:12:22 +10307136 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
7137 goto out;
7138
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07007139 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
Rusty Russell9e01c1b2009-01-01 10:12:22 +10307140 goto out_free_buffer_mask;
7141
Steven Rostedt07d777f2011-09-22 14:01:55 -04007142 /* Only allocate trace_printk buffers if a trace_printk exists */
7143 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
Steven Rostedt81698832012-10-11 10:15:05 -04007144 /* Must be called before global_trace.buffer is allocated */
Steven Rostedt07d777f2011-09-22 14:01:55 -04007145 trace_printk_init_buffers();
7146
Steven Rostedt73c51622009-03-11 13:42:01 -04007147 /* To save memory, keep the ring buffer size to its minimum */
7148 if (ring_buffer_expanded)
7149 ring_buf_size = trace_buf_size;
7150 else
7151 ring_buf_size = 1;
7152
Rusty Russell9e01c1b2009-01-01 10:12:22 +10307153 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07007154 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02007155
Steven Rostedt2b6080f2012-05-11 13:29:49 -04007156 raw_spin_lock_init(&global_trace.start_lock);
7157
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04007158 /* Used for event triggers */
7159 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
7160 if (!temp_buffer)
7161 goto out_free_cpumask;
7162
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09007163 if (trace_create_savedcmd() < 0)
7164 goto out_free_temp_buffer;
7165
Steven Rostedtab464282008-05-12 21:21:00 +02007166 /* TODO: make the number of buffers hot pluggable with CPUS */
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05007167 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
Steven Rostedt3928a8a2008-09-29 23:02:41 -04007168 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
7169 WARN_ON(1);
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09007170 goto out_free_savedcmd;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04007171 }
Steven Rostedta7603ff2012-08-06 16:24:11 -04007172
Steven Rostedt499e5472012-02-22 15:50:28 -05007173 if (global_trace.buffer_disabled)
7174 tracing_off();
Steven Rostedt3928a8a2008-09-29 23:02:41 -04007175
Steven Rostedte1e232c2014-02-10 23:38:46 -05007176 if (trace_boot_clock) {
7177 ret = tracing_set_clock(&global_trace, trace_boot_clock);
7178 if (ret < 0)
7179 pr_warning("Trace clock %s not defined, going back to default\n",
7180 trace_boot_clock);
7181 }
7182
Steven Rostedt (Red Hat)ca164312013-05-23 11:51:10 -04007183 /*
7184 * register_tracer() might reference current_trace, so it
7185 * needs to be set before we register anything. This is
7186 * just a bootstrap of current_trace anyway.
7187 */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04007188 global_trace.current_trace = &nop_trace;
7189
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05007190 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7191
Steven Rostedt (Red Hat)4104d322014-01-10 17:01:58 -05007192 ftrace_init_global_array_ops(&global_trace);
7193
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04007194 init_trace_flags_index(&global_trace);
7195
Steven Rostedt (Red Hat)ca164312013-05-23 11:51:10 -04007196 register_tracer(&nop_trace);
7197
Steven Rostedt60a11772008-05-12 21:20:44 +02007198 /* All seems OK, enable tracing */
7199 tracing_disabled = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04007200
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007201 atomic_notifier_chain_register(&panic_notifier_list,
7202 &trace_panic_notifier);
7203
7204 register_die_notifier(&trace_die_notifier);
Frederic Weisbecker2fc1dfb2009-03-16 01:45:03 +01007205
Steven Rostedtae63b312012-05-03 23:09:03 -04007206 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
7207
7208 INIT_LIST_HEAD(&global_trace.systems);
7209 INIT_LIST_HEAD(&global_trace.events);
7210 list_add(&global_trace.list, &ftrace_trace_arrays);
7211
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04007212 while (trace_boot_options) {
7213 char *option;
7214
7215 option = strsep(&trace_boot_options, ",");
Steven Rostedt2b6080f2012-05-11 13:29:49 -04007216 trace_set_options(&global_trace, option);
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04007217 }
7218
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04007219 register_snapshot_cmd();
7220
Frederic Weisbecker2fc1dfb2009-03-16 01:45:03 +01007221 return 0;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007222
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09007223out_free_savedcmd:
7224 free_saved_cmdlines_buffer(savedcmd);
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04007225out_free_temp_buffer:
7226 ring_buffer_free(temp_buffer);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10307227out_free_cpumask:
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07007228 free_cpumask_var(global_trace.tracing_cpumask);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10307229out_free_buffer_mask:
7230 free_cpumask_var(tracing_buffer_mask);
7231out:
7232 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02007233}
Steven Rostedtb2821ae2009-02-02 21:38:32 -05007234
Steven Rostedt (Red Hat)5f893b22014-12-12 20:05:10 -05007235void __init trace_init(void)
7236{
Steven Rostedt (Red Hat)0daa23022014-12-12 22:27:10 -05007237 if (tracepoint_printk) {
7238 tracepoint_print_iter =
7239 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
7240 if (WARN_ON(!tracepoint_print_iter))
7241 tracepoint_printk = 0;
7242 }
Steven Rostedt (Red Hat)5f893b22014-12-12 20:05:10 -05007243 tracer_alloc_buffers();
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04007244 trace_event_init();
Steven Rostedt (Red Hat)5f893b22014-12-12 20:05:10 -05007245}
7246
Steven Rostedtb2821ae2009-02-02 21:38:32 -05007247__init static int clear_boot_tracer(void)
7248{
7249 /*
7250 * The default tracer at boot buffer is an init section.
7251 * This function is called in lateinit. If we did not
7252 * find the boot tracer, then clear it out, to prevent
7253 * later registration from accessing the buffer that is
7254 * about to be freed.
7255 */
7256 if (!default_bootup_tracer)
7257 return 0;
7258
7259 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
7260 default_bootup_tracer);
7261 default_bootup_tracer = NULL;
7262
7263 return 0;
7264}
7265
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05007266fs_initcall(tracer_init_tracefs);
Steven Rostedtb2821ae2009-02-02 21:38:32 -05007267late_initcall(clear_boot_tracer);