oprofile: comment cleanup
[linux-2.6.git] / drivers / oprofile / cpu_buffer.c
1 /**
2  * @file cpu_buffer.c
3  *
4  * @remark Copyright 2002 OProfile authors
5  * @remark Read the file COPYING
6  *
7  * @author John Levon <levon@movementarian.org>
8  * @author Barry Kasindorf <barry.kasindorf@amd.com>
9  *
10  * Each CPU has a local buffer that stores PC value/event
11  * pairs. We also log context switches when we notice them.
12  * Eventually each CPU's buffer is processed into the global
13  * event buffer by sync_buffer().
14  *
15  * We use a local buffer for two reasons: an NMI or similar
16  * interrupt cannot synchronise, and high sampling rates
17  * would lead to catastrophic global synchronisation if
18  * a global buffer was used.
19  */
20
21 #include <linux/sched.h>
22 #include <linux/oprofile.h>
23 #include <linux/vmalloc.h>
24 #include <linux/errno.h>
25
26 #include "event_buffer.h"
27 #include "cpu_buffer.h"
28 #include "buffer_sync.h"
29 #include "oprof.h"
30
31 DEFINE_PER_CPU(struct oprofile_cpu_buffer, cpu_buffer);
32
33 static void wq_sync_buffer(struct work_struct *work);
34
35 #define DEFAULT_TIMER_EXPIRE (HZ / 10)
36 static int work_enabled;
37
38 void free_cpu_buffers(void)
39 {
40         int i;
41
42         for_each_possible_cpu(i) {
43                 vfree(per_cpu(cpu_buffer, i).buffer);
44                 per_cpu(cpu_buffer, i).buffer = NULL;
45         }
46 }
47
48 unsigned long oprofile_get_cpu_buffer_size(void)
49 {
50         return fs_cpu_buffer_size;
51 }
52
53 void oprofile_cpu_buffer_inc_smpl_lost(void)
54 {
55         struct oprofile_cpu_buffer *cpu_buf
56                 = &__get_cpu_var(cpu_buffer);
57
58         cpu_buf->sample_lost_overflow++;
59 }
60
61 int alloc_cpu_buffers(void)
62 {
63         int i;
64
65         unsigned long buffer_size = fs_cpu_buffer_size;
66
67         for_each_possible_cpu(i) {
68                 struct oprofile_cpu_buffer *b = &per_cpu(cpu_buffer, i);
69
70                 b->buffer = vmalloc_node(sizeof(struct op_sample) * buffer_size,
71                         cpu_to_node(i));
72                 if (!b->buffer)
73                         goto fail;
74
75                 b->last_task = NULL;
76                 b->last_is_kernel = -1;
77                 b->tracing = 0;
78                 b->buffer_size = buffer_size;
79                 b->tail_pos = 0;
80                 b->head_pos = 0;
81                 b->sample_received = 0;
82                 b->sample_lost_overflow = 0;
83                 b->backtrace_aborted = 0;
84                 b->sample_invalid_eip = 0;
85                 b->cpu = i;
86                 INIT_DELAYED_WORK(&b->work, wq_sync_buffer);
87         }
88         return 0;
89
90 fail:
91         free_cpu_buffers();
92         return -ENOMEM;
93 }
94
95 void start_cpu_work(void)
96 {
97         int i;
98
99         work_enabled = 1;
100
101         for_each_online_cpu(i) {
102                 struct oprofile_cpu_buffer *b = &per_cpu(cpu_buffer, i);
103
104                 /*
105                  * Spread the work by 1 jiffy per cpu so they dont all
106                  * fire at once.
107                  */
108                 schedule_delayed_work_on(i, &b->work, DEFAULT_TIMER_EXPIRE + i);
109         }
110 }
111
112 void end_cpu_work(void)
113 {
114         int i;
115
116         work_enabled = 0;
117
118         for_each_online_cpu(i) {
119                 struct oprofile_cpu_buffer *b = &per_cpu(cpu_buffer, i);
120
121                 cancel_delayed_work(&b->work);
122         }
123
124         flush_scheduled_work();
125 }
126
127 /* Resets the cpu buffer to a sane state. */
128 void cpu_buffer_reset(struct oprofile_cpu_buffer *cpu_buf)
129 {
130         /*
131          * reset these to invalid values; the next sample collected
132          * will populate the buffer with proper values to initialize
133          * the buffer
134          */
135         cpu_buf->last_is_kernel = -1;
136         cpu_buf->last_task = NULL;
137 }
138
139 /* compute number of available slots in cpu_buffer queue */
140 static unsigned long nr_available_slots(struct oprofile_cpu_buffer const *b)
141 {
142         unsigned long head = b->head_pos;
143         unsigned long tail = b->tail_pos;
144
145         if (tail > head)
146                 return (tail - head) - 1;
147
148         return tail + (b->buffer_size - head) - 1;
149 }
150
151 static void increment_head(struct oprofile_cpu_buffer *b)
152 {
153         unsigned long new_head = b->head_pos + 1;
154
155         /*
156          * Ensure anything written to the slot before we increment is
157          * visible
158          */
159         wmb();
160
161         if (new_head < b->buffer_size)
162                 b->head_pos = new_head;
163         else
164                 b->head_pos = 0;
165 }
166
167 static inline void
168 add_sample(struct oprofile_cpu_buffer *cpu_buf,
169            unsigned long pc, unsigned long event)
170 {
171         struct op_sample *entry = &cpu_buf->buffer[cpu_buf->head_pos];
172         entry->eip = pc;
173         entry->event = event;
174         increment_head(cpu_buf);
175 }
176
177 static inline void
178 add_code(struct oprofile_cpu_buffer *buffer, unsigned long value)
179 {
180         add_sample(buffer, ESCAPE_CODE, value);
181 }
182
183 /* This must be safe from any context. It's safe writing here
184  * because of the head/tail separation of the writer and reader
185  * of the CPU buffer.
186  *
187  * is_kernel is needed because on some architectures you cannot
188  * tell if you are in kernel or user space simply by looking at
189  * pc. We tag this in the buffer by generating kernel enter/exit
190  * events whenever is_kernel changes
191  */
192 static int log_sample(struct oprofile_cpu_buffer *cpu_buf, unsigned long pc,
193                       int is_kernel, unsigned long event)
194 {
195         struct task_struct *task;
196
197         cpu_buf->sample_received++;
198
199         if (pc == ESCAPE_CODE) {
200                 cpu_buf->sample_invalid_eip++;
201                 return 0;
202         }
203
204         if (nr_available_slots(cpu_buf) < 3) {
205                 cpu_buf->sample_lost_overflow++;
206                 return 0;
207         }
208
209         is_kernel = !!is_kernel;
210
211         task = current;
212
213         /* notice a switch from user->kernel or vice versa */
214         if (cpu_buf->last_is_kernel != is_kernel) {
215                 cpu_buf->last_is_kernel = is_kernel;
216                 add_code(cpu_buf, is_kernel);
217         }
218
219         /* notice a task switch */
220         if (cpu_buf->last_task != task) {
221                 cpu_buf->last_task = task;
222                 add_code(cpu_buf, (unsigned long)task);
223         }
224
225         add_sample(cpu_buf, pc, event);
226         return 1;
227 }
228
229 static int oprofile_begin_trace(struct oprofile_cpu_buffer *cpu_buf)
230 {
231         if (nr_available_slots(cpu_buf) < 4) {
232                 cpu_buf->sample_lost_overflow++;
233                 return 0;
234         }
235
236         add_code(cpu_buf, CPU_TRACE_BEGIN);
237         cpu_buf->tracing = 1;
238         return 1;
239 }
240
241 static void oprofile_end_trace(struct oprofile_cpu_buffer *cpu_buf)
242 {
243         cpu_buf->tracing = 0;
244 }
245
246 void oprofile_add_ext_sample(unsigned long pc, struct pt_regs * const regs,
247                                 unsigned long event, int is_kernel)
248 {
249         struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(cpu_buffer);
250
251         if (!backtrace_depth) {
252                 log_sample(cpu_buf, pc, is_kernel, event);
253                 return;
254         }
255
256         if (!oprofile_begin_trace(cpu_buf))
257                 return;
258
259         /*
260          * if log_sample() fail we can't backtrace since we lost the
261          * source of this event
262          */
263         if (log_sample(cpu_buf, pc, is_kernel, event))
264                 oprofile_ops.backtrace(regs, backtrace_depth);
265         oprofile_end_trace(cpu_buf);
266 }
267
268 void oprofile_add_sample(struct pt_regs * const regs, unsigned long event)
269 {
270         int is_kernel = !user_mode(regs);
271         unsigned long pc = profile_pc(regs);
272
273         oprofile_add_ext_sample(pc, regs, event, is_kernel);
274 }
275
276 #ifdef CONFIG_OPROFILE_IBS
277
278 #define MAX_IBS_SAMPLE_SIZE 14
279
280 void oprofile_add_ibs_sample(struct pt_regs *const regs,
281                              unsigned int *const ibs_sample, int ibs_code)
282 {
283         int is_kernel = !user_mode(regs);
284         struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(cpu_buffer);
285         struct task_struct *task;
286
287         cpu_buf->sample_received++;
288
289         if (nr_available_slots(cpu_buf) < MAX_IBS_SAMPLE_SIZE) {
290                 /* we can't backtrace since we lost the source of this event */
291                 cpu_buf->sample_lost_overflow++;
292                 return;
293         }
294
295         /* notice a switch from user->kernel or vice versa */
296         if (cpu_buf->last_is_kernel != is_kernel) {
297                 cpu_buf->last_is_kernel = is_kernel;
298                 add_code(cpu_buf, is_kernel);
299         }
300
301         /* notice a task switch */
302         if (!is_kernel) {
303                 task = current;
304                 if (cpu_buf->last_task != task) {
305                         cpu_buf->last_task = task;
306                         add_code(cpu_buf, (unsigned long)task);
307                 }
308         }
309
310         add_code(cpu_buf, ibs_code);
311         add_sample(cpu_buf, ibs_sample[0], ibs_sample[1]);
312         add_sample(cpu_buf, ibs_sample[2], ibs_sample[3]);
313         add_sample(cpu_buf, ibs_sample[4], ibs_sample[5]);
314
315         if (ibs_code == IBS_OP_BEGIN) {
316                 add_sample(cpu_buf, ibs_sample[6], ibs_sample[7]);
317                 add_sample(cpu_buf, ibs_sample[8], ibs_sample[9]);
318                 add_sample(cpu_buf, ibs_sample[10], ibs_sample[11]);
319         }
320
321         if (backtrace_depth)
322                 oprofile_ops.backtrace(regs, backtrace_depth);
323 }
324
325 #endif
326
327 void oprofile_add_pc(unsigned long pc, int is_kernel, unsigned long event)
328 {
329         struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(cpu_buffer);
330         log_sample(cpu_buf, pc, is_kernel, event);
331 }
332
333 void oprofile_add_trace(unsigned long pc)
334 {
335         struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(cpu_buffer);
336
337         if (!cpu_buf->tracing)
338                 return;
339
340         if (nr_available_slots(cpu_buf) < 1) {
341                 cpu_buf->tracing = 0;
342                 cpu_buf->sample_lost_overflow++;
343                 return;
344         }
345
346         /*
347          * broken frame can give an eip with the same value as an
348          * escape code, abort the trace if we get it
349          */
350         if (pc == ESCAPE_CODE) {
351                 cpu_buf->tracing = 0;
352                 cpu_buf->backtrace_aborted++;
353                 return;
354         }
355
356         add_sample(cpu_buf, pc, 0);
357 }
358
359 /*
360  * This serves to avoid cpu buffer overflow, and makes sure
361  * the task mortuary progresses
362  *
363  * By using schedule_delayed_work_on and then schedule_delayed_work
364  * we guarantee this will stay on the correct cpu
365  */
366 static void wq_sync_buffer(struct work_struct *work)
367 {
368         struct oprofile_cpu_buffer *b =
369                 container_of(work, struct oprofile_cpu_buffer, work.work);
370         if (b->cpu != smp_processor_id()) {
371                 printk(KERN_DEBUG "WQ on CPU%d, prefer CPU%d\n",
372                        smp_processor_id(), b->cpu);
373
374                 if (!cpu_online(b->cpu)) {
375                         cancel_delayed_work(&b->work);
376                         return;
377                 }
378         }
379         sync_buffer(b->cpu);
380
381         /* don't re-add the work if we're shutting down */
382         if (work_enabled)
383                 schedule_delayed_work(&b->work, DEFAULT_TIMER_EXPIRE);
384 }