17bbdb811aa753309664268f582a10dc5459e4ab
[linux-2.6.git] / block / blktrace.c
1 /*
2  * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk>
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11  * GNU General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public License
14  * along with this program; if not, write to the Free Software
15  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
16  *
17  */
18 #include <linux/kernel.h>
19 #include <linux/blkdev.h>
20 #include <linux/blktrace_api.h>
21 #include <linux/percpu.h>
22 #include <linux/init.h>
23 #include <linux/mutex.h>
24 #include <linux/debugfs.h>
25 #include <linux/time.h>
26 #include <asm/uaccess.h>
27
28 static DEFINE_PER_CPU(unsigned long long, blk_trace_cpu_offset) = { 0, };
29 static unsigned int blktrace_seq __read_mostly = 1;
30
31 /*
32  * Send out a notify message.
33  */
34 static unsigned int trace_note(struct blk_trace *bt, pid_t pid, int action,
35                                const void *data, size_t len)
36 {
37         struct blk_io_trace *t;
38
39         t = relay_reserve(bt->rchan, sizeof(*t) + len);
40         if (t) {
41                 const int cpu = smp_processor_id();
42
43                 t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
44                 t->time = sched_clock() - per_cpu(blk_trace_cpu_offset, cpu);
45                 t->device = bt->dev;
46                 t->action = action;
47                 t->pid = pid;
48                 t->cpu = cpu;
49                 t->pdu_len = len;
50                 memcpy((void *) t + sizeof(*t), data, len);
51         }
52
53         return blktrace_seq;
54 }
55
56 /*
57  * Send out a notify for this process, if we haven't done so since a trace
58  * started
59  */
60 static void trace_note_tsk(struct blk_trace *bt, struct task_struct *tsk)
61 {
62         tsk->btrace_seq = trace_note(bt, tsk->pid,
63                         BLK_TN_PROCESS,
64                         tsk->comm, sizeof(tsk->comm));
65 }
66
67 static void trace_note_time(struct blk_trace *bt)
68 {
69         struct timespec now;
70         unsigned long flags;
71         u32 words[2];
72
73         getnstimeofday(&now);
74         words[0] = now.tv_sec;
75         words[1] = now.tv_nsec;
76
77         local_irq_save(flags);
78         trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words));
79         local_irq_restore(flags);
80 }
81
82 static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector,
83                          pid_t pid)
84 {
85         if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0)
86                 return 1;
87         if (sector < bt->start_lba || sector > bt->end_lba)
88                 return 1;
89         if (bt->pid && pid != bt->pid)
90                 return 1;
91
92         return 0;
93 }
94
95 /*
96  * Data direction bit lookup
97  */
98 static u32 ddir_act[2] __read_mostly = { BLK_TC_ACT(BLK_TC_READ), BLK_TC_ACT(BLK_TC_WRITE) };
99
100 /*
101  * Bio action bits of interest
102  */
103 static u32 bio_act[9] __read_mostly = { 0, BLK_TC_ACT(BLK_TC_BARRIER), BLK_TC_ACT(BLK_TC_SYNC), 0, BLK_TC_ACT(BLK_TC_AHEAD), 0, 0, 0, BLK_TC_ACT(BLK_TC_META) };
104
105 /*
106  * More could be added as needed, taking care to increment the decrementer
107  * to get correct indexing
108  */
109 #define trace_barrier_bit(rw)   \
110         (((rw) & (1 << BIO_RW_BARRIER)) >> (BIO_RW_BARRIER - 0))
111 #define trace_sync_bit(rw)      \
112         (((rw) & (1 << BIO_RW_SYNC)) >> (BIO_RW_SYNC - 1))
113 #define trace_ahead_bit(rw)     \
114         (((rw) & (1 << BIO_RW_AHEAD)) << (2 - BIO_RW_AHEAD))
115 #define trace_meta_bit(rw)      \
116         (((rw) & (1 << BIO_RW_META)) >> (BIO_RW_META - 3))
117
118 /*
119  * The worker for the various blk_add_trace*() types. Fills out a
120  * blk_io_trace structure and places it in a per-cpu subbuffer.
121  */
122 void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,
123                      int rw, u32 what, int error, int pdu_len, void *pdu_data)
124 {
125         struct task_struct *tsk = current;
126         struct blk_io_trace *t;
127         unsigned long flags;
128         unsigned long *sequence;
129         pid_t pid;
130         int cpu;
131
132         if (unlikely(bt->trace_state != Blktrace_running))
133                 return;
134
135         what |= ddir_act[rw & WRITE];
136         what |= bio_act[trace_barrier_bit(rw)];
137         what |= bio_act[trace_sync_bit(rw)];
138         what |= bio_act[trace_ahead_bit(rw)];
139         what |= bio_act[trace_meta_bit(rw)];
140
141         pid = tsk->pid;
142         if (unlikely(act_log_check(bt, what, sector, pid)))
143                 return;
144
145         /*
146          * A word about the locking here - we disable interrupts to reserve
147          * some space in the relay per-cpu buffer, to prevent an irq
148          * from coming in and stepping on our toes. Once reserved, it's
149          * enough to get preemption disabled to prevent read of this data
150          * before we are through filling it. get_cpu()/put_cpu() does this
151          * for us
152          */
153         local_irq_save(flags);
154
155         if (unlikely(tsk->btrace_seq != blktrace_seq))
156                 trace_note_tsk(bt, tsk);
157
158         t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len);
159         if (t) {
160                 cpu = smp_processor_id();
161                 sequence = per_cpu_ptr(bt->sequence, cpu);
162
163                 t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
164                 t->sequence = ++(*sequence);
165                 t->time = sched_clock() - per_cpu(blk_trace_cpu_offset, cpu);
166                 t->sector = sector;
167                 t->bytes = bytes;
168                 t->action = what;
169                 t->pid = pid;
170                 t->device = bt->dev;
171                 t->cpu = cpu;
172                 t->error = error;
173                 t->pdu_len = pdu_len;
174
175                 if (pdu_len)
176                         memcpy((void *) t + sizeof(*t), pdu_data, pdu_len);
177         }
178
179         local_irq_restore(flags);
180 }
181
182 EXPORT_SYMBOL_GPL(__blk_add_trace);
183
184 static struct dentry *blk_tree_root;
185 static struct mutex blk_tree_mutex;
186 static unsigned int root_users;
187
188 static inline void blk_remove_root(void)
189 {
190         if (blk_tree_root) {
191                 debugfs_remove(blk_tree_root);
192                 blk_tree_root = NULL;
193         }
194 }
195
196 static void blk_remove_tree(struct dentry *dir)
197 {
198         mutex_lock(&blk_tree_mutex);
199         debugfs_remove(dir);
200         if (--root_users == 0)
201                 blk_remove_root();
202         mutex_unlock(&blk_tree_mutex);
203 }
204
205 static struct dentry *blk_create_tree(const char *blk_name)
206 {
207         struct dentry *dir = NULL;
208
209         mutex_lock(&blk_tree_mutex);
210
211         if (!blk_tree_root) {
212                 blk_tree_root = debugfs_create_dir("block", NULL);
213                 if (!blk_tree_root)
214                         goto err;
215         }
216
217         dir = debugfs_create_dir(blk_name, blk_tree_root);
218         if (dir)
219                 root_users++;
220         else
221                 blk_remove_root();
222
223 err:
224         mutex_unlock(&blk_tree_mutex);
225         return dir;
226 }
227
228 static void blk_trace_cleanup(struct blk_trace *bt)
229 {
230         relay_close(bt->rchan);
231         debugfs_remove(bt->dropped_file);
232         blk_remove_tree(bt->dir);
233         free_percpu(bt->sequence);
234         kfree(bt);
235 }
236
237 static int blk_trace_remove(request_queue_t *q)
238 {
239         struct blk_trace *bt;
240
241         bt = xchg(&q->blk_trace, NULL);
242         if (!bt)
243                 return -EINVAL;
244
245         if (bt->trace_state == Blktrace_setup ||
246             bt->trace_state == Blktrace_stopped)
247                 blk_trace_cleanup(bt);
248
249         return 0;
250 }
251
252 static int blk_dropped_open(struct inode *inode, struct file *filp)
253 {
254         filp->private_data = inode->i_private;
255
256         return 0;
257 }
258
259 static ssize_t blk_dropped_read(struct file *filp, char __user *buffer,
260                                 size_t count, loff_t *ppos)
261 {
262         struct blk_trace *bt = filp->private_data;
263         char buf[16];
264
265         snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped));
266
267         return simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf));
268 }
269
270 static struct file_operations blk_dropped_fops = {
271         .owner =        THIS_MODULE,
272         .open =         blk_dropped_open,
273         .read =         blk_dropped_read,
274 };
275
276 /*
277  * Keep track of how many times we encountered a full subbuffer, to aid
278  * the user space app in telling how many lost events there were.
279  */
280 static int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf,
281                                      void *prev_subbuf, size_t prev_padding)
282 {
283         struct blk_trace *bt;
284
285         if (!relay_buf_full(buf))
286                 return 1;
287
288         bt = buf->chan->private_data;
289         atomic_inc(&bt->dropped);
290         return 0;
291 }
292
293 static int blk_remove_buf_file_callback(struct dentry *dentry)
294 {
295         debugfs_remove(dentry);
296         return 0;
297 }
298
299 static struct dentry *blk_create_buf_file_callback(const char *filename,
300                                                    struct dentry *parent,
301                                                    int mode,
302                                                    struct rchan_buf *buf,
303                                                    int *is_global)
304 {
305         return debugfs_create_file(filename, mode, parent, buf,
306                                         &relay_file_operations);
307 }
308
309 static struct rchan_callbacks blk_relay_callbacks = {
310         .subbuf_start           = blk_subbuf_start_callback,
311         .create_buf_file        = blk_create_buf_file_callback,
312         .remove_buf_file        = blk_remove_buf_file_callback,
313 };
314
315 /*
316  * Setup everything required to start tracing
317  */
318 static int blk_trace_setup(request_queue_t *q, struct block_device *bdev,
319                            char __user *arg)
320 {
321         struct blk_user_trace_setup buts;
322         struct blk_trace *old_bt, *bt = NULL;
323         struct dentry *dir = NULL;
324         char b[BDEVNAME_SIZE];
325         int ret, i;
326
327         if (copy_from_user(&buts, arg, sizeof(buts)))
328                 return -EFAULT;
329
330         if (!buts.buf_size || !buts.buf_nr)
331                 return -EINVAL;
332
333         strcpy(buts.name, bdevname(bdev, b));
334
335         /*
336          * some device names have larger paths - convert the slashes
337          * to underscores for this to work as expected
338          */
339         for (i = 0; i < strlen(buts.name); i++)
340                 if (buts.name[i] == '/')
341                         buts.name[i] = '_';
342
343         if (copy_to_user(arg, &buts, sizeof(buts)))
344                 return -EFAULT;
345
346         ret = -ENOMEM;
347         bt = kzalloc(sizeof(*bt), GFP_KERNEL);
348         if (!bt)
349                 goto err;
350
351         bt->sequence = alloc_percpu(unsigned long);
352         if (!bt->sequence)
353                 goto err;
354
355         ret = -ENOENT;
356         dir = blk_create_tree(buts.name);
357         if (!dir)
358                 goto err;
359
360         bt->dir = dir;
361         bt->dev = bdev->bd_dev;
362         atomic_set(&bt->dropped, 0);
363
364         ret = -EIO;
365         bt->dropped_file = debugfs_create_file("dropped", 0444, dir, bt, &blk_dropped_fops);
366         if (!bt->dropped_file)
367                 goto err;
368
369         bt->rchan = relay_open("trace", dir, buts.buf_size, buts.buf_nr, &blk_relay_callbacks);
370         if (!bt->rchan)
371                 goto err;
372         bt->rchan->private_data = bt;
373
374         bt->act_mask = buts.act_mask;
375         if (!bt->act_mask)
376                 bt->act_mask = (u16) -1;
377
378         bt->start_lba = buts.start_lba;
379         bt->end_lba = buts.end_lba;
380         if (!bt->end_lba)
381                 bt->end_lba = -1ULL;
382
383         bt->pid = buts.pid;
384         bt->trace_state = Blktrace_setup;
385
386         ret = -EBUSY;
387         old_bt = xchg(&q->blk_trace, bt);
388         if (old_bt) {
389                 (void) xchg(&q->blk_trace, old_bt);
390                 goto err;
391         }
392
393         return 0;
394 err:
395         if (dir)
396                 blk_remove_tree(dir);
397         if (bt) {
398                 if (bt->dropped_file)
399                         debugfs_remove(bt->dropped_file);
400                 if (bt->sequence)
401                         free_percpu(bt->sequence);
402                 if (bt->rchan)
403                         relay_close(bt->rchan);
404                 kfree(bt);
405         }
406         return ret;
407 }
408
409 static int blk_trace_startstop(request_queue_t *q, int start)
410 {
411         struct blk_trace *bt;
412         int ret;
413
414         if ((bt = q->blk_trace) == NULL)
415                 return -EINVAL;
416
417         /*
418          * For starting a trace, we can transition from a setup or stopped
419          * trace. For stopping a trace, the state must be running
420          */
421         ret = -EINVAL;
422         if (start) {
423                 if (bt->trace_state == Blktrace_setup ||
424                     bt->trace_state == Blktrace_stopped) {
425                         blktrace_seq++;
426                         smp_mb();
427                         bt->trace_state = Blktrace_running;
428
429                         trace_note_time(bt);
430                         ret = 0;
431                 }
432         } else {
433                 if (bt->trace_state == Blktrace_running) {
434                         bt->trace_state = Blktrace_stopped;
435                         relay_flush(bt->rchan);
436                         ret = 0;
437                 }
438         }
439
440         return ret;
441 }
442
443 /**
444  * blk_trace_ioctl: - handle the ioctls associated with tracing
445  * @bdev:       the block device
446  * @cmd:        the ioctl cmd
447  * @arg:        the argument data, if any
448  *
449  **/
450 int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg)
451 {
452         request_queue_t *q;
453         int ret, start = 0;
454
455         q = bdev_get_queue(bdev);
456         if (!q)
457                 return -ENXIO;
458
459         mutex_lock(&bdev->bd_mutex);
460
461         switch (cmd) {
462         case BLKTRACESETUP:
463                 ret = blk_trace_setup(q, bdev, arg);
464                 break;
465         case BLKTRACESTART:
466                 start = 1;
467         case BLKTRACESTOP:
468                 ret = blk_trace_startstop(q, start);
469                 break;
470         case BLKTRACETEARDOWN:
471                 ret = blk_trace_remove(q);
472                 break;
473         default:
474                 ret = -ENOTTY;
475                 break;
476         }
477
478         mutex_unlock(&bdev->bd_mutex);
479         return ret;
480 }
481
482 /**
483  * blk_trace_shutdown: - stop and cleanup trace structures
484  * @q:    the request queue associated with the device
485  *
486  **/
487 void blk_trace_shutdown(request_queue_t *q)
488 {
489         if (q->blk_trace) {
490                 blk_trace_startstop(q, 0);
491                 blk_trace_remove(q);
492         }
493 }
494
495 /*
496  * Average offset over two calls to sched_clock() with a gettimeofday()
497  * in the middle
498  */
499 static void blk_check_time(unsigned long long *t)
500 {
501         unsigned long long a, b;
502         struct timeval tv;
503
504         a = sched_clock();
505         do_gettimeofday(&tv);
506         b = sched_clock();
507
508         *t = tv.tv_sec * 1000000000 + tv.tv_usec * 1000;
509         *t -= (a + b) / 2;
510 }
511
512 /*
513  * calibrate our inter-CPU timings
514  */
515 static void blk_trace_check_cpu_time(void *data)
516 {
517         unsigned long long *t;
518         int cpu = get_cpu();
519
520         t = &per_cpu(blk_trace_cpu_offset, cpu);
521
522         /*
523          * Just call it twice, hopefully the second call will be cache hot
524          * and a little more precise
525          */
526         blk_check_time(t);
527         blk_check_time(t);
528
529         put_cpu();
530 }
531
532 static void blk_trace_set_ht_offsets(void)
533 {
534 #if defined(CONFIG_SCHED_SMT)
535         int cpu, i;
536
537         /*
538          * now make sure HT siblings have the same time offset
539          */
540         preempt_disable();
541         for_each_online_cpu(cpu) {
542                 unsigned long long *cpu_off, *sibling_off;
543
544                 for_each_cpu_mask(i, cpu_sibling_map[cpu]) {
545                         if (i == cpu)
546                                 continue;
547
548                         cpu_off = &per_cpu(blk_trace_cpu_offset, cpu);
549                         sibling_off = &per_cpu(blk_trace_cpu_offset, i);
550                         *sibling_off = *cpu_off;
551                 }
552         }
553         preempt_enable();
554 #endif
555 }
556
557 static __init int blk_trace_init(void)
558 {
559         mutex_init(&blk_tree_mutex);
560         on_each_cpu(blk_trace_check_cpu_time, NULL, 1, 1);
561         blk_trace_set_ht_offsets();
562
563         return 0;
564 }
565
566 module_init(blk_trace_init);
567