[PATCH] Block queue IO tracing support (blktrace) as of 2006-03-23
[linux-2.6.git] / block / blktrace.c
1 /*
2  * Copyright (C) 2006 Jens Axboe <axboe@suse.de>
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11  * GNU General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public License
14  * along with this program; if not, write to the Free Software
15  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
16  *
17  */
18 #include <linux/config.h>
19 #include <linux/kernel.h>
20 #include <linux/blkdev.h>
21 #include <linux/blktrace_api.h>
22 #include <linux/percpu.h>
23 #include <linux/init.h>
24 #include <linux/mutex.h>
25 #include <linux/debugfs.h>
26 #include <asm/uaccess.h>
27
28 static DEFINE_PER_CPU(unsigned long long, blk_trace_cpu_offset) = { 0, };
29 static unsigned int blktrace_seq __read_mostly = 1;
30
31 /*
32  * Send out a notify for this process, if we haven't done so since a trace
33  * started
34  */
35 static void trace_note_tsk(struct blk_trace *bt, struct task_struct *tsk)
36 {
37         struct blk_io_trace *t;
38
39         t = relay_reserve(bt->rchan, sizeof(*t) + sizeof(tsk->comm));
40         if (t) {
41                 t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
42                 t->device = bt->dev;
43                 t->action = BLK_TC_ACT(BLK_TC_NOTIFY);
44                 t->pid = tsk->pid;
45                 t->cpu = smp_processor_id();
46                 t->pdu_len = sizeof(tsk->comm);
47                 memcpy((void *) t + sizeof(*t), tsk->comm, t->pdu_len);
48                 tsk->btrace_seq = blktrace_seq;
49         }
50 }
51
52 static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector,
53                          pid_t pid)
54 {
55         if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0)
56                 return 1;
57         if (sector < bt->start_lba || sector > bt->end_lba)
58                 return 1;
59         if (bt->pid && pid != bt->pid)
60                 return 1;
61
62         return 0;
63 }
64
65 /*
66  * Data direction bit lookup
67  */
68 static u32 ddir_act[2] __read_mostly = { BLK_TC_ACT(BLK_TC_READ), BLK_TC_ACT(BLK_TC_WRITE) };
69
70 /*
71  * Bio action bits of interest
72  */
73 static u32 bio_act[3] __read_mostly = { 0, BLK_TC_ACT(BLK_TC_BARRIER), BLK_TC_ACT(BLK_TC_SYNC) };
74
75 /*
76  * More could be added as needed, taking care to increment the decrementer
77  * to get correct indexing
78  */
79 #define trace_barrier_bit(rw)   \
80         (((rw) & (1 << BIO_RW_BARRIER)) >> (BIO_RW_BARRIER - 0))
81 #define trace_sync_bit(rw)      \
82         (((rw) & (1 << BIO_RW_SYNC)) >> (BIO_RW_SYNC - 1))
83
84 /*
85  * The worker for the various blk_add_trace*() types. Fills out a
86  * blk_io_trace structure and places it in a per-cpu subbuffer.
87  */
88 void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,
89                      int rw, u32 what, int error, int pdu_len, void *pdu_data)
90 {
91         struct task_struct *tsk = current;
92         struct blk_io_trace *t;
93         unsigned long flags;
94         unsigned long *sequence;
95         pid_t pid;
96         int cpu;
97
98         if (unlikely(bt->trace_state != Blktrace_running))
99                 return;
100
101         what |= ddir_act[rw & WRITE];
102         what |= bio_act[trace_barrier_bit(rw)];
103         what |= bio_act[trace_sync_bit(rw)];
104
105         pid = tsk->pid;
106         if (unlikely(act_log_check(bt, what, sector, pid)))
107                 return;
108
109         /*
110          * A word about the locking here - we disable interrupts to reserve
111          * some space in the relay per-cpu buffer, to prevent an irq
112          * from coming in and stepping on our toes. Once reserved, it's
113          * enough to get preemption disabled to prevent read of this data
114          * before we are through filling it. get_cpu()/put_cpu() does this
115          * for us
116          */
117         local_irq_save(flags);
118
119         if (unlikely(tsk->btrace_seq != blktrace_seq))
120                 trace_note_tsk(bt, tsk);
121
122         t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len);
123         if (t) {
124                 cpu = smp_processor_id();
125                 sequence = per_cpu_ptr(bt->sequence, cpu);
126
127                 t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
128                 t->sequence = ++(*sequence);
129                 t->time = sched_clock() - per_cpu(blk_trace_cpu_offset, cpu);
130                 t->sector = sector;
131                 t->bytes = bytes;
132                 t->action = what;
133                 t->pid = pid;
134                 t->device = bt->dev;
135                 t->cpu = cpu;
136                 t->error = error;
137                 t->pdu_len = pdu_len;
138
139                 if (pdu_len)
140                         memcpy((void *) t + sizeof(*t), pdu_data, pdu_len);
141         }
142
143         local_irq_restore(flags);
144 }
145
146 EXPORT_SYMBOL_GPL(__blk_add_trace);
147
148 static struct dentry *blk_tree_root;
149 static struct mutex blk_tree_mutex;
150 static unsigned int root_users;
151
152 static inline void blk_remove_root(void)
153 {
154         if (blk_tree_root) {
155                 debugfs_remove(blk_tree_root);
156                 blk_tree_root = NULL;
157         }
158 }
159
160 static void blk_remove_tree(struct dentry *dir)
161 {
162         mutex_lock(&blk_tree_mutex);
163         debugfs_remove(dir);
164         if (--root_users == 0)
165                 blk_remove_root();
166         mutex_unlock(&blk_tree_mutex);
167 }
168
169 static struct dentry *blk_create_tree(const char *blk_name)
170 {
171         struct dentry *dir = NULL;
172
173         mutex_lock(&blk_tree_mutex);
174
175         if (!blk_tree_root) {
176                 blk_tree_root = debugfs_create_dir("block", NULL);
177                 if (!blk_tree_root)
178                         goto err;
179         }
180
181         dir = debugfs_create_dir(blk_name, blk_tree_root);
182         if (dir)
183                 root_users++;
184         else
185                 blk_remove_root();
186
187 err:
188         mutex_unlock(&blk_tree_mutex);
189         return dir;
190 }
191
192 static void blk_trace_cleanup(struct blk_trace *bt)
193 {
194         relay_close(bt->rchan);
195         debugfs_remove(bt->dropped_file);
196         blk_remove_tree(bt->dir);
197         free_percpu(bt->sequence);
198         kfree(bt);
199 }
200
201 static int blk_trace_remove(request_queue_t *q)
202 {
203         struct blk_trace *bt;
204
205         bt = xchg(&q->blk_trace, NULL);
206         if (!bt)
207                 return -EINVAL;
208
209         if (bt->trace_state == Blktrace_setup ||
210             bt->trace_state == Blktrace_stopped)
211                 blk_trace_cleanup(bt);
212
213         return 0;
214 }
215
216 static int blk_dropped_open(struct inode *inode, struct file *filp)
217 {
218         filp->private_data = inode->u.generic_ip;
219
220         return 0;
221 }
222
223 static ssize_t blk_dropped_read(struct file *filp, char __user *buffer,
224                                 size_t count, loff_t *ppos)
225 {
226         struct blk_trace *bt = filp->private_data;
227         char buf[16];
228
229         snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped));
230
231         return simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf));
232 }
233
234 static struct file_operations blk_dropped_fops = {
235         .owner =        THIS_MODULE,
236         .open =         blk_dropped_open,
237         .read =         blk_dropped_read,
238 };
239
240 /*
241  * Keep track of how many times we encountered a full subbuffer, to aid
242  * the user space app in telling how many lost events there were.
243  */
244 static int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf,
245                                      void *prev_subbuf, size_t prev_padding)
246 {
247         struct blk_trace *bt;
248
249         if (!relay_buf_full(buf))
250                 return 1;
251
252         bt = buf->chan->private_data;
253         atomic_inc(&bt->dropped);
254         return 0;
255 }
256
257 static int blk_remove_buf_file_callback(struct dentry *dentry)
258 {
259         debugfs_remove(dentry);
260         return 0;
261 }
262
263 static struct dentry *blk_create_buf_file_callback(const char *filename,
264                                                    struct dentry *parent,
265                                                    int mode,
266                                                    struct rchan_buf *buf,
267                                                    int *is_global)
268 {
269         return debugfs_create_file(filename, mode, parent, buf,
270                                         &relay_file_operations);
271 }
272
273 static struct rchan_callbacks blk_relay_callbacks = {
274         .subbuf_start           = blk_subbuf_start_callback,
275         .create_buf_file        = blk_create_buf_file_callback,
276         .remove_buf_file        = blk_remove_buf_file_callback,
277 };
278
279 /*
280  * Setup everything required to start tracing
281  */
282 static int blk_trace_setup(request_queue_t *q, struct block_device *bdev,
283                            char __user *arg)
284 {
285         struct blk_user_trace_setup buts;
286         struct blk_trace *old_bt, *bt = NULL;
287         struct dentry *dir = NULL;
288         char b[BDEVNAME_SIZE];
289         int ret, i;
290
291         if (copy_from_user(&buts, arg, sizeof(buts)))
292                 return -EFAULT;
293
294         if (!buts.buf_size || !buts.buf_nr)
295                 return -EINVAL;
296
297         strcpy(buts.name, bdevname(bdev, b));
298
299         /*
300          * some device names have larger paths - convert the slashes
301          * to underscores for this to work as expected
302          */
303         for (i = 0; i < strlen(buts.name); i++)
304                 if (buts.name[i] == '/')
305                         buts.name[i] = '_';
306
307         if (copy_to_user(arg, &buts, sizeof(buts)))
308                 return -EFAULT;
309
310         ret = -ENOMEM;
311         bt = kzalloc(sizeof(*bt), GFP_KERNEL);
312         if (!bt)
313                 goto err;
314
315         bt->sequence = alloc_percpu(unsigned long);
316         if (!bt->sequence)
317                 goto err;
318
319         ret = -ENOENT;
320         dir = blk_create_tree(buts.name);
321         if (!dir)
322                 goto err;
323
324         bt->dir = dir;
325         bt->dev = bdev->bd_dev;
326         atomic_set(&bt->dropped, 0);
327
328         ret = -EIO;
329         bt->dropped_file = debugfs_create_file("dropped", 0444, dir, bt, &blk_dropped_fops);
330         if (!bt->dropped_file)
331                 goto err;
332
333         bt->rchan = relay_open("trace", dir, buts.buf_size, buts.buf_nr, &blk_relay_callbacks);
334         if (!bt->rchan)
335                 goto err;
336         bt->rchan->private_data = bt;
337
338         bt->act_mask = buts.act_mask;
339         if (!bt->act_mask)
340                 bt->act_mask = (u16) -1;
341
342         bt->start_lba = buts.start_lba;
343         bt->end_lba = buts.end_lba;
344         if (!bt->end_lba)
345                 bt->end_lba = -1ULL;
346
347         bt->pid = buts.pid;
348         bt->trace_state = Blktrace_setup;
349
350         ret = -EBUSY;
351         old_bt = xchg(&q->blk_trace, bt);
352         if (old_bt) {
353                 (void) xchg(&q->blk_trace, old_bt);
354                 goto err;
355         }
356
357         return 0;
358 err:
359         if (dir)
360                 blk_remove_tree(dir);
361         if (bt) {
362                 if (bt->dropped_file)
363                         debugfs_remove(bt->dropped_file);
364                 if (bt->sequence)
365                         free_percpu(bt->sequence);
366                 if (bt->rchan)
367                         relay_close(bt->rchan);
368                 kfree(bt);
369         }
370         return ret;
371 }
372
373 static int blk_trace_startstop(request_queue_t *q, int start)
374 {
375         struct blk_trace *bt;
376         int ret;
377
378         if ((bt = q->blk_trace) == NULL)
379                 return -EINVAL;
380
381         /*
382          * For starting a trace, we can transition from a setup or stopped
383          * trace. For stopping a trace, the state must be running
384          */
385         ret = -EINVAL;
386         if (start) {
387                 if (bt->trace_state == Blktrace_setup ||
388                     bt->trace_state == Blktrace_stopped) {
389                         blktrace_seq++;
390                         smp_mb();
391                         bt->trace_state = Blktrace_running;
392                         ret = 0;
393                 }
394         } else {
395                 if (bt->trace_state == Blktrace_running) {
396                         bt->trace_state = Blktrace_stopped;
397                         relay_flush(bt->rchan);
398                         ret = 0;
399                 }
400         }
401
402         return ret;
403 }
404
405 /**
406  * blk_trace_ioctl: - handle the ioctls associated with tracing
407  * @bdev:       the block device
408  * @cmd:        the ioctl cmd
409  * @arg:        the argument data, if any
410  *
411  **/
412 int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg)
413 {
414         request_queue_t *q;
415         int ret, start = 0;
416
417         q = bdev_get_queue(bdev);
418         if (!q)
419                 return -ENXIO;
420
421         mutex_lock(&bdev->bd_mutex);
422
423         switch (cmd) {
424         case BLKTRACESETUP:
425                 ret = blk_trace_setup(q, bdev, arg);
426                 break;
427         case BLKTRACESTART:
428                 start = 1;
429         case BLKTRACESTOP:
430                 ret = blk_trace_startstop(q, start);
431                 break;
432         case BLKTRACETEARDOWN:
433                 ret = blk_trace_remove(q);
434                 break;
435         default:
436                 ret = -ENOTTY;
437                 break;
438         }
439
440         mutex_unlock(&bdev->bd_mutex);
441         return ret;
442 }
443
444 /**
445  * blk_trace_shutdown: - stop and cleanup trace structures
446  * @q:    the request queue associated with the device
447  *
448  **/
449 void blk_trace_shutdown(request_queue_t *q)
450 {
451         blk_trace_startstop(q, 0);
452         blk_trace_remove(q);
453 }
454
455 /*
456  * Average offset over two calls to sched_clock() with a gettimeofday()
457  * in the middle
458  */
459 static void blk_check_time(unsigned long long *t)
460 {
461         unsigned long long a, b;
462         struct timeval tv;
463
464         a = sched_clock();
465         do_gettimeofday(&tv);
466         b = sched_clock();
467
468         *t = tv.tv_sec * 1000000000 + tv.tv_usec * 1000;
469         *t -= (a + b) / 2;
470 }
471
472 static void blk_trace_check_cpu_time(void *data)
473 {
474         unsigned long long *t;
475         int cpu = get_cpu();
476
477         t = &per_cpu(blk_trace_cpu_offset, cpu);
478
479         /*
480          * Just call it twice, hopefully the second call will be cache hot
481          * and a little more precise
482          */
483         blk_check_time(t);
484         blk_check_time(t);
485
486         put_cpu();
487 }
488
489 /*
490  * Call blk_trace_check_cpu_time() on each CPU to calibrate our inter-CPU
491  * timings
492  */
493 static void blk_trace_calibrate_offsets(void)
494 {
495         unsigned long flags;
496
497         smp_call_function(blk_trace_check_cpu_time, NULL, 1, 1);
498         local_irq_save(flags);
499         blk_trace_check_cpu_time(NULL);
500         local_irq_restore(flags);
501 }
502
503 static void blk_trace_set_ht_offsets(void)
504 {
505 #if defined(CONFIG_SCHED_SMT)
506         int cpu, i;
507
508         /*
509          * now make sure HT siblings have the same time offset
510          */
511         preempt_disable();
512         for_each_online_cpu(cpu) {
513                 unsigned long long *cpu_off, *sibling_off;
514
515                 for_each_cpu_mask(i, cpu_sibling_map[cpu]) {
516                         if (i == cpu)
517                                 continue;
518
519                         cpu_off = &per_cpu(blk_trace_cpu_offset, cpu);
520                         sibling_off = &per_cpu(blk_trace_cpu_offset, i);
521                         *sibling_off = *cpu_off;
522                 }
523         }
524         preempt_enable();
525 #endif
526 }
527
528 static __init int blk_trace_init(void)
529 {
530         mutex_init(&blk_tree_mutex);
531         blk_trace_calibrate_offsets();
532         blk_trace_set_ht_offsets();
533
534         return 0;
535 }
536
537 module_init(blk_trace_init);
538