tracing: remove extra latency_trace method from trace structure
[linux-3.10.git] / kernel / trace / blktrace.c
1 /*
2  * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk>
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11  * GNU General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public License
14  * along with this program; if not, write to the Free Software
15  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
16  *
17  */
18 #include <linux/kernel.h>
19 #include <linux/blkdev.h>
20 #include <linux/blktrace_api.h>
21 #include <linux/percpu.h>
22 #include <linux/init.h>
23 #include <linux/mutex.h>
24 #include <linux/debugfs.h>
25 #include <linux/time.h>
26 #include <trace/block.h>
27 #include <linux/uaccess.h>
28 #include "trace_output.h"
29
30 static unsigned int blktrace_seq __read_mostly = 1;
31
32 static struct trace_array *blk_tr;
33 static int __read_mostly  blk_tracer_enabled;
34
35 /* Select an alternative, minimalistic output than the original one */
36 #define TRACE_BLK_OPT_CLASSIC   0x1
37
38 static struct tracer_opt blk_tracer_opts[] = {
39         /* Default disable the minimalistic output */
40         { TRACER_OPT(blk_classic, TRACE_BLK_OPT_CLASSIC) },
41         { }
42 };
43
44 static struct tracer_flags blk_tracer_flags = {
45         .val  = 0,
46         .opts = blk_tracer_opts,
47 };
48
49 /* Global reference count of probes */
50 static DEFINE_MUTEX(blk_probe_mutex);
51 static atomic_t blk_probes_ref = ATOMIC_INIT(0);
52
53 static int blk_register_tracepoints(void);
54 static void blk_unregister_tracepoints(void);
55
56 /*
57  * Send out a notify message.
58  */
59 static void trace_note(struct blk_trace *bt, pid_t pid, int action,
60                        const void *data, size_t len)
61 {
62         struct blk_io_trace *t;
63
64         if (!bt->rchan)
65                 return;
66
67         t = relay_reserve(bt->rchan, sizeof(*t) + len);
68         if (t) {
69                 const int cpu = smp_processor_id();
70
71                 t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
72                 t->time = ktime_to_ns(ktime_get());
73                 t->device = bt->dev;
74                 t->action = action;
75                 t->pid = pid;
76                 t->cpu = cpu;
77                 t->pdu_len = len;
78                 memcpy((void *) t + sizeof(*t), data, len);
79         }
80 }
81
82 /*
83  * Send out a notify for this process, if we haven't done so since a trace
84  * started
85  */
86 static void trace_note_tsk(struct blk_trace *bt, struct task_struct *tsk)
87 {
88         tsk->btrace_seq = blktrace_seq;
89         trace_note(bt, tsk->pid, BLK_TN_PROCESS, tsk->comm, sizeof(tsk->comm));
90 }
91
92 static void trace_note_time(struct blk_trace *bt)
93 {
94         struct timespec now;
95         unsigned long flags;
96         u32 words[2];
97
98         getnstimeofday(&now);
99         words[0] = now.tv_sec;
100         words[1] = now.tv_nsec;
101
102         local_irq_save(flags);
103         trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words));
104         local_irq_restore(flags);
105 }
106
107 void __trace_note_message(struct blk_trace *bt, const char *fmt, ...)
108 {
109         int n;
110         va_list args;
111         unsigned long flags;
112         char *buf;
113
114         if (blk_tr) {
115                 va_start(args, fmt);
116                 ftrace_vprintk(fmt, args);
117                 va_end(args);
118                 return;
119         }
120
121         if (!bt->msg_data)
122                 return;
123
124         local_irq_save(flags);
125         buf = per_cpu_ptr(bt->msg_data, smp_processor_id());
126         va_start(args, fmt);
127         n = vscnprintf(buf, BLK_TN_MAX_MSG, fmt, args);
128         va_end(args);
129
130         trace_note(bt, 0, BLK_TN_MESSAGE, buf, n);
131         local_irq_restore(flags);
132 }
133 EXPORT_SYMBOL_GPL(__trace_note_message);
134
135 static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector,
136                          pid_t pid)
137 {
138         if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0)
139                 return 1;
140         if (sector < bt->start_lba || sector > bt->end_lba)
141                 return 1;
142         if (bt->pid && pid != bt->pid)
143                 return 1;
144
145         return 0;
146 }
147
148 /*
149  * Data direction bit lookup
150  */
151 static u32 ddir_act[2] __read_mostly = { BLK_TC_ACT(BLK_TC_READ),
152                                          BLK_TC_ACT(BLK_TC_WRITE) };
153
154 /* The ilog2() calls fall out because they're constant */
155 #define MASK_TC_BIT(rw, __name) ((rw & (1 << BIO_RW_ ## __name)) << \
156           (ilog2(BLK_TC_ ## __name) + BLK_TC_SHIFT - BIO_RW_ ## __name))
157
158 /*
159  * The worker for the various blk_add_trace*() types. Fills out a
160  * blk_io_trace structure and places it in a per-cpu subbuffer.
161  */
162 static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,
163                      int rw, u32 what, int error, int pdu_len, void *pdu_data)
164 {
165         struct task_struct *tsk = current;
166         struct ring_buffer_event *event = NULL;
167         struct blk_io_trace *t;
168         unsigned long flags = 0;
169         unsigned long *sequence;
170         pid_t pid;
171         int cpu, pc = 0;
172
173         if (unlikely(bt->trace_state != Blktrace_running ||
174                      !blk_tracer_enabled))
175                 return;
176
177         what |= ddir_act[rw & WRITE];
178         what |= MASK_TC_BIT(rw, BARRIER);
179         what |= MASK_TC_BIT(rw, SYNCIO);
180         what |= MASK_TC_BIT(rw, AHEAD);
181         what |= MASK_TC_BIT(rw, META);
182         what |= MASK_TC_BIT(rw, DISCARD);
183
184         pid = tsk->pid;
185         if (unlikely(act_log_check(bt, what, sector, pid)))
186                 return;
187         cpu = raw_smp_processor_id();
188
189         if (blk_tr) {
190                 tracing_record_cmdline(current);
191
192                 pc = preempt_count();
193                 event = trace_buffer_lock_reserve(blk_tr, TRACE_BLK,
194                                                   sizeof(*t) + pdu_len,
195                                                   0, pc);
196                 if (!event)
197                         return;
198                 t = ring_buffer_event_data(event);
199                 goto record_it;
200         }
201
202         /*
203          * A word about the locking here - we disable interrupts to reserve
204          * some space in the relay per-cpu buffer, to prevent an irq
205          * from coming in and stepping on our toes.
206          */
207         local_irq_save(flags);
208
209         if (unlikely(tsk->btrace_seq != blktrace_seq))
210                 trace_note_tsk(bt, tsk);
211
212         t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len);
213         if (t) {
214                 sequence = per_cpu_ptr(bt->sequence, cpu);
215
216                 t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
217                 t->sequence = ++(*sequence);
218                 t->time = ktime_to_ns(ktime_get());
219 record_it:
220                 /*
221                  * These two are not needed in ftrace as they are in the
222                  * generic trace_entry, filled by tracing_generic_entry_update,
223                  * but for the trace_event->bin() synthesizer benefit we do it
224                  * here too.
225                  */
226                 t->cpu = cpu;
227                 t->pid = pid;
228
229                 t->sector = sector;
230                 t->bytes = bytes;
231                 t->action = what;
232                 t->device = bt->dev;
233                 t->error = error;
234                 t->pdu_len = pdu_len;
235
236                 if (pdu_len)
237                         memcpy((void *) t + sizeof(*t), pdu_data, pdu_len);
238
239                 if (blk_tr) {
240                         trace_buffer_unlock_commit(blk_tr, event, 0, pc);
241                         return;
242                 }
243         }
244
245         local_irq_restore(flags);
246 }
247
248 static struct dentry *blk_tree_root;
249 static DEFINE_MUTEX(blk_tree_mutex);
250
251 static void blk_trace_cleanup(struct blk_trace *bt)
252 {
253         debugfs_remove(bt->msg_file);
254         debugfs_remove(bt->dropped_file);
255         relay_close(bt->rchan);
256         free_percpu(bt->sequence);
257         free_percpu(bt->msg_data);
258         kfree(bt);
259         mutex_lock(&blk_probe_mutex);
260         if (atomic_dec_and_test(&blk_probes_ref))
261                 blk_unregister_tracepoints();
262         mutex_unlock(&blk_probe_mutex);
263 }
264
265 int blk_trace_remove(struct request_queue *q)
266 {
267         struct blk_trace *bt;
268
269         bt = xchg(&q->blk_trace, NULL);
270         if (!bt)
271                 return -EINVAL;
272
273         if (bt->trace_state == Blktrace_setup ||
274             bt->trace_state == Blktrace_stopped)
275                 blk_trace_cleanup(bt);
276
277         return 0;
278 }
279 EXPORT_SYMBOL_GPL(blk_trace_remove);
280
281 static int blk_dropped_open(struct inode *inode, struct file *filp)
282 {
283         filp->private_data = inode->i_private;
284
285         return 0;
286 }
287
288 static ssize_t blk_dropped_read(struct file *filp, char __user *buffer,
289                                 size_t count, loff_t *ppos)
290 {
291         struct blk_trace *bt = filp->private_data;
292         char buf[16];
293
294         snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped));
295
296         return simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf));
297 }
298
299 static const struct file_operations blk_dropped_fops = {
300         .owner =        THIS_MODULE,
301         .open =         blk_dropped_open,
302         .read =         blk_dropped_read,
303 };
304
305 static int blk_msg_open(struct inode *inode, struct file *filp)
306 {
307         filp->private_data = inode->i_private;
308
309         return 0;
310 }
311
312 static ssize_t blk_msg_write(struct file *filp, const char __user *buffer,
313                                 size_t count, loff_t *ppos)
314 {
315         char *msg;
316         struct blk_trace *bt;
317
318         if (count > BLK_TN_MAX_MSG)
319                 return -EINVAL;
320
321         msg = kmalloc(count, GFP_KERNEL);
322         if (msg == NULL)
323                 return -ENOMEM;
324
325         if (copy_from_user(msg, buffer, count)) {
326                 kfree(msg);
327                 return -EFAULT;
328         }
329
330         bt = filp->private_data;
331         __trace_note_message(bt, "%s", msg);
332         kfree(msg);
333
334         return count;
335 }
336
337 static const struct file_operations blk_msg_fops = {
338         .owner =        THIS_MODULE,
339         .open =         blk_msg_open,
340         .write =        blk_msg_write,
341 };
342
343 /*
344  * Keep track of how many times we encountered a full subbuffer, to aid
345  * the user space app in telling how many lost events there were.
346  */
347 static int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf,
348                                      void *prev_subbuf, size_t prev_padding)
349 {
350         struct blk_trace *bt;
351
352         if (!relay_buf_full(buf))
353                 return 1;
354
355         bt = buf->chan->private_data;
356         atomic_inc(&bt->dropped);
357         return 0;
358 }
359
360 static int blk_remove_buf_file_callback(struct dentry *dentry)
361 {
362         struct dentry *parent = dentry->d_parent;
363         debugfs_remove(dentry);
364
365         /*
366         * this will fail for all but the last file, but that is ok. what we
367         * care about is the top level buts->name directory going away, when
368         * the last trace file is gone. Then we don't have to rmdir() that
369         * manually on trace stop, so it nicely solves the issue with
370         * force killing of running traces.
371         */
372
373         debugfs_remove(parent);
374         return 0;
375 }
376
377 static struct dentry *blk_create_buf_file_callback(const char *filename,
378                                                    struct dentry *parent,
379                                                    int mode,
380                                                    struct rchan_buf *buf,
381                                                    int *is_global)
382 {
383         return debugfs_create_file(filename, mode, parent, buf,
384                                         &relay_file_operations);
385 }
386
387 static struct rchan_callbacks blk_relay_callbacks = {
388         .subbuf_start           = blk_subbuf_start_callback,
389         .create_buf_file        = blk_create_buf_file_callback,
390         .remove_buf_file        = blk_remove_buf_file_callback,
391 };
392
393 /*
394  * Setup everything required to start tracing
395  */
396 int do_blk_trace_setup(struct request_queue *q, char *name, dev_t dev,
397                         struct blk_user_trace_setup *buts)
398 {
399         struct blk_trace *old_bt, *bt = NULL;
400         struct dentry *dir = NULL;
401         int ret, i;
402
403         if (!buts->buf_size || !buts->buf_nr)
404                 return -EINVAL;
405
406         strncpy(buts->name, name, BLKTRACE_BDEV_SIZE);
407         buts->name[BLKTRACE_BDEV_SIZE - 1] = '\0';
408
409         /*
410          * some device names have larger paths - convert the slashes
411          * to underscores for this to work as expected
412          */
413         for (i = 0; i < strlen(buts->name); i++)
414                 if (buts->name[i] == '/')
415                         buts->name[i] = '_';
416
417         ret = -ENOMEM;
418         bt = kzalloc(sizeof(*bt), GFP_KERNEL);
419         if (!bt)
420                 goto err;
421
422         bt->sequence = alloc_percpu(unsigned long);
423         if (!bt->sequence)
424                 goto err;
425
426         bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG);
427         if (!bt->msg_data)
428                 goto err;
429
430         ret = -ENOENT;
431
432         if (!blk_tree_root) {
433                 blk_tree_root = debugfs_create_dir("block", NULL);
434                 if (!blk_tree_root)
435                         return -ENOMEM;
436         }
437
438         dir = debugfs_create_dir(buts->name, blk_tree_root);
439
440         if (!dir)
441                 goto err;
442
443         bt->dir = dir;
444         bt->dev = dev;
445         atomic_set(&bt->dropped, 0);
446
447         ret = -EIO;
448         bt->dropped_file = debugfs_create_file("dropped", 0444, dir, bt,
449                                                &blk_dropped_fops);
450         if (!bt->dropped_file)
451                 goto err;
452
453         bt->msg_file = debugfs_create_file("msg", 0222, dir, bt, &blk_msg_fops);
454         if (!bt->msg_file)
455                 goto err;
456
457         bt->rchan = relay_open("trace", dir, buts->buf_size,
458                                 buts->buf_nr, &blk_relay_callbacks, bt);
459         if (!bt->rchan)
460                 goto err;
461
462         bt->act_mask = buts->act_mask;
463         if (!bt->act_mask)
464                 bt->act_mask = (u16) -1;
465
466         bt->start_lba = buts->start_lba;
467         bt->end_lba = buts->end_lba;
468         if (!bt->end_lba)
469                 bt->end_lba = -1ULL;
470
471         bt->pid = buts->pid;
472         bt->trace_state = Blktrace_setup;
473
474         mutex_lock(&blk_probe_mutex);
475         if (atomic_add_return(1, &blk_probes_ref) == 1) {
476                 ret = blk_register_tracepoints();
477                 if (ret)
478                         goto probe_err;
479         }
480         mutex_unlock(&blk_probe_mutex);
481
482         ret = -EBUSY;
483         old_bt = xchg(&q->blk_trace, bt);
484         if (old_bt) {
485                 (void) xchg(&q->blk_trace, old_bt);
486                 goto err;
487         }
488
489         return 0;
490 probe_err:
491         atomic_dec(&blk_probes_ref);
492         mutex_unlock(&blk_probe_mutex);
493 err:
494         if (bt) {
495                 if (bt->msg_file)
496                         debugfs_remove(bt->msg_file);
497                 if (bt->dropped_file)
498                         debugfs_remove(bt->dropped_file);
499                 free_percpu(bt->sequence);
500                 free_percpu(bt->msg_data);
501                 if (bt->rchan)
502                         relay_close(bt->rchan);
503                 kfree(bt);
504         }
505         return ret;
506 }
507
508 int blk_trace_setup(struct request_queue *q, char *name, dev_t dev,
509                     char __user *arg)
510 {
511         struct blk_user_trace_setup buts;
512         int ret;
513
514         ret = copy_from_user(&buts, arg, sizeof(buts));
515         if (ret)
516                 return -EFAULT;
517
518         ret = do_blk_trace_setup(q, name, dev, &buts);
519         if (ret)
520                 return ret;
521
522         if (copy_to_user(arg, &buts, sizeof(buts)))
523                 return -EFAULT;
524
525         return 0;
526 }
527 EXPORT_SYMBOL_GPL(blk_trace_setup);
528
529 int blk_trace_startstop(struct request_queue *q, int start)
530 {
531         int ret;
532         struct blk_trace *bt = q->blk_trace;
533
534         if (bt == NULL)
535                 return -EINVAL;
536
537         /*
538          * For starting a trace, we can transition from a setup or stopped
539          * trace. For stopping a trace, the state must be running
540          */
541         ret = -EINVAL;
542         if (start) {
543                 if (bt->trace_state == Blktrace_setup ||
544                     bt->trace_state == Blktrace_stopped) {
545                         blktrace_seq++;
546                         smp_mb();
547                         bt->trace_state = Blktrace_running;
548
549                         trace_note_time(bt);
550                         ret = 0;
551                 }
552         } else {
553                 if (bt->trace_state == Blktrace_running) {
554                         bt->trace_state = Blktrace_stopped;
555                         relay_flush(bt->rchan);
556                         ret = 0;
557                 }
558         }
559
560         return ret;
561 }
562 EXPORT_SYMBOL_GPL(blk_trace_startstop);
563
564 /**
565  * blk_trace_ioctl: - handle the ioctls associated with tracing
566  * @bdev:       the block device
567  * @cmd:        the ioctl cmd
568  * @arg:        the argument data, if any
569  *
570  **/
571 int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg)
572 {
573         struct request_queue *q;
574         int ret, start = 0;
575         char b[BDEVNAME_SIZE];
576
577         q = bdev_get_queue(bdev);
578         if (!q)
579                 return -ENXIO;
580
581         mutex_lock(&bdev->bd_mutex);
582
583         switch (cmd) {
584         case BLKTRACESETUP:
585                 bdevname(bdev, b);
586                 ret = blk_trace_setup(q, b, bdev->bd_dev, arg);
587                 break;
588         case BLKTRACESTART:
589                 start = 1;
590         case BLKTRACESTOP:
591                 ret = blk_trace_startstop(q, start);
592                 break;
593         case BLKTRACETEARDOWN:
594                 ret = blk_trace_remove(q);
595                 break;
596         default:
597                 ret = -ENOTTY;
598                 break;
599         }
600
601         mutex_unlock(&bdev->bd_mutex);
602         return ret;
603 }
604
605 /**
606  * blk_trace_shutdown: - stop and cleanup trace structures
607  * @q:    the request queue associated with the device
608  *
609  **/
610 void blk_trace_shutdown(struct request_queue *q)
611 {
612         if (q->blk_trace) {
613                 blk_trace_startstop(q, 0);
614                 blk_trace_remove(q);
615         }
616 }
617
618 /*
619  * blktrace probes
620  */
621
622 /**
623  * blk_add_trace_rq - Add a trace for a request oriented action
624  * @q:          queue the io is for
625  * @rq:         the source request
626  * @what:       the action
627  *
628  * Description:
629  *     Records an action against a request. Will log the bio offset + size.
630  *
631  **/
632 static void blk_add_trace_rq(struct request_queue *q, struct request *rq,
633                                     u32 what)
634 {
635         struct blk_trace *bt = q->blk_trace;
636         int rw = rq->cmd_flags & 0x03;
637
638         if (likely(!bt))
639                 return;
640
641         if (blk_discard_rq(rq))
642                 rw |= (1 << BIO_RW_DISCARD);
643
644         if (blk_pc_request(rq)) {
645                 what |= BLK_TC_ACT(BLK_TC_PC);
646                 __blk_add_trace(bt, 0, rq->data_len, rw, what, rq->errors,
647                                 sizeof(rq->cmd), rq->cmd);
648         } else  {
649                 what |= BLK_TC_ACT(BLK_TC_FS);
650                 __blk_add_trace(bt, rq->hard_sector, rq->hard_nr_sectors << 9,
651                                 rw, what, rq->errors, 0, NULL);
652         }
653 }
654
655 static void blk_add_trace_rq_abort(struct request_queue *q, struct request *rq)
656 {
657         blk_add_trace_rq(q, rq, BLK_TA_ABORT);
658 }
659
660 static void blk_add_trace_rq_insert(struct request_queue *q, struct request *rq)
661 {
662         blk_add_trace_rq(q, rq, BLK_TA_INSERT);
663 }
664
665 static void blk_add_trace_rq_issue(struct request_queue *q, struct request *rq)
666 {
667         blk_add_trace_rq(q, rq, BLK_TA_ISSUE);
668 }
669
670 static void blk_add_trace_rq_requeue(struct request_queue *q,
671                                      struct request *rq)
672 {
673         blk_add_trace_rq(q, rq, BLK_TA_REQUEUE);
674 }
675
676 static void blk_add_trace_rq_complete(struct request_queue *q,
677                                       struct request *rq)
678 {
679         blk_add_trace_rq(q, rq, BLK_TA_COMPLETE);
680 }
681
682 /**
683  * blk_add_trace_bio - Add a trace for a bio oriented action
684  * @q:          queue the io is for
685  * @bio:        the source bio
686  * @what:       the action
687  *
688  * Description:
689  *     Records an action against a bio. Will log the bio offset + size.
690  *
691  **/
692 static void blk_add_trace_bio(struct request_queue *q, struct bio *bio,
693                                      u32 what)
694 {
695         struct blk_trace *bt = q->blk_trace;
696
697         if (likely(!bt))
698                 return;
699
700         __blk_add_trace(bt, bio->bi_sector, bio->bi_size, bio->bi_rw, what,
701                         !bio_flagged(bio, BIO_UPTODATE), 0, NULL);
702 }
703
704 static void blk_add_trace_bio_bounce(struct request_queue *q, struct bio *bio)
705 {
706         blk_add_trace_bio(q, bio, BLK_TA_BOUNCE);
707 }
708
709 static void blk_add_trace_bio_complete(struct request_queue *q, struct bio *bio)
710 {
711         blk_add_trace_bio(q, bio, BLK_TA_COMPLETE);
712 }
713
714 static void blk_add_trace_bio_backmerge(struct request_queue *q,
715                                         struct bio *bio)
716 {
717         blk_add_trace_bio(q, bio, BLK_TA_BACKMERGE);
718 }
719
720 static void blk_add_trace_bio_frontmerge(struct request_queue *q,
721                                          struct bio *bio)
722 {
723         blk_add_trace_bio(q, bio, BLK_TA_FRONTMERGE);
724 }
725
726 static void blk_add_trace_bio_queue(struct request_queue *q, struct bio *bio)
727 {
728         blk_add_trace_bio(q, bio, BLK_TA_QUEUE);
729 }
730
731 static void blk_add_trace_getrq(struct request_queue *q,
732                                 struct bio *bio, int rw)
733 {
734         if (bio)
735                 blk_add_trace_bio(q, bio, BLK_TA_GETRQ);
736         else {
737                 struct blk_trace *bt = q->blk_trace;
738
739                 if (bt)
740                         __blk_add_trace(bt, 0, 0, rw, BLK_TA_GETRQ, 0, 0, NULL);
741         }
742 }
743
744
745 static void blk_add_trace_sleeprq(struct request_queue *q,
746                                   struct bio *bio, int rw)
747 {
748         if (bio)
749                 blk_add_trace_bio(q, bio, BLK_TA_SLEEPRQ);
750         else {
751                 struct blk_trace *bt = q->blk_trace;
752
753                 if (bt)
754                         __blk_add_trace(bt, 0, 0, rw, BLK_TA_SLEEPRQ,
755                                         0, 0, NULL);
756         }
757 }
758
759 static void blk_add_trace_plug(struct request_queue *q)
760 {
761         struct blk_trace *bt = q->blk_trace;
762
763         if (bt)
764                 __blk_add_trace(bt, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL);
765 }
766
767 static void blk_add_trace_unplug_io(struct request_queue *q)
768 {
769         struct blk_trace *bt = q->blk_trace;
770
771         if (bt) {
772                 unsigned int pdu = q->rq.count[READ] + q->rq.count[WRITE];
773                 __be64 rpdu = cpu_to_be64(pdu);
774
775                 __blk_add_trace(bt, 0, 0, 0, BLK_TA_UNPLUG_IO, 0,
776                                 sizeof(rpdu), &rpdu);
777         }
778 }
779
780 static void blk_add_trace_unplug_timer(struct request_queue *q)
781 {
782         struct blk_trace *bt = q->blk_trace;
783
784         if (bt) {
785                 unsigned int pdu = q->rq.count[READ] + q->rq.count[WRITE];
786                 __be64 rpdu = cpu_to_be64(pdu);
787
788                 __blk_add_trace(bt, 0, 0, 0, BLK_TA_UNPLUG_TIMER, 0,
789                                 sizeof(rpdu), &rpdu);
790         }
791 }
792
793 static void blk_add_trace_split(struct request_queue *q, struct bio *bio,
794                                 unsigned int pdu)
795 {
796         struct blk_trace *bt = q->blk_trace;
797
798         if (bt) {
799                 __be64 rpdu = cpu_to_be64(pdu);
800
801                 __blk_add_trace(bt, bio->bi_sector, bio->bi_size, bio->bi_rw,
802                                 BLK_TA_SPLIT, !bio_flagged(bio, BIO_UPTODATE),
803                                 sizeof(rpdu), &rpdu);
804         }
805 }
806
807 /**
808  * blk_add_trace_remap - Add a trace for a remap operation
809  * @q:          queue the io is for
810  * @bio:        the source bio
811  * @dev:        target device
812  * @from:       source sector
813  * @to:         target sector
814  *
815  * Description:
816  *     Device mapper or raid target sometimes need to split a bio because
817  *     it spans a stripe (or similar). Add a trace for that action.
818  *
819  **/
820 static void blk_add_trace_remap(struct request_queue *q, struct bio *bio,
821                                        dev_t dev, sector_t from, sector_t to)
822 {
823         struct blk_trace *bt = q->blk_trace;
824         struct blk_io_trace_remap r;
825
826         if (likely(!bt))
827                 return;
828
829         r.device = cpu_to_be32(dev);
830         r.device_from = cpu_to_be32(bio->bi_bdev->bd_dev);
831         r.sector = cpu_to_be64(to);
832
833         __blk_add_trace(bt, from, bio->bi_size, bio->bi_rw, BLK_TA_REMAP,
834                         !bio_flagged(bio, BIO_UPTODATE), sizeof(r), &r);
835 }
836
837 /**
838  * blk_add_driver_data - Add binary message with driver-specific data
839  * @q:          queue the io is for
840  * @rq:         io request
841  * @data:       driver-specific data
842  * @len:        length of driver-specific data
843  *
844  * Description:
845  *     Some drivers might want to write driver-specific data per request.
846  *
847  **/
848 void blk_add_driver_data(struct request_queue *q,
849                          struct request *rq,
850                          void *data, size_t len)
851 {
852         struct blk_trace *bt = q->blk_trace;
853
854         if (likely(!bt))
855                 return;
856
857         if (blk_pc_request(rq))
858                 __blk_add_trace(bt, 0, rq->data_len, 0, BLK_TA_DRV_DATA,
859                                 rq->errors, len, data);
860         else
861                 __blk_add_trace(bt, rq->hard_sector, rq->hard_nr_sectors << 9,
862                                 0, BLK_TA_DRV_DATA, rq->errors, len, data);
863 }
864 EXPORT_SYMBOL_GPL(blk_add_driver_data);
865
866 static int blk_register_tracepoints(void)
867 {
868         int ret;
869
870         ret = register_trace_block_rq_abort(blk_add_trace_rq_abort);
871         WARN_ON(ret);
872         ret = register_trace_block_rq_insert(blk_add_trace_rq_insert);
873         WARN_ON(ret);
874         ret = register_trace_block_rq_issue(blk_add_trace_rq_issue);
875         WARN_ON(ret);
876         ret = register_trace_block_rq_requeue(blk_add_trace_rq_requeue);
877         WARN_ON(ret);
878         ret = register_trace_block_rq_complete(blk_add_trace_rq_complete);
879         WARN_ON(ret);
880         ret = register_trace_block_bio_bounce(blk_add_trace_bio_bounce);
881         WARN_ON(ret);
882         ret = register_trace_block_bio_complete(blk_add_trace_bio_complete);
883         WARN_ON(ret);
884         ret = register_trace_block_bio_backmerge(blk_add_trace_bio_backmerge);
885         WARN_ON(ret);
886         ret = register_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge);
887         WARN_ON(ret);
888         ret = register_trace_block_bio_queue(blk_add_trace_bio_queue);
889         WARN_ON(ret);
890         ret = register_trace_block_getrq(blk_add_trace_getrq);
891         WARN_ON(ret);
892         ret = register_trace_block_sleeprq(blk_add_trace_sleeprq);
893         WARN_ON(ret);
894         ret = register_trace_block_plug(blk_add_trace_plug);
895         WARN_ON(ret);
896         ret = register_trace_block_unplug_timer(blk_add_trace_unplug_timer);
897         WARN_ON(ret);
898         ret = register_trace_block_unplug_io(blk_add_trace_unplug_io);
899         WARN_ON(ret);
900         ret = register_trace_block_split(blk_add_trace_split);
901         WARN_ON(ret);
902         ret = register_trace_block_remap(blk_add_trace_remap);
903         WARN_ON(ret);
904         return 0;
905 }
906
907 static void blk_unregister_tracepoints(void)
908 {
909         unregister_trace_block_remap(blk_add_trace_remap);
910         unregister_trace_block_split(blk_add_trace_split);
911         unregister_trace_block_unplug_io(blk_add_trace_unplug_io);
912         unregister_trace_block_unplug_timer(blk_add_trace_unplug_timer);
913         unregister_trace_block_plug(blk_add_trace_plug);
914         unregister_trace_block_sleeprq(blk_add_trace_sleeprq);
915         unregister_trace_block_getrq(blk_add_trace_getrq);
916         unregister_trace_block_bio_queue(blk_add_trace_bio_queue);
917         unregister_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge);
918         unregister_trace_block_bio_backmerge(blk_add_trace_bio_backmerge);
919         unregister_trace_block_bio_complete(blk_add_trace_bio_complete);
920         unregister_trace_block_bio_bounce(blk_add_trace_bio_bounce);
921         unregister_trace_block_rq_complete(blk_add_trace_rq_complete);
922         unregister_trace_block_rq_requeue(blk_add_trace_rq_requeue);
923         unregister_trace_block_rq_issue(blk_add_trace_rq_issue);
924         unregister_trace_block_rq_insert(blk_add_trace_rq_insert);
925         unregister_trace_block_rq_abort(blk_add_trace_rq_abort);
926
927         tracepoint_synchronize_unregister();
928 }
929
930 /*
931  * struct blk_io_tracer formatting routines
932  */
933
934 static void fill_rwbs(char *rwbs, const struct blk_io_trace *t)
935 {
936         int i = 0;
937
938         if (t->action & BLK_TC_DISCARD)
939                 rwbs[i++] = 'D';
940         else if (t->action & BLK_TC_WRITE)
941                 rwbs[i++] = 'W';
942         else if (t->bytes)
943                 rwbs[i++] = 'R';
944         else
945                 rwbs[i++] = 'N';
946
947         if (t->action & BLK_TC_AHEAD)
948                 rwbs[i++] = 'A';
949         if (t->action & BLK_TC_BARRIER)
950                 rwbs[i++] = 'B';
951         if (t->action & BLK_TC_SYNC)
952                 rwbs[i++] = 'S';
953         if (t->action & BLK_TC_META)
954                 rwbs[i++] = 'M';
955
956         rwbs[i] = '\0';
957 }
958
959 static inline
960 const struct blk_io_trace *te_blk_io_trace(const struct trace_entry *ent)
961 {
962         return (const struct blk_io_trace *)ent;
963 }
964
965 static inline const void *pdu_start(const struct trace_entry *ent)
966 {
967         return te_blk_io_trace(ent) + 1;
968 }
969
970 static inline u32 t_sec(const struct trace_entry *ent)
971 {
972         return te_blk_io_trace(ent)->bytes >> 9;
973 }
974
975 static inline unsigned long long t_sector(const struct trace_entry *ent)
976 {
977         return te_blk_io_trace(ent)->sector;
978 }
979
980 static inline __u16 t_error(const struct trace_entry *ent)
981 {
982         return te_blk_io_trace(ent)->sector;
983 }
984
985 static __u64 get_pdu_int(const struct trace_entry *ent)
986 {
987         const __u64 *val = pdu_start(ent);
988         return be64_to_cpu(*val);
989 }
990
991 static void get_pdu_remap(const struct trace_entry *ent,
992                           struct blk_io_trace_remap *r)
993 {
994         const struct blk_io_trace_remap *__r = pdu_start(ent);
995         __u64 sector = __r->sector;
996
997         r->device = be32_to_cpu(__r->device);
998         r->device_from = be32_to_cpu(__r->device_from);
999         r->sector = be64_to_cpu(sector);
1000 }
1001
1002 static int blk_log_action_iter(struct trace_iterator *iter, const char *act)
1003 {
1004         char rwbs[6];
1005         unsigned long long ts  = ns2usecs(iter->ts);
1006         unsigned long usec_rem = do_div(ts, USEC_PER_SEC);
1007         unsigned secs          = (unsigned long)ts;
1008         const struct trace_entry *ent = iter->ent;
1009         const struct blk_io_trace *t = (const struct blk_io_trace *)ent;
1010
1011         fill_rwbs(rwbs, t);
1012
1013         return trace_seq_printf(&iter->seq,
1014                                 "%3d,%-3d %2d %5d.%06lu %5u %2s %3s ",
1015                                 MAJOR(t->device), MINOR(t->device), iter->cpu,
1016                                 secs, usec_rem, ent->pid, act, rwbs);
1017 }
1018
1019 static int blk_log_action_seq(struct trace_seq *s, const struct blk_io_trace *t,
1020                               const char *act)
1021 {
1022         char rwbs[6];
1023         fill_rwbs(rwbs, t);
1024         return trace_seq_printf(s, "%3d,%-3d %2s %3s ",
1025                                 MAJOR(t->device), MINOR(t->device), act, rwbs);
1026 }
1027
1028 static int blk_log_generic(struct trace_seq *s, const struct trace_entry *ent)
1029 {
1030         const char *cmd = trace_find_cmdline(ent->pid);
1031
1032         if (t_sec(ent))
1033                 return trace_seq_printf(s, "%llu + %u [%s]\n",
1034                                         t_sector(ent), t_sec(ent), cmd);
1035         return trace_seq_printf(s, "[%s]\n", cmd);
1036 }
1037
1038 static int blk_log_with_error(struct trace_seq *s,
1039                               const struct trace_entry *ent)
1040 {
1041         if (t_sec(ent))
1042                 return trace_seq_printf(s, "%llu + %u [%d]\n", t_sector(ent),
1043                                         t_sec(ent), t_error(ent));
1044         return trace_seq_printf(s, "%llu [%d]\n", t_sector(ent), t_error(ent));
1045 }
1046
1047 static int blk_log_remap(struct trace_seq *s, const struct trace_entry *ent)
1048 {
1049         struct blk_io_trace_remap r = { .device = 0, };
1050
1051         get_pdu_remap(ent, &r);
1052         return trace_seq_printf(s, "%llu + %u <- (%d,%d) %llu\n",
1053                                t_sector(ent),
1054                                t_sec(ent), MAJOR(r.device), MINOR(r.device),
1055                                (unsigned long long)r.sector);
1056 }
1057
1058 static int blk_log_plug(struct trace_seq *s, const struct trace_entry *ent)
1059 {
1060         return trace_seq_printf(s, "[%s]\n", trace_find_cmdline(ent->pid));
1061 }
1062
1063 static int blk_log_unplug(struct trace_seq *s, const struct trace_entry *ent)
1064 {
1065         return trace_seq_printf(s, "[%s] %llu\n", trace_find_cmdline(ent->pid),
1066                                 get_pdu_int(ent));
1067 }
1068
1069 static int blk_log_split(struct trace_seq *s, const struct trace_entry *ent)
1070 {
1071         return trace_seq_printf(s, "%llu / %llu [%s]\n", t_sector(ent),
1072                                 get_pdu_int(ent), trace_find_cmdline(ent->pid));
1073 }
1074
1075 /*
1076  * struct tracer operations
1077  */
1078
1079 static void blk_tracer_print_header(struct seq_file *m)
1080 {
1081         if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC))
1082                 return;
1083         seq_puts(m, "# DEV   CPU TIMESTAMP     PID ACT FLG\n"
1084                     "#  |     |     |           |   |   |\n");
1085 }
1086
1087 static void blk_tracer_start(struct trace_array *tr)
1088 {
1089         mutex_lock(&blk_probe_mutex);
1090         if (atomic_add_return(1, &blk_probes_ref) == 1)
1091                 if (blk_register_tracepoints())
1092                         atomic_dec(&blk_probes_ref);
1093         mutex_unlock(&blk_probe_mutex);
1094         trace_flags &= ~TRACE_ITER_CONTEXT_INFO;
1095 }
1096
1097 static int blk_tracer_init(struct trace_array *tr)
1098 {
1099         blk_tr = tr;
1100         blk_tracer_start(tr);
1101         mutex_lock(&blk_probe_mutex);
1102         blk_tracer_enabled++;
1103         mutex_unlock(&blk_probe_mutex);
1104         return 0;
1105 }
1106
1107 static void blk_tracer_stop(struct trace_array *tr)
1108 {
1109         trace_flags |= TRACE_ITER_CONTEXT_INFO;
1110         mutex_lock(&blk_probe_mutex);
1111         if (atomic_dec_and_test(&blk_probes_ref))
1112                 blk_unregister_tracepoints();
1113         mutex_unlock(&blk_probe_mutex);
1114 }
1115
1116 static void blk_tracer_reset(struct trace_array *tr)
1117 {
1118         if (!atomic_read(&blk_probes_ref))
1119                 return;
1120
1121         mutex_lock(&blk_probe_mutex);
1122         blk_tracer_enabled--;
1123         WARN_ON(blk_tracer_enabled < 0);
1124         mutex_unlock(&blk_probe_mutex);
1125
1126         blk_tracer_stop(tr);
1127 }
1128
1129 static struct {
1130         const char *act[2];
1131         int        (*print)(struct trace_seq *s, const struct trace_entry *ent);
1132 } what2act[] __read_mostly = {
1133         [__BLK_TA_QUEUE]        = {{  "Q", "queue" },      blk_log_generic },
1134         [__BLK_TA_BACKMERGE]    = {{  "M", "backmerge" },  blk_log_generic },
1135         [__BLK_TA_FRONTMERGE]   = {{  "F", "frontmerge" }, blk_log_generic },
1136         [__BLK_TA_GETRQ]        = {{  "G", "getrq" },      blk_log_generic },
1137         [__BLK_TA_SLEEPRQ]      = {{  "S", "sleeprq" },    blk_log_generic },
1138         [__BLK_TA_REQUEUE]      = {{  "R", "requeue" },    blk_log_with_error },
1139         [__BLK_TA_ISSUE]        = {{  "D", "issue" },      blk_log_generic },
1140         [__BLK_TA_COMPLETE]     = {{  "C", "complete" },   blk_log_with_error },
1141         [__BLK_TA_PLUG]         = {{  "P", "plug" },       blk_log_plug },
1142         [__BLK_TA_UNPLUG_IO]    = {{  "U", "unplug_io" },  blk_log_unplug },
1143         [__BLK_TA_UNPLUG_TIMER] = {{ "UT", "unplug_timer" }, blk_log_unplug },
1144         [__BLK_TA_INSERT]       = {{  "I", "insert" },     blk_log_generic },
1145         [__BLK_TA_SPLIT]        = {{  "X", "split" },      blk_log_split },
1146         [__BLK_TA_BOUNCE]       = {{  "B", "bounce" },     blk_log_generic },
1147         [__BLK_TA_REMAP]        = {{  "A", "remap" },      blk_log_remap },
1148 };
1149
1150 static enum print_line_t blk_trace_event_print(struct trace_iterator *iter,
1151                                                int flags)
1152 {
1153         struct trace_seq *s = &iter->seq;
1154         const struct blk_io_trace *t = (struct blk_io_trace *)iter->ent;
1155         const u16 what = t->action & ((1 << BLK_TC_SHIFT) - 1);
1156         int ret;
1157
1158         if (!trace_print_context(iter))
1159                 return TRACE_TYPE_PARTIAL_LINE;
1160
1161         if (unlikely(what == 0 || what > ARRAY_SIZE(what2act)))
1162                 ret = trace_seq_printf(s, "Bad pc action %x\n", what);
1163         else {
1164                 const bool long_act = !!(trace_flags & TRACE_ITER_VERBOSE);
1165                 ret = blk_log_action_seq(s, t, what2act[what].act[long_act]);
1166                 if (ret)
1167                         ret = what2act[what].print(s, iter->ent);
1168         }
1169
1170         return ret ? TRACE_TYPE_HANDLED : TRACE_TYPE_PARTIAL_LINE;
1171 }
1172
1173 static int blk_trace_synthesize_old_trace(struct trace_iterator *iter)
1174 {
1175         struct trace_seq *s = &iter->seq;
1176         struct blk_io_trace *t = (struct blk_io_trace *)iter->ent;
1177         const int offset = offsetof(struct blk_io_trace, sector);
1178         struct blk_io_trace old = {
1179                 .magic    = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION,
1180                 .time     = ns2usecs(iter->ts),
1181         };
1182
1183         if (!trace_seq_putmem(s, &old, offset))
1184                 return 0;
1185         return trace_seq_putmem(s, &t->sector,
1186                                 sizeof(old) - offset + t->pdu_len);
1187 }
1188
1189 static enum print_line_t
1190 blk_trace_event_print_binary(struct trace_iterator *iter, int flags)
1191 {
1192         return blk_trace_synthesize_old_trace(iter) ?
1193                         TRACE_TYPE_HANDLED : TRACE_TYPE_PARTIAL_LINE;
1194 }
1195
1196 static enum print_line_t blk_tracer_print_line(struct trace_iterator *iter)
1197 {
1198         const struct blk_io_trace *t;
1199         u16 what;
1200         int ret;
1201
1202         if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC))
1203                 return TRACE_TYPE_UNHANDLED;
1204
1205         t = (const struct blk_io_trace *)iter->ent;
1206         what = t->action & ((1 << BLK_TC_SHIFT) - 1);
1207
1208         if (unlikely(what == 0 || what > ARRAY_SIZE(what2act)))
1209                 ret = trace_seq_printf(&iter->seq, "Bad pc action %x\n", what);
1210         else {
1211                 const bool long_act = !!(trace_flags & TRACE_ITER_VERBOSE);
1212                 ret = blk_log_action_iter(iter, what2act[what].act[long_act]);
1213                 if (ret)
1214                         ret = what2act[what].print(&iter->seq, iter->ent);
1215         }
1216
1217         return ret ? TRACE_TYPE_HANDLED : TRACE_TYPE_PARTIAL_LINE;
1218 }
1219
1220 static struct tracer blk_tracer __read_mostly = {
1221         .name           = "blk",
1222         .init           = blk_tracer_init,
1223         .reset          = blk_tracer_reset,
1224         .start          = blk_tracer_start,
1225         .stop           = blk_tracer_stop,
1226         .print_header   = blk_tracer_print_header,
1227         .print_line     = blk_tracer_print_line,
1228         .flags          = &blk_tracer_flags,
1229 };
1230
1231 static struct trace_event trace_blk_event = {
1232         .type           = TRACE_BLK,
1233         .trace          = blk_trace_event_print,
1234         .binary         = blk_trace_event_print_binary,
1235 };
1236
1237 static int __init init_blk_tracer(void)
1238 {
1239         if (!register_ftrace_event(&trace_blk_event)) {
1240                 pr_warning("Warning: could not register block events\n");
1241                 return 1;
1242         }
1243
1244         if (register_tracer(&blk_tracer) != 0) {
1245                 pr_warning("Warning: could not register the block tracer\n");
1246                 unregister_ftrace_event(&trace_blk_event);
1247                 return 1;
1248         }
1249
1250         return 0;
1251 }
1252
1253 device_initcall(init_blk_tracer);
1254
1255 static int blk_trace_remove_queue(struct request_queue *q)
1256 {
1257         struct blk_trace *bt;
1258
1259         bt = xchg(&q->blk_trace, NULL);
1260         if (bt == NULL)
1261                 return -EINVAL;
1262
1263         kfree(bt);
1264         return 0;
1265 }
1266
1267 /*
1268  * Setup everything required to start tracing
1269  */
1270 static int blk_trace_setup_queue(struct request_queue *q, dev_t dev)
1271 {
1272         struct blk_trace *old_bt, *bt = NULL;
1273         int ret;
1274
1275         ret = -ENOMEM;
1276         bt = kzalloc(sizeof(*bt), GFP_KERNEL);
1277         if (!bt)
1278                 goto err;
1279
1280         bt->dev = dev;
1281         bt->act_mask = (u16)-1;
1282         bt->end_lba = -1ULL;
1283         bt->trace_state = Blktrace_running;
1284
1285         old_bt = xchg(&q->blk_trace, bt);
1286         if (old_bt != NULL) {
1287                 (void)xchg(&q->blk_trace, old_bt);
1288                 kfree(bt);
1289                 ret = -EBUSY;
1290         }
1291         return 0;
1292 err:
1293         return ret;
1294 }
1295
1296 /*
1297  * sysfs interface to enable and configure tracing
1298  */
1299
1300 static ssize_t sysfs_blk_trace_enable_show(struct device *dev,
1301                                            struct device_attribute *attr,
1302                                            char *buf)
1303 {
1304         struct hd_struct *p = dev_to_part(dev);
1305         struct block_device *bdev;
1306         ssize_t ret = -ENXIO;
1307
1308         lock_kernel();
1309         bdev = bdget(part_devt(p));
1310         if (bdev != NULL) {
1311                 struct request_queue *q = bdev_get_queue(bdev);
1312
1313                 if (q != NULL) {
1314                         mutex_lock(&bdev->bd_mutex);
1315                         ret = sprintf(buf, "%u\n", !!q->blk_trace);
1316                         mutex_unlock(&bdev->bd_mutex);
1317                 }
1318
1319                 bdput(bdev);
1320         }
1321
1322         unlock_kernel();
1323         return ret;
1324 }
1325
1326 static ssize_t sysfs_blk_trace_enable_store(struct device *dev,
1327                                             struct device_attribute *attr,
1328                                             const char *buf, size_t count)
1329 {
1330         struct block_device *bdev;
1331         struct request_queue *q;
1332         struct hd_struct *p;
1333         int value;
1334         ssize_t ret = -ENXIO;
1335
1336         if (count == 0 || sscanf(buf, "%d", &value) != 1)
1337                 goto out;
1338
1339         lock_kernel();
1340         p = dev_to_part(dev);
1341         bdev = bdget(part_devt(p));
1342         if (bdev == NULL)
1343                 goto out_unlock_kernel;
1344
1345         q = bdev_get_queue(bdev);
1346         if (q == NULL)
1347                 goto out_bdput;
1348
1349         mutex_lock(&bdev->bd_mutex);
1350         if (value)
1351                 ret = blk_trace_setup_queue(q, bdev->bd_dev);
1352         else
1353                 ret = blk_trace_remove_queue(q);
1354         mutex_unlock(&bdev->bd_mutex);
1355
1356         if (ret == 0)
1357                 ret = count;
1358 out_bdput:
1359         bdput(bdev);
1360 out_unlock_kernel:
1361         unlock_kernel();
1362 out:
1363         return ret;
1364 }
1365
1366 static ssize_t sysfs_blk_trace_attr_show(struct device *dev,
1367                                          struct device_attribute *attr,
1368                                          char *buf);
1369 static ssize_t sysfs_blk_trace_attr_store(struct device *dev,
1370                                           struct device_attribute *attr,
1371                                           const char *buf, size_t count);
1372 #define BLK_TRACE_DEVICE_ATTR(_name) \
1373         DEVICE_ATTR(_name, S_IRUGO | S_IWUSR, \
1374                     sysfs_blk_trace_attr_show, \
1375                     sysfs_blk_trace_attr_store)
1376
1377 static DEVICE_ATTR(enable, S_IRUGO | S_IWUSR,
1378                    sysfs_blk_trace_enable_show, sysfs_blk_trace_enable_store);
1379 static BLK_TRACE_DEVICE_ATTR(act_mask);
1380 static BLK_TRACE_DEVICE_ATTR(pid);
1381 static BLK_TRACE_DEVICE_ATTR(start_lba);
1382 static BLK_TRACE_DEVICE_ATTR(end_lba);
1383
1384 static struct attribute *blk_trace_attrs[] = {
1385         &dev_attr_enable.attr,
1386         &dev_attr_act_mask.attr,
1387         &dev_attr_pid.attr,
1388         &dev_attr_start_lba.attr,
1389         &dev_attr_end_lba.attr,
1390         NULL
1391 };
1392
1393 struct attribute_group blk_trace_attr_group = {
1394         .name  = "trace",
1395         .attrs = blk_trace_attrs,
1396 };
1397
1398 static int blk_str2act_mask(const char *str)
1399 {
1400         int mask = 0;
1401         char *copy = kstrdup(str, GFP_KERNEL), *s;
1402
1403         if (copy == NULL)
1404                 return -ENOMEM;
1405
1406         s = strstrip(copy);
1407
1408         while (1) {
1409                 char *sep = strchr(s, ',');
1410
1411                 if (sep != NULL)
1412                         *sep = '\0';
1413
1414                 if (strcasecmp(s, "barrier") == 0)
1415                         mask |= BLK_TC_BARRIER;
1416                 else if (strcasecmp(s, "complete") == 0)
1417                         mask |= BLK_TC_COMPLETE;
1418                 else if (strcasecmp(s, "fs") == 0)
1419                         mask |= BLK_TC_FS;
1420                 else if (strcasecmp(s, "issue") == 0)
1421                         mask |= BLK_TC_ISSUE;
1422                 else if (strcasecmp(s, "pc") == 0)
1423                         mask |= BLK_TC_PC;
1424                 else if (strcasecmp(s, "queue") == 0)
1425                         mask |= BLK_TC_QUEUE;
1426                 else if (strcasecmp(s, "read") == 0)
1427                         mask |= BLK_TC_READ;
1428                 else if (strcasecmp(s, "requeue") == 0)
1429                         mask |= BLK_TC_REQUEUE;
1430                 else if (strcasecmp(s, "sync") == 0)
1431                         mask |= BLK_TC_SYNC;
1432                 else if (strcasecmp(s, "write") == 0)
1433                         mask |= BLK_TC_WRITE;
1434
1435                 if (sep == NULL)
1436                         break;
1437
1438                 s = sep + 1;
1439         }
1440         kfree(copy);
1441
1442         return mask;
1443 }
1444
1445 static ssize_t sysfs_blk_trace_attr_show(struct device *dev,
1446                                          struct device_attribute *attr,
1447                                          char *buf)
1448 {
1449         struct hd_struct *p = dev_to_part(dev);
1450         struct request_queue *q;
1451         struct block_device *bdev;
1452         ssize_t ret = -ENXIO;
1453
1454         lock_kernel();
1455         bdev = bdget(part_devt(p));
1456         if (bdev == NULL)
1457                 goto out_unlock_kernel;
1458
1459         q = bdev_get_queue(bdev);
1460         if (q == NULL)
1461                 goto out_bdput;
1462         mutex_lock(&bdev->bd_mutex);
1463         if (q->blk_trace == NULL)
1464                 ret = sprintf(buf, "disabled\n");
1465         else if (attr == &dev_attr_act_mask)
1466                 ret = sprintf(buf, "%#x\n", q->blk_trace->act_mask);
1467         else if (attr == &dev_attr_pid)
1468                 ret = sprintf(buf, "%u\n", q->blk_trace->pid);
1469         else if (attr == &dev_attr_start_lba)
1470                 ret = sprintf(buf, "%llu\n", q->blk_trace->start_lba);
1471         else if (attr == &dev_attr_end_lba)
1472                 ret = sprintf(buf, "%llu\n", q->blk_trace->end_lba);
1473         mutex_unlock(&bdev->bd_mutex);
1474 out_bdput:
1475         bdput(bdev);
1476 out_unlock_kernel:
1477         unlock_kernel();
1478         return ret;
1479 }
1480
1481 static ssize_t sysfs_blk_trace_attr_store(struct device *dev,
1482                                           struct device_attribute *attr,
1483                                           const char *buf, size_t count)
1484 {
1485         struct block_device *bdev;
1486         struct request_queue *q;
1487         struct hd_struct *p;
1488         u64 value;
1489         ssize_t ret = -ENXIO;
1490
1491         if (count == 0)
1492                 goto out;
1493
1494         if (attr == &dev_attr_act_mask) {
1495                 if (sscanf(buf, "%llx", &value) != 1) {
1496                         /* Assume it is a list of trace category names */
1497                         value = blk_str2act_mask(buf);
1498                         if (value < 0)
1499                                 goto out;
1500                 }
1501         } else if (sscanf(buf, "%llu", &value) != 1)
1502                 goto out;
1503
1504         lock_kernel();
1505         p = dev_to_part(dev);
1506         bdev = bdget(part_devt(p));
1507         if (bdev == NULL)
1508                 goto out_unlock_kernel;
1509
1510         q = bdev_get_queue(bdev);
1511         if (q == NULL)
1512                 goto out_bdput;
1513
1514         mutex_lock(&bdev->bd_mutex);
1515         ret = 0;
1516         if (q->blk_trace == NULL)
1517                 ret = blk_trace_setup_queue(q, bdev->bd_dev);
1518
1519         if (ret == 0) {
1520                 if (attr == &dev_attr_act_mask)
1521                         q->blk_trace->act_mask = value;
1522                 else if (attr == &dev_attr_pid)
1523                         q->blk_trace->pid = value;
1524                 else if (attr == &dev_attr_start_lba)
1525                         q->blk_trace->start_lba = value;
1526                 else if (attr == &dev_attr_end_lba)
1527                         q->blk_trace->end_lba = value;
1528                 ret = count;
1529         }
1530         mutex_unlock(&bdev->bd_mutex);
1531 out_bdput:
1532         bdput(bdev);
1533 out_unlock_kernel:
1534         unlock_kernel();
1535 out:
1536         return ret;
1537 }