2 * Block device elevator/IO-scheduler.
4 * Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
6 * 30042000 Jens Axboe <axboe@kernel.dk> :
8 * Split the elevator a bit so that it is possible to choose a different
9 * one or even write a new "plug in". There are three pieces:
10 * - elevator_fn, inserts a new request in the queue list
11 * - elevator_merge_fn, decides whether a new buffer can be merged with
13 * - elevator_dequeue_fn, called when a request is taken off the active list
15 * 20082000 Dave Jones <davej@suse.de> :
16 * Removed tests for max-bomb-segments, which was breaking elvtune
17 * when run without -bN
20 * - Rework again to work with bio instead of buffer_heads
21 * - loose bi_dev comparisons, partition handling is right now
22 * - completely modularize elevator setup and teardown
25 #include <linux/kernel.h>
27 #include <linux/blkdev.h>
28 #include <linux/elevator.h>
29 #include <linux/bio.h>
30 #include <linux/module.h>
31 #include <linux/slab.h>
32 #include <linux/init.h>
33 #include <linux/compiler.h>
34 #include <linux/delay.h>
35 #include <linux/blktrace_api.h>
36 #include <linux/hash.h>
38 #include <asm/uaccess.h>
40 static DEFINE_SPINLOCK(elv_list_lock);
41 static LIST_HEAD(elv_list);
46 static const int elv_hash_shift = 6;
47 #define ELV_HASH_BLOCK(sec) ((sec) >> 3)
48 #define ELV_HASH_FN(sec) (hash_long(ELV_HASH_BLOCK((sec)), elv_hash_shift))
49 #define ELV_HASH_ENTRIES (1 << elv_hash_shift)
50 #define rq_hash_key(rq) ((rq)->sector + (rq)->nr_sectors)
51 #define ELV_ON_HASH(rq) (!hlist_unhashed(&(rq)->hash))
54 * Query io scheduler to see if the current process issuing bio may be
57 static int elv_iosched_allow_merge(struct request *rq, struct bio *bio)
59 request_queue_t *q = rq->q;
60 elevator_t *e = q->elevator;
62 if (e->ops->elevator_allow_merge_fn)
63 return e->ops->elevator_allow_merge_fn(q, rq, bio);
69 * can we safely merge with this request?
71 inline int elv_rq_merge_ok(struct request *rq, struct bio *bio)
73 if (!rq_mergeable(rq))
77 * different data direction or already started, don't merge
79 if (bio_data_dir(bio) != rq_data_dir(rq))
83 * must be same device and not a special request
85 if (rq->rq_disk != bio->bi_bdev->bd_disk || rq->special)
88 if (!elv_iosched_allow_merge(rq, bio))
93 EXPORT_SYMBOL(elv_rq_merge_ok);
95 static inline int elv_try_merge(struct request *__rq, struct bio *bio)
97 int ret = ELEVATOR_NO_MERGE;
100 * we can merge and sequence is ok, check if it's possible
102 if (elv_rq_merge_ok(__rq, bio)) {
103 if (__rq->sector + __rq->nr_sectors == bio->bi_sector)
104 ret = ELEVATOR_BACK_MERGE;
105 else if (__rq->sector - bio_sectors(bio) == bio->bi_sector)
106 ret = ELEVATOR_FRONT_MERGE;
112 static struct elevator_type *elevator_find(const char *name)
114 struct elevator_type *e;
115 struct list_head *entry;
117 list_for_each(entry, &elv_list) {
119 e = list_entry(entry, struct elevator_type, list);
121 if (!strcmp(e->elevator_name, name))
128 static void elevator_put(struct elevator_type *e)
130 module_put(e->elevator_owner);
133 static struct elevator_type *elevator_get(const char *name)
135 struct elevator_type *e;
137 spin_lock_irq(&elv_list_lock);
139 e = elevator_find(name);
140 if (e && !try_module_get(e->elevator_owner))
143 spin_unlock_irq(&elv_list_lock);
148 static void *elevator_init_queue(request_queue_t *q, struct elevator_queue *eq)
150 return eq->ops->elevator_init_fn(q);
153 static void elevator_attach(request_queue_t *q, struct elevator_queue *eq,
157 eq->elevator_data = data;
160 static char chosen_elevator[16];
162 static int __init elevator_setup(char *str)
165 * Be backwards-compatible with previous kernels, so users
166 * won't get the wrong elevator.
168 if (!strcmp(str, "as"))
169 strcpy(chosen_elevator, "anticipatory");
171 strncpy(chosen_elevator, str, sizeof(chosen_elevator) - 1);
175 __setup("elevator=", elevator_setup);
177 static struct kobj_type elv_ktype;
179 static elevator_t *elevator_alloc(request_queue_t *q, struct elevator_type *e)
184 eq = kmalloc_node(sizeof(elevator_t), GFP_KERNEL, q->node);
188 memset(eq, 0, sizeof(*eq));
190 eq->elevator_type = e;
191 kobject_init(&eq->kobj);
192 snprintf(eq->kobj.name, KOBJ_NAME_LEN, "%s", "iosched");
193 eq->kobj.ktype = &elv_ktype;
194 mutex_init(&eq->sysfs_lock);
196 eq->hash = kmalloc_node(sizeof(struct hlist_head) * ELV_HASH_ENTRIES,
197 GFP_KERNEL, q->node);
201 for (i = 0; i < ELV_HASH_ENTRIES; i++)
202 INIT_HLIST_HEAD(&eq->hash[i]);
211 static void elevator_release(struct kobject *kobj)
213 elevator_t *e = container_of(kobj, elevator_t, kobj);
215 elevator_put(e->elevator_type);
220 int elevator_init(request_queue_t *q, char *name)
222 struct elevator_type *e = NULL;
223 struct elevator_queue *eq;
227 INIT_LIST_HEAD(&q->queue_head);
228 q->last_merge = NULL;
230 q->boundary_rq = NULL;
232 if (name && !(e = elevator_get(name)))
235 if (!e && *chosen_elevator && !(e = elevator_get(chosen_elevator)))
236 printk("I/O scheduler %s not found\n", chosen_elevator);
238 if (!e && !(e = elevator_get(CONFIG_DEFAULT_IOSCHED))) {
239 printk("Default I/O scheduler not found, using no-op\n");
240 e = elevator_get("noop");
243 eq = elevator_alloc(q, e);
247 data = elevator_init_queue(q, eq);
249 kobject_put(&eq->kobj);
253 elevator_attach(q, eq, data);
257 EXPORT_SYMBOL(elevator_init);
259 void elevator_exit(elevator_t *e)
261 mutex_lock(&e->sysfs_lock);
262 if (e->ops->elevator_exit_fn)
263 e->ops->elevator_exit_fn(e);
265 mutex_unlock(&e->sysfs_lock);
267 kobject_put(&e->kobj);
270 EXPORT_SYMBOL(elevator_exit);
272 static void elv_activate_rq(request_queue_t *q, struct request *rq)
274 elevator_t *e = q->elevator;
276 if (e->ops->elevator_activate_req_fn)
277 e->ops->elevator_activate_req_fn(q, rq);
280 static void elv_deactivate_rq(request_queue_t *q, struct request *rq)
282 elevator_t *e = q->elevator;
284 if (e->ops->elevator_deactivate_req_fn)
285 e->ops->elevator_deactivate_req_fn(q, rq);
288 static inline void __elv_rqhash_del(struct request *rq)
290 hlist_del_init(&rq->hash);
293 static void elv_rqhash_del(request_queue_t *q, struct request *rq)
296 __elv_rqhash_del(rq);
299 static void elv_rqhash_add(request_queue_t *q, struct request *rq)
301 elevator_t *e = q->elevator;
303 BUG_ON(ELV_ON_HASH(rq));
304 hlist_add_head(&rq->hash, &e->hash[ELV_HASH_FN(rq_hash_key(rq))]);
307 static void elv_rqhash_reposition(request_queue_t *q, struct request *rq)
309 __elv_rqhash_del(rq);
310 elv_rqhash_add(q, rq);
313 static struct request *elv_rqhash_find(request_queue_t *q, sector_t offset)
315 elevator_t *e = q->elevator;
316 struct hlist_head *hash_list = &e->hash[ELV_HASH_FN(offset)];
317 struct hlist_node *entry, *next;
320 hlist_for_each_entry_safe(rq, entry, next, hash_list, hash) {
321 BUG_ON(!ELV_ON_HASH(rq));
323 if (unlikely(!rq_mergeable(rq))) {
324 __elv_rqhash_del(rq);
328 if (rq_hash_key(rq) == offset)
336 * RB-tree support functions for inserting/lookup/removal of requests
337 * in a sorted RB tree.
339 struct request *elv_rb_add(struct rb_root *root, struct request *rq)
341 struct rb_node **p = &root->rb_node;
342 struct rb_node *parent = NULL;
343 struct request *__rq;
347 __rq = rb_entry(parent, struct request, rb_node);
349 if (rq->sector < __rq->sector)
351 else if (rq->sector > __rq->sector)
357 rb_link_node(&rq->rb_node, parent, p);
358 rb_insert_color(&rq->rb_node, root);
362 EXPORT_SYMBOL(elv_rb_add);
364 void elv_rb_del(struct rb_root *root, struct request *rq)
366 BUG_ON(RB_EMPTY_NODE(&rq->rb_node));
367 rb_erase(&rq->rb_node, root);
368 RB_CLEAR_NODE(&rq->rb_node);
371 EXPORT_SYMBOL(elv_rb_del);
373 struct request *elv_rb_find(struct rb_root *root, sector_t sector)
375 struct rb_node *n = root->rb_node;
379 rq = rb_entry(n, struct request, rb_node);
381 if (sector < rq->sector)
383 else if (sector > rq->sector)
392 EXPORT_SYMBOL(elv_rb_find);
395 * Insert rq into dispatch queue of q. Queue lock must be held on
396 * entry. rq is sort insted into the dispatch queue. To be used by
397 * specific elevators.
399 void elv_dispatch_sort(request_queue_t *q, struct request *rq)
402 struct list_head *entry;
404 if (q->last_merge == rq)
405 q->last_merge = NULL;
407 elv_rqhash_del(q, rq);
411 boundary = q->end_sector;
413 list_for_each_prev(entry, &q->queue_head) {
414 struct request *pos = list_entry_rq(entry);
416 if (pos->cmd_flags & (REQ_SOFTBARRIER|REQ_HARDBARRIER|REQ_STARTED))
418 if (rq->sector >= boundary) {
419 if (pos->sector < boundary)
422 if (pos->sector >= boundary)
425 if (rq->sector >= pos->sector)
429 list_add(&rq->queuelist, entry);
432 EXPORT_SYMBOL(elv_dispatch_sort);
435 * Insert rq into dispatch queue of q. Queue lock must be held on
436 * entry. rq is added to the back of the dispatch queue. To be used by
437 * specific elevators.
439 void elv_dispatch_add_tail(struct request_queue *q, struct request *rq)
441 if (q->last_merge == rq)
442 q->last_merge = NULL;
444 elv_rqhash_del(q, rq);
448 q->end_sector = rq_end_sector(rq);
450 list_add_tail(&rq->queuelist, &q->queue_head);
453 EXPORT_SYMBOL(elv_dispatch_add_tail);
455 int elv_merge(request_queue_t *q, struct request **req, struct bio *bio)
457 elevator_t *e = q->elevator;
458 struct request *__rq;
462 * First try one-hit cache.
465 ret = elv_try_merge(q->last_merge, bio);
466 if (ret != ELEVATOR_NO_MERGE) {
467 *req = q->last_merge;
473 * See if our hash lookup can find a potential backmerge.
475 __rq = elv_rqhash_find(q, bio->bi_sector);
476 if (__rq && elv_rq_merge_ok(__rq, bio)) {
478 return ELEVATOR_BACK_MERGE;
481 if (e->ops->elevator_merge_fn)
482 return e->ops->elevator_merge_fn(q, req, bio);
484 return ELEVATOR_NO_MERGE;
487 void elv_merged_request(request_queue_t *q, struct request *rq, int type)
489 elevator_t *e = q->elevator;
491 if (e->ops->elevator_merged_fn)
492 e->ops->elevator_merged_fn(q, rq, type);
494 if (type == ELEVATOR_BACK_MERGE)
495 elv_rqhash_reposition(q, rq);
500 void elv_merge_requests(request_queue_t *q, struct request *rq,
501 struct request *next)
503 elevator_t *e = q->elevator;
505 if (e->ops->elevator_merge_req_fn)
506 e->ops->elevator_merge_req_fn(q, rq, next);
508 elv_rqhash_reposition(q, rq);
509 elv_rqhash_del(q, next);
515 void elv_requeue_request(request_queue_t *q, struct request *rq)
518 * it already went through dequeue, we need to decrement the
519 * in_flight count again
521 if (blk_account_rq(rq)) {
523 if (blk_sorted_rq(rq))
524 elv_deactivate_rq(q, rq);
527 rq->cmd_flags &= ~REQ_STARTED;
529 elv_insert(q, rq, ELEVATOR_INSERT_REQUEUE);
532 static void elv_drain_elevator(request_queue_t *q)
535 while (q->elevator->ops->elevator_dispatch_fn(q, 1))
537 if (q->nr_sorted == 0)
539 if (printed++ < 10) {
540 printk(KERN_ERR "%s: forced dispatching is broken "
541 "(nr_sorted=%u), please report this\n",
542 q->elevator->elevator_type->elevator_name, q->nr_sorted);
546 void elv_insert(request_queue_t *q, struct request *rq, int where)
548 struct list_head *pos;
552 blk_add_trace_rq(q, rq, BLK_TA_INSERT);
557 case ELEVATOR_INSERT_FRONT:
558 rq->cmd_flags |= REQ_SOFTBARRIER;
560 list_add(&rq->queuelist, &q->queue_head);
563 case ELEVATOR_INSERT_BACK:
564 rq->cmd_flags |= REQ_SOFTBARRIER;
565 elv_drain_elevator(q);
566 list_add_tail(&rq->queuelist, &q->queue_head);
568 * We kick the queue here for the following reasons.
569 * - The elevator might have returned NULL previously
570 * to delay requests and returned them now. As the
571 * queue wasn't empty before this request, ll_rw_blk
572 * won't run the queue on return, resulting in hang.
573 * - Usually, back inserted requests won't be merged
574 * with anything. There's no point in delaying queue
581 case ELEVATOR_INSERT_SORT:
582 BUG_ON(!blk_fs_request(rq));
583 rq->cmd_flags |= REQ_SORTED;
585 if (rq_mergeable(rq)) {
586 elv_rqhash_add(q, rq);
592 * Some ioscheds (cfq) run q->request_fn directly, so
593 * rq cannot be accessed after calling
594 * elevator_add_req_fn.
596 q->elevator->ops->elevator_add_req_fn(q, rq);
599 case ELEVATOR_INSERT_REQUEUE:
601 * If ordered flush isn't in progress, we do front
602 * insertion; otherwise, requests should be requeued
605 rq->cmd_flags |= REQ_SOFTBARRIER;
608 * Most requeues happen because of a busy condition,
609 * don't force unplug of the queue for that case.
613 if (q->ordseq == 0) {
614 list_add(&rq->queuelist, &q->queue_head);
618 ordseq = blk_ordered_req_seq(rq);
620 list_for_each(pos, &q->queue_head) {
621 struct request *pos_rq = list_entry_rq(pos);
622 if (ordseq <= blk_ordered_req_seq(pos_rq))
626 list_add_tail(&rq->queuelist, pos);
630 printk(KERN_ERR "%s: bad insertion point %d\n",
631 __FUNCTION__, where);
635 if (unplug_it && blk_queue_plugged(q)) {
636 int nrq = q->rq.count[READ] + q->rq.count[WRITE]
639 if (nrq >= q->unplug_thresh)
640 __generic_unplug_device(q);
644 void __elv_add_request(request_queue_t *q, struct request *rq, int where,
648 rq->cmd_flags |= REQ_ORDERED_COLOR;
650 if (rq->cmd_flags & (REQ_SOFTBARRIER | REQ_HARDBARRIER)) {
652 * toggle ordered color
654 if (blk_barrier_rq(rq))
658 * barriers implicitly indicate back insertion
660 if (where == ELEVATOR_INSERT_SORT)
661 where = ELEVATOR_INSERT_BACK;
664 * this request is scheduling boundary, update
667 if (blk_fs_request(rq)) {
668 q->end_sector = rq_end_sector(rq);
671 } else if (!(rq->cmd_flags & REQ_ELVPRIV) && where == ELEVATOR_INSERT_SORT)
672 where = ELEVATOR_INSERT_BACK;
677 elv_insert(q, rq, where);
680 EXPORT_SYMBOL(__elv_add_request);
682 void elv_add_request(request_queue_t *q, struct request *rq, int where,
687 spin_lock_irqsave(q->queue_lock, flags);
688 __elv_add_request(q, rq, where, plug);
689 spin_unlock_irqrestore(q->queue_lock, flags);
692 EXPORT_SYMBOL(elv_add_request);
694 static inline struct request *__elv_next_request(request_queue_t *q)
699 while (!list_empty(&q->queue_head)) {
700 rq = list_entry_rq(q->queue_head.next);
701 if (blk_do_ordered(q, &rq))
705 if (!q->elevator->ops->elevator_dispatch_fn(q, 0))
710 struct request *elv_next_request(request_queue_t *q)
715 while ((rq = __elv_next_request(q)) != NULL) {
716 if (!(rq->cmd_flags & REQ_STARTED)) {
718 * This is the first time the device driver
719 * sees this request (possibly after
720 * requeueing). Notify IO scheduler.
722 if (blk_sorted_rq(rq))
723 elv_activate_rq(q, rq);
726 * just mark as started even if we don't start
727 * it, a request that has been delayed should
728 * not be passed by new incoming requests
730 rq->cmd_flags |= REQ_STARTED;
731 blk_add_trace_rq(q, rq, BLK_TA_ISSUE);
734 if (!q->boundary_rq || q->boundary_rq == rq) {
735 q->end_sector = rq_end_sector(rq);
736 q->boundary_rq = NULL;
739 if ((rq->cmd_flags & REQ_DONTPREP) || !q->prep_rq_fn)
742 ret = q->prep_rq_fn(q, rq);
743 if (ret == BLKPREP_OK) {
745 } else if (ret == BLKPREP_DEFER) {
747 * the request may have been (partially) prepped.
748 * we need to keep this request in the front to
749 * avoid resource deadlock. REQ_STARTED will
750 * prevent other fs requests from passing this one.
754 } else if (ret == BLKPREP_KILL) {
755 int nr_bytes = rq->hard_nr_sectors << 9;
758 nr_bytes = rq->data_len;
760 blkdev_dequeue_request(rq);
761 rq->cmd_flags |= REQ_QUIET;
762 end_that_request_chunk(rq, 0, nr_bytes);
763 end_that_request_last(rq, 0);
765 printk(KERN_ERR "%s: bad return=%d\n", __FUNCTION__,
774 EXPORT_SYMBOL(elv_next_request);
776 void elv_dequeue_request(request_queue_t *q, struct request *rq)
778 BUG_ON(list_empty(&rq->queuelist));
779 BUG_ON(ELV_ON_HASH(rq));
781 list_del_init(&rq->queuelist);
784 * the time frame between a request being removed from the lists
785 * and to it is freed is accounted as io that is in progress at
788 if (blk_account_rq(rq))
792 EXPORT_SYMBOL(elv_dequeue_request);
794 int elv_queue_empty(request_queue_t *q)
796 elevator_t *e = q->elevator;
798 if (!list_empty(&q->queue_head))
801 if (e->ops->elevator_queue_empty_fn)
802 return e->ops->elevator_queue_empty_fn(q);
807 EXPORT_SYMBOL(elv_queue_empty);
809 struct request *elv_latter_request(request_queue_t *q, struct request *rq)
811 elevator_t *e = q->elevator;
813 if (e->ops->elevator_latter_req_fn)
814 return e->ops->elevator_latter_req_fn(q, rq);
818 struct request *elv_former_request(request_queue_t *q, struct request *rq)
820 elevator_t *e = q->elevator;
822 if (e->ops->elevator_former_req_fn)
823 return e->ops->elevator_former_req_fn(q, rq);
827 int elv_set_request(request_queue_t *q, struct request *rq, gfp_t gfp_mask)
829 elevator_t *e = q->elevator;
831 if (e->ops->elevator_set_req_fn)
832 return e->ops->elevator_set_req_fn(q, rq, gfp_mask);
834 rq->elevator_private = NULL;
838 void elv_put_request(request_queue_t *q, struct request *rq)
840 elevator_t *e = q->elevator;
842 if (e->ops->elevator_put_req_fn)
843 e->ops->elevator_put_req_fn(rq);
846 int elv_may_queue(request_queue_t *q, int rw)
848 elevator_t *e = q->elevator;
850 if (e->ops->elevator_may_queue_fn)
851 return e->ops->elevator_may_queue_fn(q, rw);
853 return ELV_MQUEUE_MAY;
856 void elv_completed_request(request_queue_t *q, struct request *rq)
858 elevator_t *e = q->elevator;
861 * request is released from the driver, io must be done
863 if (blk_account_rq(rq)) {
865 if (blk_sorted_rq(rq) && e->ops->elevator_completed_req_fn)
866 e->ops->elevator_completed_req_fn(q, rq);
870 * Check if the queue is waiting for fs requests to be
871 * drained for flush sequence.
873 if (unlikely(q->ordseq)) {
874 struct request *first_rq = list_entry_rq(q->queue_head.next);
875 if (q->in_flight == 0 &&
876 blk_ordered_cur_seq(q) == QUEUE_ORDSEQ_DRAIN &&
877 blk_ordered_req_seq(first_rq) > QUEUE_ORDSEQ_DRAIN) {
878 blk_ordered_complete_seq(q, QUEUE_ORDSEQ_DRAIN, 0);
884 #define to_elv(atr) container_of((atr), struct elv_fs_entry, attr)
887 elv_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
889 elevator_t *e = container_of(kobj, elevator_t, kobj);
890 struct elv_fs_entry *entry = to_elv(attr);
896 mutex_lock(&e->sysfs_lock);
897 error = e->ops ? entry->show(e, page) : -ENOENT;
898 mutex_unlock(&e->sysfs_lock);
903 elv_attr_store(struct kobject *kobj, struct attribute *attr,
904 const char *page, size_t length)
906 elevator_t *e = container_of(kobj, elevator_t, kobj);
907 struct elv_fs_entry *entry = to_elv(attr);
913 mutex_lock(&e->sysfs_lock);
914 error = e->ops ? entry->store(e, page, length) : -ENOENT;
915 mutex_unlock(&e->sysfs_lock);
919 static struct sysfs_ops elv_sysfs_ops = {
920 .show = elv_attr_show,
921 .store = elv_attr_store,
924 static struct kobj_type elv_ktype = {
925 .sysfs_ops = &elv_sysfs_ops,
926 .release = elevator_release,
929 int elv_register_queue(struct request_queue *q)
931 elevator_t *e = q->elevator;
934 e->kobj.parent = &q->kobj;
936 error = kobject_add(&e->kobj);
938 struct elv_fs_entry *attr = e->elevator_type->elevator_attrs;
940 while (attr->attr.name) {
941 if (sysfs_create_file(&e->kobj, &attr->attr))
946 kobject_uevent(&e->kobj, KOBJ_ADD);
951 static void __elv_unregister_queue(elevator_t *e)
953 kobject_uevent(&e->kobj, KOBJ_REMOVE);
954 kobject_del(&e->kobj);
957 void elv_unregister_queue(struct request_queue *q)
960 __elv_unregister_queue(q->elevator);
963 int elv_register(struct elevator_type *e)
965 spin_lock_irq(&elv_list_lock);
966 BUG_ON(elevator_find(e->elevator_name));
967 list_add_tail(&e->list, &elv_list);
968 spin_unlock_irq(&elv_list_lock);
970 printk(KERN_INFO "io scheduler %s registered", e->elevator_name);
971 if (!strcmp(e->elevator_name, chosen_elevator) ||
972 (!*chosen_elevator &&
973 !strcmp(e->elevator_name, CONFIG_DEFAULT_IOSCHED)))
974 printk(" (default)");
978 EXPORT_SYMBOL_GPL(elv_register);
980 void elv_unregister(struct elevator_type *e)
982 struct task_struct *g, *p;
985 * Iterate every thread in the process to remove the io contexts.
988 read_lock(&tasklist_lock);
989 do_each_thread(g, p) {
992 e->ops.trim(p->io_context);
994 } while_each_thread(g, p);
995 read_unlock(&tasklist_lock);
998 spin_lock_irq(&elv_list_lock);
999 list_del_init(&e->list);
1000 spin_unlock_irq(&elv_list_lock);
1002 EXPORT_SYMBOL_GPL(elv_unregister);
1005 * switch to new_e io scheduler. be careful not to introduce deadlocks -
1006 * we don't free the old io scheduler, before we have allocated what we
1007 * need for the new one. this way we have a chance of going back to the old
1008 * one, if the new one fails init for some reason.
1010 static int elevator_switch(request_queue_t *q, struct elevator_type *new_e)
1012 elevator_t *old_elevator, *e;
1016 * Allocate new elevator
1018 e = elevator_alloc(q, new_e);
1022 data = elevator_init_queue(q, e);
1024 kobject_put(&e->kobj);
1029 * Turn on BYPASS and drain all requests w/ elevator private data
1031 spin_lock_irq(q->queue_lock);
1033 set_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);
1035 elv_drain_elevator(q);
1037 while (q->rq.elvpriv) {
1040 spin_unlock_irq(q->queue_lock);
1042 spin_lock_irq(q->queue_lock);
1043 elv_drain_elevator(q);
1047 * Remember old elevator.
1049 old_elevator = q->elevator;
1052 * attach and start new elevator
1054 elevator_attach(q, e, data);
1056 spin_unlock_irq(q->queue_lock);
1058 __elv_unregister_queue(old_elevator);
1060 if (elv_register_queue(q))
1064 * finally exit old elevator and turn off BYPASS.
1066 elevator_exit(old_elevator);
1067 clear_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);
1072 * switch failed, exit the new io scheduler and reattach the old
1073 * one again (along with re-adding the sysfs dir)
1076 q->elevator = old_elevator;
1077 elv_register_queue(q);
1078 clear_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);
1082 ssize_t elv_iosched_store(request_queue_t *q, const char *name, size_t count)
1084 char elevator_name[ELV_NAME_MAX];
1086 struct elevator_type *e;
1088 elevator_name[sizeof(elevator_name) - 1] = '\0';
1089 strncpy(elevator_name, name, sizeof(elevator_name) - 1);
1090 len = strlen(elevator_name);
1092 if (len && elevator_name[len - 1] == '\n')
1093 elevator_name[len - 1] = '\0';
1095 e = elevator_get(elevator_name);
1097 printk(KERN_ERR "elevator: type %s not found\n", elevator_name);
1101 if (!strcmp(elevator_name, q->elevator->elevator_type->elevator_name)) {
1106 if (!elevator_switch(q, e))
1107 printk(KERN_ERR "elevator: switch to %s failed\n",elevator_name);
1111 ssize_t elv_iosched_show(request_queue_t *q, char *name)
1113 elevator_t *e = q->elevator;
1114 struct elevator_type *elv = e->elevator_type;
1115 struct list_head *entry;
1118 spin_lock_irq(&elv_list_lock);
1119 list_for_each(entry, &elv_list) {
1120 struct elevator_type *__e;
1122 __e = list_entry(entry, struct elevator_type, list);
1123 if (!strcmp(elv->elevator_name, __e->elevator_name))
1124 len += sprintf(name+len, "[%s] ", elv->elevator_name);
1126 len += sprintf(name+len, "%s ", __e->elevator_name);
1128 spin_unlock_irq(&elv_list_lock);
1130 len += sprintf(len+name, "\n");
1134 struct request *elv_rb_former_request(request_queue_t *q, struct request *rq)
1136 struct rb_node *rbprev = rb_prev(&rq->rb_node);
1139 return rb_entry_rq(rbprev);
1144 EXPORT_SYMBOL(elv_rb_former_request);
1146 struct request *elv_rb_latter_request(request_queue_t *q, struct request *rq)
1148 struct rb_node *rbnext = rb_next(&rq->rb_node);
1151 return rb_entry_rq(rbnext);
1156 EXPORT_SYMBOL(elv_rb_latter_request);