cfq-iosched: implement slower async initiate and queue ramp up
Jens Axboe [Sat, 3 Oct 2009 14:26:03 +0000 (16:26 +0200)]
This slowly ramps up the async queue depth based on the time
passed since the sync IO, and doesn't allow async at all until
a sync slice period has passed.

Signed-off-by: Jens Axboe <jens.axboe@oracle.com>

block/blk-core.c
block/cfq-iosched.c
include/linux/blkdev.h

index ddaaea4..a8c7fbe 100644 (file)
@@ -2492,6 +2492,14 @@ int kblockd_schedule_work(struct request_queue *q, struct work_struct *work)
 }
 EXPORT_SYMBOL(kblockd_schedule_work);
 
+int kblockd_schedule_delayed_work(struct request_queue *q,
+                                 struct delayed_work *work,
+                                 unsigned long delay)
+{
+       return queue_delayed_work(kblockd_workqueue, work, delay);
+}
+EXPORT_SYMBOL(kblockd_schedule_delayed_work);
+
 int __init blk_dev_init(void)
 {
        BUILD_BUG_ON(__REQ_NR_BITS > 8 *
index 70b48ea..fce8a74 100644 (file)
@@ -150,7 +150,7 @@ struct cfq_data {
         * idle window management
         */
        struct timer_list idle_slice_timer;
-       struct work_struct unplug_work;
+       struct delayed_work unplug_work;
 
        struct cfq_queue *active_queue;
        struct cfq_io_context *active_cic;
@@ -268,11 +268,13 @@ static inline int cfq_bio_sync(struct bio *bio)
  * scheduler run of queue, if there are requests pending and no one in the
  * driver that will restart queueing
  */
-static inline void cfq_schedule_dispatch(struct cfq_data *cfqd)
+static inline void cfq_schedule_dispatch(struct cfq_data *cfqd,
+                                        unsigned long delay)
 {
        if (cfqd->busy_queues) {
                cfq_log(cfqd, "schedule dispatch");
-               kblockd_schedule_work(cfqd->queue, &cfqd->unplug_work);
+               kblockd_schedule_delayed_work(cfqd->queue, &cfqd->unplug_work,
+                                               delay);
        }
 }
 
@@ -1316,8 +1318,6 @@ static int cfq_dispatch_requests(struct request_queue *q, int force)
         * Does this cfqq already have too much IO in flight?
         */
        if (cfqq->dispatched >= max_dispatch) {
-               unsigned long load_at = cfqd->last_end_sync_rq + cfq_slice_sync;
-
                /*
                 * idle queue must always only have a single IO in flight
                 */
@@ -1331,20 +1331,36 @@ static int cfq_dispatch_requests(struct request_queue *q, int force)
                        return 0;
 
                /*
-                * If a sync request has completed recently, don't overload
-                * the dispatch queue yet with async requests.
+                * Sole queue user, allow bigger slice
                 */
-               if (cfqd->cfq_desktop && !cfq_cfqq_sync(cfqq)
-                   && time_before(jiffies, load_at))
-                       return 0;
+               max_dispatch *= 4;
+       }
+
+       /*
+        * Async queues must wait a bit before being allowed dispatch.
+        * We also ramp up the dispatch depth gradually for async IO,
+        * based on the last sync IO we serviced
+        */
+       if (!cfq_cfqq_sync(cfqq) && cfqd->cfq_desktop) {
+               unsigned long last_sync = jiffies - cfqd->last_end_sync_rq;
+               unsigned int depth;
 
                /*
-                * we are the only queue, allow up to 4 times of 'quantum'
+                * must wait a bit longer
                 */
-               if (cfqq->dispatched >= 4 * max_dispatch)
+               if (last_sync < cfq_slice_sync) {
+                       cfq_schedule_dispatch(cfqd, cfq_slice_sync - last_sync);
                        return 0;
+               }
+
+               depth = last_sync / cfq_slice_sync;
+               if (depth < max_dispatch)
+                       max_dispatch = depth;
        }
 
+       if (cfqq->dispatched >= max_dispatch)
+               return 0;
+
        /*
         * Dispatch a request from this cfqq
         */
@@ -1389,7 +1405,7 @@ static void cfq_put_queue(struct cfq_queue *cfqq)
 
        if (unlikely(cfqd->active_queue == cfqq)) {
                __cfq_slice_expired(cfqd, cfqq, 0);
-               cfq_schedule_dispatch(cfqd);
+               cfq_schedule_dispatch(cfqd, 0);
        }
 
        kmem_cache_free(cfq_pool, cfqq);
@@ -1484,7 +1500,7 @@ static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq)
 {
        if (unlikely(cfqq == cfqd->active_queue)) {
                __cfq_slice_expired(cfqd, cfqq, 0);
-               cfq_schedule_dispatch(cfqd);
+               cfq_schedule_dispatch(cfqd, 0);
        }
 
        cfq_put_queue(cfqq);
@@ -2201,7 +2217,7 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq)
        }
 
        if (!rq_in_driver(cfqd))
-               cfq_schedule_dispatch(cfqd);
+               cfq_schedule_dispatch(cfqd, 0);
 }
 
 /*
@@ -2331,7 +2347,7 @@ queue_fail:
        if (cic)
                put_io_context(cic->ioc);
 
-       cfq_schedule_dispatch(cfqd);
+       cfq_schedule_dispatch(cfqd, 0);
        spin_unlock_irqrestore(q->queue_lock, flags);
        cfq_log(cfqd, "set_request fail");
        return 1;
@@ -2340,7 +2356,7 @@ queue_fail:
 static void cfq_kick_queue(struct work_struct *work)
 {
        struct cfq_data *cfqd =
-               container_of(work, struct cfq_data, unplug_work);
+               container_of(work, struct cfq_data, unplug_work.work);
        struct request_queue *q = cfqd->queue;
 
        spin_lock_irq(q->queue_lock);
@@ -2394,7 +2410,7 @@ static void cfq_idle_slice_timer(unsigned long data)
 expire:
        cfq_slice_expired(cfqd, timed_out);
 out_kick:
-       cfq_schedule_dispatch(cfqd);
+       cfq_schedule_dispatch(cfqd, 0);
 out_cont:
        spin_unlock_irqrestore(cfqd->queue->queue_lock, flags);
 }
@@ -2402,7 +2418,7 @@ out_cont:
 static void cfq_shutdown_timer_wq(struct cfq_data *cfqd)
 {
        del_timer_sync(&cfqd->idle_slice_timer);
-       cancel_work_sync(&cfqd->unplug_work);
+       cancel_delayed_work_sync(&cfqd->unplug_work);
 }
 
 static void cfq_put_async_queues(struct cfq_data *cfqd)
@@ -2484,7 +2500,7 @@ static void *cfq_init_queue(struct request_queue *q)
        cfqd->idle_slice_timer.function = cfq_idle_slice_timer;
        cfqd->idle_slice_timer.data = (unsigned long) cfqd;
 
-       INIT_WORK(&cfqd->unplug_work, cfq_kick_queue);
+       INIT_DELAYED_WORK(&cfqd->unplug_work, cfq_kick_queue);
 
        cfqd->cfq_quantum = cfq_quantum;
        cfqd->cfq_fifo_expire[0] = cfq_fifo_expire[0];
index 1a03b71..a732393 100644 (file)
@@ -1147,7 +1147,11 @@ static inline void put_dev_sector(Sector p)
 }
 
 struct work_struct;
+struct delayed_work;
 int kblockd_schedule_work(struct request_queue *q, struct work_struct *work);
+int kblockd_schedule_delayed_work(struct request_queue *q,
+                                       struct delayed_work *work,
+                                       unsigned long delay);
 
 #define MODULE_ALIAS_BLOCKDEV(major,minor) \
        MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))