block: bdev_stack_limits wrapper
[linux-2.6.git] / block / blk-core.c
index 93691d2..718897e 100644 (file)
 #include <linux/swap.h>
 #include <linux/writeback.h>
 #include <linux/task_io_accounting_ops.h>
-#include <linux/blktrace_api.h>
 #include <linux/fault-inject.h>
-#include <trace/block.h>
+
+#define CREATE_TRACE_POINTS
+#include <trace/events/block.h>
 
 #include "blk.h"
 
-DEFINE_TRACE(block_plug);
-DEFINE_TRACE(block_unplug_io);
-DEFINE_TRACE(block_unplug_timer);
-DEFINE_TRACE(block_getrq);
-DEFINE_TRACE(block_sleeprq);
-DEFINE_TRACE(block_rq_requeue);
-DEFINE_TRACE(block_bio_backmerge);
-DEFINE_TRACE(block_bio_frontmerge);
-DEFINE_TRACE(block_bio_queue);
-DEFINE_TRACE(block_rq_complete);
-DEFINE_TRACE(block_remap);     /* Also used in drivers/md/dm.c */
 EXPORT_TRACEPOINT_SYMBOL_GPL(block_remap);
+EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
+EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
 
 static int __make_request(struct request_queue *q, struct bio *bio);
 
@@ -78,7 +70,7 @@ static void drive_stat_acct(struct request *rq, int new_io)
                part_stat_inc(cpu, part, merges[rw]);
        else {
                part_round_stats(cpu, part);
-               part_inc_in_flight(part);
+               part_inc_in_flight(part, rw);
        }
 
        part_stat_unlock();
@@ -506,6 +498,12 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
 
        q->backing_dev_info.unplug_io_fn = blk_backing_dev_unplug;
        q->backing_dev_info.unplug_io_data = q;
+       q->backing_dev_info.ra_pages =
+                       (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE;
+       q->backing_dev_info.state = 0;
+       q->backing_dev_info.capabilities = BDI_CAP_MAP_COPY;
+       q->backing_dev_info.name = "block";
+
        err = bdi_init(&q->backing_dev_info);
        if (err) {
                kmem_cache_free(blk_requestq_cachep, q);
@@ -579,13 +577,6 @@ blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
                return NULL;
        }
 
-       /*
-        * if caller didn't supply a lock, they get per-queue locking with
-        * our embedded lock
-        */
-       if (!lock)
-               lock = &q->__queue_lock;
-
        q->request_fn           = rfn;
        q->prep_rq_fn           = NULL;
        q->unplug_fn            = generic_unplug_device;
@@ -599,8 +590,6 @@ blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
 
        q->sg_reserved_size = INT_MAX;
 
-       blk_set_cmd_filter_defaults(&q->cmd_filter);
-
        /*
         * all done
         */
@@ -891,6 +880,61 @@ struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask)
 EXPORT_SYMBOL(blk_get_request);
 
 /**
+ * blk_make_request - given a bio, allocate a corresponding struct request.
+ * @q: target request queue
+ * @bio:  The bio describing the memory mappings that will be submitted for IO.
+ *        It may be a chained-bio properly constructed by block/bio layer.
+ * @gfp_mask: gfp flags to be used for memory allocation
+ *
+ * blk_make_request is the parallel of generic_make_request for BLOCK_PC
+ * type commands. Where the struct request needs to be farther initialized by
+ * the caller. It is passed a &struct bio, which describes the memory info of
+ * the I/O transfer.
+ *
+ * The caller of blk_make_request must make sure that bi_io_vec
+ * are set to describe the memory buffers. That bio_data_dir() will return
+ * the needed direction of the request. (And all bio's in the passed bio-chain
+ * are properly set accordingly)
+ *
+ * If called under none-sleepable conditions, mapped bio buffers must not
+ * need bouncing, by calling the appropriate masked or flagged allocator,
+ * suitable for the target device. Otherwise the call to blk_queue_bounce will
+ * BUG.
+ *
+ * WARNING: When allocating/cloning a bio-chain, careful consideration should be
+ * given to how you allocate bios. In particular, you cannot use __GFP_WAIT for
+ * anything but the first bio in the chain. Otherwise you risk waiting for IO
+ * completion of a bio that hasn't been submitted yet, thus resulting in a
+ * deadlock. Alternatively bios should be allocated using bio_kmalloc() instead
+ * of bio_alloc(), as that avoids the mempool deadlock.
+ * If possible a big IO should be split into smaller parts when allocation
+ * fails. Partial allocation should not be an error, or you risk a live-lock.
+ */
+struct request *blk_make_request(struct request_queue *q, struct bio *bio,
+                                gfp_t gfp_mask)
+{
+       struct request *rq = blk_get_request(q, bio_data_dir(bio), gfp_mask);
+
+       if (unlikely(!rq))
+               return ERR_PTR(-ENOMEM);
+
+       for_each_bio(bio) {
+               struct bio *bounce_bio = bio;
+               int ret;
+
+               blk_queue_bounce(q, &bounce_bio);
+               ret = blk_rq_append_bio(q, rq, bounce_bio);
+               if (unlikely(ret)) {
+                       blk_put_request(rq);
+                       return ERR_PTR(ret);
+               }
+       }
+
+       return rq;
+}
+EXPORT_SYMBOL(blk_make_request);
+
+/**
  * blk_requeue_request - put a request back on queue
  * @q:         request queue where request should be inserted
  * @rq:                request to be inserted
@@ -902,8 +946,6 @@ EXPORT_SYMBOL(blk_get_request);
  */
 void blk_requeue_request(struct request_queue *q, struct request *rq)
 {
-       BUG_ON(blk_queued_rq(rq));
-
        blk_delete_timer(rq);
        blk_clear_rq_complete(rq);
        trace_block_rq_requeue(q, rq);
@@ -911,6 +953,8 @@ void blk_requeue_request(struct request_queue *q, struct request *rq)
        if (blk_rq_tagged(rq))
                blk_queue_end_tag(q, rq);
 
+       BUG_ON(blk_queued_rq(rq));
+
        elv_requeue_request(q, rq);
 }
 EXPORT_SYMBOL(blk_requeue_request);
@@ -986,9 +1030,9 @@ static void part_round_stats_single(int cpu, struct hd_struct *part,
        if (now == part->stamp)
                return;
 
-       if (part->in_flight) {
+       if (part_in_flight(part)) {
                __part_stat_add(cpu, part, time_in_queue,
-                               part->in_flight * (now - part->stamp));
+                               part_in_flight(part) * (now - part->stamp));
                __part_stat_add(cpu, part, io_ticks, (now - part->stamp));
        }
        part->stamp = now;
@@ -1069,31 +1113,26 @@ void init_request_from_bio(struct request *req, struct bio *bio)
        req->cmd_type = REQ_TYPE_FS;
 
        /*
-        * inherit FAILFAST from bio (for read-ahead, and explicit FAILFAST)
+        * Inherit FAILFAST from bio (for read-ahead, and explicit
+        * FAILFAST).  FAILFAST flags are identical for req and bio.
         */
-       if (bio_rw_ahead(bio))
-               req->cmd_flags |= (REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT |
-                                  REQ_FAILFAST_DRIVER);
-       if (bio_failfast_dev(bio))
-               req->cmd_flags |= REQ_FAILFAST_DEV;
-       if (bio_failfast_transport(bio))
-               req->cmd_flags |= REQ_FAILFAST_TRANSPORT;
-       if (bio_failfast_driver(bio))
-               req->cmd_flags |= REQ_FAILFAST_DRIVER;
-
-       if (unlikely(bio_discard(bio))) {
+       if (bio_rw_flagged(bio, BIO_RW_AHEAD))
+               req->cmd_flags |= REQ_FAILFAST_MASK;
+       else
+               req->cmd_flags |= bio->bi_rw & REQ_FAILFAST_MASK;
+
+       if (unlikely(bio_rw_flagged(bio, BIO_RW_DISCARD))) {
                req->cmd_flags |= REQ_DISCARD;
-               if (bio_barrier(bio))
+               if (bio_rw_flagged(bio, BIO_RW_BARRIER))
                        req->cmd_flags |= REQ_SOFTBARRIER;
-               req->q->prepare_discard_fn(req->q, req);
-       } else if (unlikely(bio_barrier(bio)))
+       } else if (unlikely(bio_rw_flagged(bio, BIO_RW_BARRIER)))
                req->cmd_flags |= REQ_HARDBARRIER;
 
-       if (bio_sync(bio))
+       if (bio_rw_flagged(bio, BIO_RW_SYNCIO))
                req->cmd_flags |= REQ_RW_SYNC;
-       if (bio_rw_meta(bio))
+       if (bio_rw_flagged(bio, BIO_RW_META))
                req->cmd_flags |= REQ_RW_META;
-       if (bio_noidle(bio))
+       if (bio_rw_flagged(bio, BIO_RW_NOIDLE))
                req->cmd_flags |= REQ_NOIDLE;
 
        req->errors = 0;
@@ -1108,7 +1147,7 @@ void init_request_from_bio(struct request *req, struct bio *bio)
  */
 static inline bool queue_should_plug(struct request_queue *q)
 {
-       return !(blk_queue_nonrot(q) && blk_queue_tagged(q));
+       return !(blk_queue_nonrot(q) && blk_queue_queuing(q));
 }
 
 static int __make_request(struct request_queue *q, struct bio *bio)
@@ -1117,10 +1156,16 @@ static int __make_request(struct request_queue *q, struct bio *bio)
        int el_ret;
        unsigned int bytes = bio->bi_size;
        const unsigned short prio = bio_prio(bio);
-       const int sync = bio_sync(bio);
-       const int unplug = bio_unplug(bio);
+       const bool sync = bio_rw_flagged(bio, BIO_RW_SYNCIO);
+       const bool unplug = bio_rw_flagged(bio, BIO_RW_UNPLUG);
+       const unsigned int ff = bio->bi_rw & REQ_FAILFAST_MASK;
        int rw_flags;
 
+       if (bio_rw_flagged(bio, BIO_RW_BARRIER) &&
+           (q->next_ordered == QUEUE_ORDERED_NONE)) {
+               bio_endio(bio, -EOPNOTSUPP);
+               return 0;
+       }
        /*
         * low level driver can indicate that it wants pages above a
         * certain limit bounced to low memory (ie for highmem, or even
@@ -1130,7 +1175,7 @@ static int __make_request(struct request_queue *q, struct bio *bio)
 
        spin_lock_irq(q->queue_lock);
 
-       if (unlikely(bio_barrier(bio)) || elv_queue_empty(q))
+       if (unlikely(bio_rw_flagged(bio, BIO_RW_BARRIER)) || elv_queue_empty(q))
                goto get_rq;
 
        el_ret = elv_merge(q, &req, bio);
@@ -1143,6 +1188,9 @@ static int __make_request(struct request_queue *q, struct bio *bio)
 
                trace_block_bio_backmerge(q, bio);
 
+               if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
+                       blk_rq_set_mixed_merge(req);
+
                req->biotail->bi_next = bio;
                req->biotail = bio;
                req->__data_len += bytes;
@@ -1162,6 +1210,12 @@ static int __make_request(struct request_queue *q, struct bio *bio)
 
                trace_block_bio_frontmerge(q, bio);
 
+               if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) {
+                       blk_rq_set_mixed_merge(req);
+                       req->cmd_flags &= ~REQ_FAILFAST_MASK;
+                       req->cmd_flags |= ff;
+               }
+
                bio->bi_next = req->bio;
                req->bio = bio;
 
@@ -1238,7 +1292,7 @@ static inline void blk_partition_remap(struct bio *bio)
                bio->bi_bdev = bdev->bd_contains;
 
                trace_block_remap(bdev_get_queue(bio->bi_bdev), bio,
-                                   bdev->bd_dev, bio->bi_sector,
+                                   bdev->bd_dev,
                                    bio->bi_sector - p->start_sect);
        }
 }
@@ -1383,11 +1437,12 @@ static inline void __generic_make_request(struct bio *bio)
                        goto end_io;
                }
 
-               if (unlikely(nr_sectors > q->max_hw_sectors)) {
+               if (unlikely(!bio_rw_flagged(bio, BIO_RW_DISCARD) &&
+                            nr_sectors > queue_max_hw_sectors(q))) {
                        printk(KERN_ERR "bio too big device %s (%u > %u)\n",
-                               bdevname(bio->bi_bdev, b),
-                               bio_sectors(bio),
-                               q->max_hw_sectors);
+                              bdevname(bio->bi_bdev, b),
+                              bio_sectors(bio),
+                              queue_max_hw_sectors(q));
                        goto end_io;
                }
 
@@ -1407,10 +1462,7 @@ static inline void __generic_make_request(struct bio *bio)
                        goto end_io;
 
                if (old_sector != -1)
-                       trace_block_remap(q, bio, old_dev, bio->bi_sector,
-                                           old_sector);
-
-               trace_block_bio_queue(q, bio);
+                       trace_block_remap(q, bio, old_dev, old_sector);
 
                old_sector = bio->bi_sector;
                old_dev = bio->bi_bdev->bd_dev;
@@ -1418,16 +1470,14 @@ static inline void __generic_make_request(struct bio *bio)
                if (bio_check_eod(bio, nr_sectors))
                        goto end_io;
 
-               if (bio_discard(bio) && !q->prepare_discard_fn) {
-                       err = -EOPNOTSUPP;
-                       goto end_io;
-               }
-               if (bio_barrier(bio) && bio_has_data(bio) &&
-                   (q->next_ordered == QUEUE_ORDERED_NONE)) {
+               if (bio_rw_flagged(bio, BIO_RW_DISCARD) &&
+                   !blk_queue_discard(q)) {
                        err = -EOPNOTSUPP;
                        goto end_io;
                }
 
+               trace_block_bio_queue(q, bio);
+
                ret = q->make_request_fn(q, bio);
        } while (ret);
 
@@ -1554,8 +1604,8 @@ EXPORT_SYMBOL(submit_bio);
  */
 int blk_rq_check_limits(struct request_queue *q, struct request *rq)
 {
-       if (blk_rq_sectors(rq) > q->max_sectors ||
-           blk_rq_bytes(rq) > q->max_hw_sectors << 9) {
+       if (blk_rq_sectors(rq) > queue_max_sectors(q) ||
+           blk_rq_bytes(rq) > queue_max_hw_sectors(q) << 9) {
                printk(KERN_ERR "%s: over max size limit.\n", __func__);
                return -EIO;
        }
@@ -1567,8 +1617,8 @@ int blk_rq_check_limits(struct request_queue *q, struct request *rq)
         * limitation.
         */
        blk_recalc_rq_segments(rq);
-       if (rq->nr_phys_segments > q->max_phys_segments ||
-           rq->nr_phys_segments > q->max_hw_segments) {
+       if (rq->nr_phys_segments > queue_max_phys_segments(q) ||
+           rq->nr_phys_segments > queue_max_hw_segments(q)) {
                printk(KERN_ERR "%s: over max segments limit.\n", __func__);
                return -EIO;
        }
@@ -1612,6 +1662,50 @@ int blk_insert_cloned_request(struct request_queue *q, struct request *rq)
 }
 EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
 
+/**
+ * blk_rq_err_bytes - determine number of bytes till the next failure boundary
+ * @rq: request to examine
+ *
+ * Description:
+ *     A request could be merge of IOs which require different failure
+ *     handling.  This function determines the number of bytes which
+ *     can be failed from the beginning of the request without
+ *     crossing into area which need to be retried further.
+ *
+ * Return:
+ *     The number of bytes to fail.
+ *
+ * Context:
+ *     queue_lock must be held.
+ */
+unsigned int blk_rq_err_bytes(const struct request *rq)
+{
+       unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
+       unsigned int bytes = 0;
+       struct bio *bio;
+
+       if (!(rq->cmd_flags & REQ_MIXED_MERGE))
+               return blk_rq_bytes(rq);
+
+       /*
+        * Currently the only 'mixing' which can happen is between
+        * different fastfail types.  We can safely fail portions
+        * which have all the failfast bits that the first one has -
+        * the ones which are at least as eager to fail as the first
+        * one.
+        */
+       for (bio = rq->bio; bio; bio = bio->bi_next) {
+               if ((bio->bi_rw & ff) != ff)
+                       break;
+               bytes += bio->bi_size;
+       }
+
+       /* this could lead to infinite loop */
+       BUG_ON(blk_rq_bytes(rq) && !bytes);
+       return bytes;
+}
+EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
+
 static void blk_account_io_completion(struct request *req, unsigned int bytes)
 {
        if (blk_do_io_stat(req)) {
@@ -1645,7 +1739,7 @@ static void blk_account_io_done(struct request *req)
                part_stat_inc(cpu, part, ios[rw]);
                part_stat_add(cpu, part, ticks[rw], duration);
                part_round_stats(cpu, part);
-               part_dec_in_flight(part);
+               part_dec_in_flight(part, rw);
 
                part_stat_unlock();
        }
@@ -1735,6 +1829,11 @@ struct request *blk_peek_request(struct request_queue *q)
                        break;
                } else if (ret == BLKPREP_KILL) {
                        rq->cmd_flags |= REQ_QUIET;
+                       /*
+                        * Mark this request as started so we don't trigger
+                        * any debug logic in the end I/O path.
+                        */
+                       blk_start_request(rq);
                        __blk_end_request_all(rq, -EIO);
                } else {
                        printk(KERN_ERR "%s: bad return=%d\n", __func__, ret);
@@ -1760,8 +1859,15 @@ void blk_dequeue_request(struct request *rq)
         * and to it is freed is accounted as io that is in progress at
         * the driver side.
         */
-       if (blk_account_rq(rq))
-               q->in_flight++;
+       if (blk_account_rq(rq)) {
+               q->in_flight[rq_is_sync(rq)]++;
+               /*
+                * Mark this device as supporting hardware queuing, if
+                * we have more IOs in flight than 4.
+                */
+               if (!blk_queue_queuing(q) && queue_in_flight(q) > 4)
+                       set_bit(QUEUE_FLAG_CQ, &q->queue_flags);
+       }
 }
 
 /**
@@ -1783,9 +1889,13 @@ void blk_start_request(struct request *req)
        blk_dequeue_request(req);
 
        /*
-        * We are now handing the request to the hardware, add the
-        * timeout handler.
+        * We are now handing the request to the hardware, initialize
+        * resid_len to full count and add the timeout handler.
         */
+       req->resid_len = blk_rq_bytes(req);
+       if (unlikely(blk_bidi_rq(req)))
+               req->next_rq->resid_len = blk_rq_bytes(req->next_rq);
+
        blk_add_timer(req);
 }
 EXPORT_SYMBOL(blk_start_request);
@@ -1818,14 +1928,14 @@ EXPORT_SYMBOL(blk_fetch_request);
 
 /**
  * blk_update_request - Special helper function for request stacking drivers
- * @rq:              the request being processed
+ * @req:      the request being processed
  * @error:    %0 for success, < %0 for error
- * @nr_bytes: number of bytes to complete @rq
+ * @nr_bytes: number of bytes to complete @req
  *
  * Description:
- *     Ends I/O on a number of bytes attached to @rq, but doesn't complete
- *     the request structure even if @rq doesn't have leftover.
- *     If @rq has leftover, sets it up for the next range of segments.
+ *     Ends I/O on a number of bytes attached to @req, but doesn't complete
+ *     the request structure even if @req doesn't have leftover.
+ *     If @req has leftover, sets it up for the next range of segments.
  *
  *     This special helper function is only for request stacking drivers
  *     (e.g. request-based dm) so that they can handle partial completion.
@@ -1880,10 +1990,10 @@ bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)
                } else {
                        int idx = bio->bi_idx + next_idx;
 
-                       if (unlikely(bio->bi_idx >= bio->bi_vcnt)) {
+                       if (unlikely(idx >= bio->bi_vcnt)) {
                                blk_dump_rq_flags(req, "__end_that");
                                printk(KERN_ERR "%s: bio idx %d >= vcnt %d\n",
-                                      __func__, bio->bi_idx, bio->bi_vcnt);
+                                      __func__, idx, bio->bi_vcnt);
                                break;
                        }
 
@@ -1949,6 +2059,12 @@ bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)
        if (blk_fs_request(req) || blk_discard_rq(req))
                req->__sector += total_bytes >> 9;
 
+       /* mixed attributes always follow the first bio */
+       if (req->cmd_flags & REQ_MIXED_MERGE) {
+               req->cmd_flags &= ~REQ_FAILFAST_MASK;
+               req->cmd_flags |= req->bio->bi_rw & REQ_FAILFAST_MASK;
+       }
+
        /*
         * If total number of sectors is less than the first segment
         * size, something has gone terribly wrong.
@@ -1987,11 +2103,11 @@ static bool blk_update_bidi_request(struct request *rq, int error,
  */
 static void blk_finish_request(struct request *req, int error)
 {
-       BUG_ON(blk_queued_rq(req));
-
        if (blk_rq_tagged(req))
                blk_queue_end_tag(req->q, req);
 
+       BUG_ON(blk_queued_rq(req));
+
        if (unlikely(laptop_mode) && blk_fs_request(req))
                laptop_io_completion();
 
@@ -2026,8 +2142,8 @@ static void blk_finish_request(struct request *req, int error)
  *     %false - we are done with this request
  *     %true  - still buffers pending for this request
  **/
-bool blk_end_bidi_request(struct request *rq, int error,
-                         unsigned int nr_bytes, unsigned int bidi_bytes)
+static bool blk_end_bidi_request(struct request *rq, int error,
+                                unsigned int nr_bytes, unsigned int bidi_bytes)
 {
        struct request_queue *q = rq->q;
        unsigned long flags;
@@ -2041,7 +2157,6 @@ bool blk_end_bidi_request(struct request *rq, int error,
 
        return false;
 }
-EXPORT_SYMBOL_GPL(blk_end_bidi_request);
 
 /**
  * __blk_end_bidi_request - Complete a bidi request with queue lock held
@@ -2058,8 +2173,8 @@ EXPORT_SYMBOL_GPL(blk_end_bidi_request);
  *     %false - we are done with this request
  *     %true  - still buffers pending for this request
  **/
-bool __blk_end_bidi_request(struct request *rq, int error,
-                           unsigned int nr_bytes, unsigned int bidi_bytes)
+static bool __blk_end_bidi_request(struct request *rq, int error,
+                                  unsigned int nr_bytes, unsigned int bidi_bytes)
 {
        if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
                return true;
@@ -2068,14 +2183,169 @@ bool __blk_end_bidi_request(struct request *rq, int error,
 
        return false;
 }
-EXPORT_SYMBOL_GPL(__blk_end_bidi_request);
+
+/**
+ * blk_end_request - Helper function for drivers to complete the request.
+ * @rq:       the request being processed
+ * @error:    %0 for success, < %0 for error
+ * @nr_bytes: number of bytes to complete
+ *
+ * Description:
+ *     Ends I/O on a number of bytes attached to @rq.
+ *     If @rq has leftover, sets it up for the next range of segments.
+ *
+ * Return:
+ *     %false - we are done with this request
+ *     %true  - still buffers pending for this request
+ **/
+bool blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
+{
+       return blk_end_bidi_request(rq, error, nr_bytes, 0);
+}
+EXPORT_SYMBOL(blk_end_request);
+
+/**
+ * blk_end_request_all - Helper function for drives to finish the request.
+ * @rq: the request to finish
+ * @error: %0 for success, < %0 for error
+ *
+ * Description:
+ *     Completely finish @rq.
+ */
+void blk_end_request_all(struct request *rq, int error)
+{
+       bool pending;
+       unsigned int bidi_bytes = 0;
+
+       if (unlikely(blk_bidi_rq(rq)))
+               bidi_bytes = blk_rq_bytes(rq->next_rq);
+
+       pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
+       BUG_ON(pending);
+}
+EXPORT_SYMBOL(blk_end_request_all);
+
+/**
+ * blk_end_request_cur - Helper function to finish the current request chunk.
+ * @rq: the request to finish the current chunk for
+ * @error: %0 for success, < %0 for error
+ *
+ * Description:
+ *     Complete the current consecutively mapped chunk from @rq.
+ *
+ * Return:
+ *     %false - we are done with this request
+ *     %true  - still buffers pending for this request
+ */
+bool blk_end_request_cur(struct request *rq, int error)
+{
+       return blk_end_request(rq, error, blk_rq_cur_bytes(rq));
+}
+EXPORT_SYMBOL(blk_end_request_cur);
+
+/**
+ * blk_end_request_err - Finish a request till the next failure boundary.
+ * @rq: the request to finish till the next failure boundary for
+ * @error: must be negative errno
+ *
+ * Description:
+ *     Complete @rq till the next failure boundary.
+ *
+ * Return:
+ *     %false - we are done with this request
+ *     %true  - still buffers pending for this request
+ */
+bool blk_end_request_err(struct request *rq, int error)
+{
+       WARN_ON(error >= 0);
+       return blk_end_request(rq, error, blk_rq_err_bytes(rq));
+}
+EXPORT_SYMBOL_GPL(blk_end_request_err);
+
+/**
+ * __blk_end_request - Helper function for drivers to complete the request.
+ * @rq:       the request being processed
+ * @error:    %0 for success, < %0 for error
+ * @nr_bytes: number of bytes to complete
+ *
+ * Description:
+ *     Must be called with queue lock held unlike blk_end_request().
+ *
+ * Return:
+ *     %false - we are done with this request
+ *     %true  - still buffers pending for this request
+ **/
+bool __blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
+{
+       return __blk_end_bidi_request(rq, error, nr_bytes, 0);
+}
+EXPORT_SYMBOL(__blk_end_request);
+
+/**
+ * __blk_end_request_all - Helper function for drives to finish the request.
+ * @rq: the request to finish
+ * @error: %0 for success, < %0 for error
+ *
+ * Description:
+ *     Completely finish @rq.  Must be called with queue lock held.
+ */
+void __blk_end_request_all(struct request *rq, int error)
+{
+       bool pending;
+       unsigned int bidi_bytes = 0;
+
+       if (unlikely(blk_bidi_rq(rq)))
+               bidi_bytes = blk_rq_bytes(rq->next_rq);
+
+       pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
+       BUG_ON(pending);
+}
+EXPORT_SYMBOL(__blk_end_request_all);
+
+/**
+ * __blk_end_request_cur - Helper function to finish the current request chunk.
+ * @rq: the request to finish the current chunk for
+ * @error: %0 for success, < %0 for error
+ *
+ * Description:
+ *     Complete the current consecutively mapped chunk from @rq.  Must
+ *     be called with queue lock held.
+ *
+ * Return:
+ *     %false - we are done with this request
+ *     %true  - still buffers pending for this request
+ */
+bool __blk_end_request_cur(struct request *rq, int error)
+{
+       return __blk_end_request(rq, error, blk_rq_cur_bytes(rq));
+}
+EXPORT_SYMBOL(__blk_end_request_cur);
+
+/**
+ * __blk_end_request_err - Finish a request till the next failure boundary.
+ * @rq: the request to finish till the next failure boundary for
+ * @error: must be negative errno
+ *
+ * Description:
+ *     Complete @rq till the next failure boundary.  Must be called
+ *     with queue lock held.
+ *
+ * Return:
+ *     %false - we are done with this request
+ *     %true  - still buffers pending for this request
+ */
+bool __blk_end_request_err(struct request *rq, int error)
+{
+       WARN_ON(error >= 0);
+       return __blk_end_request(rq, error, blk_rq_err_bytes(rq));
+}
+EXPORT_SYMBOL_GPL(__blk_end_request_err);
 
 void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
                     struct bio *bio)
 {
-       /* Bit 0 (R/W) is identical in rq->cmd_flags and bio->bi_rw, and
-          we want BIO_RW_AHEAD (bit 1) to imply REQ_FAILFAST (bit 1). */
-       rq->cmd_flags |= (bio->bi_rw & 3);
+       /* Bit 0 (R/W) is identical in rq->cmd_flags and bio->bi_rw */
+       rq->cmd_flags |= bio->bi_rw & REQ_RW;
 
        if (bio_has_data(bio)) {
                rq->nr_phys_segments = bio_phys_segments(q, bio);
@@ -2088,6 +2358,25 @@ void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
                rq->rq_disk = bio->bi_bdev->bd_disk;
 }
 
+#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
+/**
+ * rq_flush_dcache_pages - Helper function to flush all pages in a request
+ * @rq: the request to be flushed
+ *
+ * Description:
+ *     Flush all pages in @rq.
+ */
+void rq_flush_dcache_pages(struct request *rq)
+{
+       struct req_iterator iter;
+       struct bio_vec *bvec;
+
+       rq_for_each_segment(bvec, rq, iter)
+               flush_dcache_page(bvec->bv_page);
+}
+EXPORT_SYMBOL_GPL(rq_flush_dcache_pages);
+#endif
+
 /**
  * blk_lld_busy - Check if underlying low-level drivers of a device are busy
  * @q : the queue of the device being checked
@@ -2116,6 +2405,106 @@ int blk_lld_busy(struct request_queue *q)
 }
 EXPORT_SYMBOL_GPL(blk_lld_busy);
 
+/**
+ * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
+ * @rq: the clone request to be cleaned up
+ *
+ * Description:
+ *     Free all bios in @rq for a cloned request.
+ */
+void blk_rq_unprep_clone(struct request *rq)
+{
+       struct bio *bio;
+
+       while ((bio = rq->bio) != NULL) {
+               rq->bio = bio->bi_next;
+
+               bio_put(bio);
+       }
+}
+EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
+
+/*
+ * Copy attributes of the original request to the clone request.
+ * The actual data parts (e.g. ->cmd, ->buffer, ->sense) are not copied.
+ */
+static void __blk_rq_prep_clone(struct request *dst, struct request *src)
+{
+       dst->cpu = src->cpu;
+       dst->cmd_flags = (rq_data_dir(src) | REQ_NOMERGE);
+       dst->cmd_type = src->cmd_type;
+       dst->__sector = blk_rq_pos(src);
+       dst->__data_len = blk_rq_bytes(src);
+       dst->nr_phys_segments = src->nr_phys_segments;
+       dst->ioprio = src->ioprio;
+       dst->extra_len = src->extra_len;
+}
+
+/**
+ * blk_rq_prep_clone - Helper function to setup clone request
+ * @rq: the request to be setup
+ * @rq_src: original request to be cloned
+ * @bs: bio_set that bios for clone are allocated from
+ * @gfp_mask: memory allocation mask for bio
+ * @bio_ctr: setup function to be called for each clone bio.
+ *           Returns %0 for success, non %0 for failure.
+ * @data: private data to be passed to @bio_ctr
+ *
+ * Description:
+ *     Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
+ *     The actual data parts of @rq_src (e.g. ->cmd, ->buffer, ->sense)
+ *     are not copied, and copying such parts is the caller's responsibility.
+ *     Also, pages which the original bios are pointing to are not copied
+ *     and the cloned bios just point same pages.
+ *     So cloned bios must be completed before original bios, which means
+ *     the caller must complete @rq before @rq_src.
+ */
+int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
+                     struct bio_set *bs, gfp_t gfp_mask,
+                     int (*bio_ctr)(struct bio *, struct bio *, void *),
+                     void *data)
+{
+       struct bio *bio, *bio_src;
+
+       if (!bs)
+               bs = fs_bio_set;
+
+       blk_rq_init(NULL, rq);
+
+       __rq_for_each_bio(bio_src, rq_src) {
+               bio = bio_alloc_bioset(gfp_mask, bio_src->bi_max_vecs, bs);
+               if (!bio)
+                       goto free_and_out;
+
+               __bio_clone(bio, bio_src);
+
+               if (bio_integrity(bio_src) &&
+                   bio_integrity_clone(bio, bio_src, gfp_mask, bs))
+                       goto free_and_out;
+
+               if (bio_ctr && bio_ctr(bio, bio_src, data))
+                       goto free_and_out;
+
+               if (rq->bio) {
+                       rq->biotail->bi_next = bio;
+                       rq->biotail = bio;
+               } else
+                       rq->bio = rq->biotail = bio;
+       }
+
+       __blk_rq_prep_clone(rq, rq_src);
+
+       return 0;
+
+free_and_out:
+       if (bio)
+               bio_free(bio, bs);
+       blk_rq_unprep_clone(rq);
+
+       return -ENOMEM;
+}
+EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
+
 int kblockd_schedule_work(struct request_queue *q, struct work_struct *work)
 {
        return queue_work(kblockd_workqueue, work);