Driver core: convert block from raw kobjects to core devices
[linux-2.6.git] / block / ll_rw_blk.c
index e0608c4..3887b2a 100644 (file)
@@ -30,6 +30,7 @@
 #include <linux/cpu.h>
 #include <linux/blktrace_api.h>
 #include <linux/fault-inject.h>
+#include <linux/scatterlist.h>
 
 /*
  * for max sense size
@@ -38,7 +39,7 @@
 
 static void blk_unplug_work(struct work_struct *work);
 static void blk_unplug_timeout(unsigned long data);
-static void drive_stat_acct(struct request *rq, int nr_sectors, int new_io);
+static void drive_stat_acct(struct request *rq, int new_io);
 static void init_request_from_bio(struct request *req, struct bio *bio);
 static int __make_request(struct request_queue *q, struct bio *bio);
 static struct io_context *current_io_context(gfp_t gfp_flags, int node);
@@ -304,23 +305,6 @@ int blk_queue_ordered(struct request_queue *q, unsigned ordered,
 
 EXPORT_SYMBOL(blk_queue_ordered);
 
-/**
- * blk_queue_issue_flush_fn - set function for issuing a flush
- * @q:     the request queue
- * @iff:   the function to be called issuing the flush
- *
- * Description:
- *   If a driver supports issuing a flush command, the support is notified
- *   to the block layer by defining it through this call.
- *
- **/
-void blk_queue_issue_flush_fn(struct request_queue *q, issue_flush_fn *iff)
-{
-       q->issue_flush_fn = iff;
-}
-
-EXPORT_SYMBOL(blk_queue_issue_flush_fn);
-
 /*
  * Cache flushing for ordered writes handling
  */
@@ -377,10 +361,12 @@ void blk_ordered_complete_seq(struct request_queue *q, unsigned seq, int error)
        /*
         * Okay, sequence complete.
         */
-       rq = q->orig_bar_rq;
-       uptodate = q->orderr ? q->orderr : 1;
+       uptodate = 1;
+       if (q->orderr)
+               uptodate = q->orderr;
 
        q->ordseq = 0;
+       rq = q->orig_bar_rq;
 
        end_that_request_first(rq, uptodate, rq->hard_nr_sectors);
        end_that_request_last(rq, uptodate);
@@ -431,7 +417,6 @@ static void queue_flush(struct request_queue *q, unsigned which)
 static inline struct request *start_ordered(struct request_queue *q,
                                            struct request *rq)
 {
-       q->bi_size = 0;
        q->orderr = 0;
        q->ordered = q->next_ordered;
        q->ordseq |= QUEUE_ORDSEQ_STARTED;
@@ -446,7 +431,8 @@ static inline struct request *start_ordered(struct request_queue *q,
        rq_init(q, rq);
        if (bio_data_dir(q->orig_bar_rq->bio) == WRITE)
                rq->cmd_flags |= REQ_RW;
-       rq->cmd_flags |= q->ordered & QUEUE_ORDERED_FUA ? REQ_FUA : 0;
+       if (q->ordered & QUEUE_ORDERED_FUA)
+               rq->cmd_flags |= REQ_FUA;
        rq->elevator_private = NULL;
        rq->elevator_private2 = NULL;
        init_request_from_bio(rq, q->orig_bar_rq->bio);
@@ -456,9 +442,12 @@ static inline struct request *start_ordered(struct request_queue *q,
         * Queue ordered sequence.  As we stack them at the head, we
         * need to queue in reverse order.  Note that we rely on that
         * no fs request uses ELEVATOR_INSERT_FRONT and thus no fs
-        * request gets inbetween ordered sequence.
+        * request gets inbetween ordered sequence. If this request is
+        * an empty barrier, we don't need to do a postflush ever since
+        * there will be no data written between the pre and post flush.
+        * Hence a single flush will suffice.
         */
-       if (q->ordered & QUEUE_ORDERED_POSTFLUSH)
+       if ((q->ordered & QUEUE_ORDERED_POSTFLUSH) && !blk_empty_barrier(rq))
                queue_flush(q, QUEUE_ORDERED_POSTFLUSH);
        else
                q->ordseq |= QUEUE_ORDSEQ_POSTFLUSH;
@@ -482,7 +471,7 @@ static inline struct request *start_ordered(struct request_queue *q,
 int blk_do_ordered(struct request_queue *q, struct request **rqp)
 {
        struct request *rq = *rqp;
-       int is_barrier = blk_fs_request(rq) && blk_barrier_rq(rq);
+       const int is_barrier = blk_fs_request(rq) && blk_barrier_rq(rq);
 
        if (!q->ordseq) {
                if (!is_barrier)
@@ -528,56 +517,36 @@ int blk_do_ordered(struct request_queue *q, struct request **rqp)
        return 1;
 }
 
-static int flush_dry_bio_endio(struct bio *bio, unsigned int bytes, int error)
-{
-       struct request_queue *q = bio->bi_private;
-
-       /*
-        * This is dry run, restore bio_sector and size.  We'll finish
-        * this request again with the original bi_end_io after an
-        * error occurs or post flush is complete.
-        */
-       q->bi_size += bytes;
-
-       if (bio->bi_size)
-               return 1;
-
-       /* Reset bio */
-       set_bit(BIO_UPTODATE, &bio->bi_flags);
-       bio->bi_size = q->bi_size;
-       bio->bi_sector -= (q->bi_size >> 9);
-       q->bi_size = 0;
-
-       return 0;
-}
-
-static int ordered_bio_endio(struct request *rq, struct bio *bio,
-                            unsigned int nbytes, int error)
+static void req_bio_endio(struct request *rq, struct bio *bio,
+                         unsigned int nbytes, int error)
 {
        struct request_queue *q = rq->q;
-       bio_end_io_t *endio;
-       void *private;
 
-       if (&q->bar_rq != rq)
-               return 0;
+       if (&q->bar_rq != rq) {
+               if (error)
+                       clear_bit(BIO_UPTODATE, &bio->bi_flags);
+               else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
+                       error = -EIO;
 
-       /*
-        * Okay, this is the barrier request in progress, dry finish it.
-        */
-       if (error && !q->orderr)
-               q->orderr = error;
-
-       endio = bio->bi_end_io;
-       private = bio->bi_private;
-       bio->bi_end_io = flush_dry_bio_endio;
-       bio->bi_private = q;
-
-       bio_endio(bio, nbytes, error);
+               if (unlikely(nbytes > bio->bi_size)) {
+                       printk("%s: want %u bytes done, only %u left\n",
+                              __FUNCTION__, nbytes, bio->bi_size);
+                       nbytes = bio->bi_size;
+               }
 
-       bio->bi_end_io = endio;
-       bio->bi_private = private;
+               bio->bi_size -= nbytes;
+               bio->bi_sector += (nbytes >> 9);
+               if (bio->bi_size == 0)
+                       bio_endio(bio, error);
+       } else {
 
-       return 1;
+               /*
+                * Okay, this is the barrier request in progress, just
+                * record the error;
+                */
+               if (error && !q->orderr)
+                       q->orderr = error;
+       }
 }
 
 /**
@@ -822,7 +791,6 @@ static int __blk_free_tags(struct blk_queue_tag *bqt)
        retval = atomic_dec_and_test(&bqt->refcnt);
        if (retval) {
                BUG_ON(bqt->busy);
-               BUG_ON(!list_empty(&bqt->busy_list));
 
                kfree(bqt->tag_index);
                bqt->tag_index = NULL;
@@ -934,7 +902,6 @@ static struct blk_queue_tag *__blk_queue_init_tags(struct request_queue *q,
        if (init_tag_map(q, tags, depth))
                goto fail;
 
-       INIT_LIST_HEAD(&tags->busy_list);
        tags->busy = 0;
        atomic_set(&tags->refcnt, 1);
        return tags;
@@ -985,6 +952,7 @@ int blk_queue_init_tags(struct request_queue *q, int depth,
         */
        q->queue_tags = tags;
        q->queue_flags |= (1 << QUEUE_FLAG_QUEUED);
+       INIT_LIST_HEAD(&q->tag_busy_list);
        return 0;
 fail:
        kfree(tags);
@@ -1088,18 +1056,16 @@ void blk_queue_end_tag(struct request_queue *q, struct request *rq)
 
        bqt->tag_index[tag] = NULL;
 
-       /*
-        * We use test_and_clear_bit's memory ordering properties here.
-        * The tag_map bit acts as a lock for tag_index[bit], so we need
-        * a barrer before clearing the bit (precisely: release semantics).
-        * Could use clear_bit_unlock when it is merged.
-        */
-       if (unlikely(!test_and_clear_bit(tag, bqt->tag_map))) {
+       if (unlikely(!test_bit(tag, bqt->tag_map))) {
                printk(KERN_ERR "%s: attempt to clear non-busy tag (%d)\n",
                       __FUNCTION__, tag);
                return;
        }
-
+       /*
+        * The tag_map bit acts as a lock for tag_index[bit], so we need
+        * unlock memory barrier semantics.
+        */
+       clear_bit_unlock(tag, bqt->tag_map);
        bqt->busy--;
 }
 
@@ -1145,17 +1111,17 @@ int blk_queue_start_tag(struct request_queue *q, struct request *rq)
                if (tag >= bqt->max_depth)
                        return 1;
 
-       } while (test_and_set_bit(tag, bqt->tag_map));
+       } while (test_and_set_bit_lock(tag, bqt->tag_map));
        /*
-        * We rely on test_and_set_bit providing lock memory ordering semantics
-        * (could use test_and_set_bit_lock when it is merged).
+        * We need lock ordering semantics given by test_and_set_bit_lock.
+        * See blk_queue_end_tag for details.
         */
 
        rq->cmd_flags |= REQ_QUEUED;
        rq->tag = tag;
        bqt->tag_index[tag] = rq;
        blkdev_dequeue_request(rq);
-       list_add(&rq->queuelist, &bqt->busy_list);
+       list_add(&rq->queuelist, &q->tag_busy_list);
        bqt->busy++;
        return 0;
 }
@@ -1176,24 +1142,10 @@ EXPORT_SYMBOL(blk_queue_start_tag);
  **/
 void blk_queue_invalidate_tags(struct request_queue *q)
 {
-       struct blk_queue_tag *bqt = q->queue_tags;
        struct list_head *tmp, *n;
-       struct request *rq;
 
-       list_for_each_safe(tmp, n, &bqt->busy_list) {
-               rq = list_entry_rq(tmp);
-
-               if (rq->tag == -1) {
-                       printk(KERN_ERR
-                              "%s: bad tag found on list\n", __FUNCTION__);
-                       list_del_init(&rq->queuelist);
-                       rq->cmd_flags &= ~REQ_QUEUED;
-               } else
-                       blk_queue_end_tag(q, rq);
-
-               rq->cmd_flags &= ~REQ_STARTED;
-               __elv_add_request(q, rq, ELEVATOR_INSERT_BACK, 0);
-       }
+       list_for_each_safe(tmp, n, &q->tag_busy_list)
+               blk_requeue_request(q, list_entry_rq(tmp));
 }
 
 EXPORT_SYMBOL(blk_queue_invalidate_tags);
@@ -1350,10 +1302,11 @@ static int blk_hw_contig_segment(struct request_queue *q, struct bio *bio,
  * must make sure sg can hold rq->nr_phys_segments entries
  */
 int blk_rq_map_sg(struct request_queue *q, struct request *rq,
-                 struct scatterlist *sg)
+                 struct scatterlist *sglist)
 {
        struct bio_vec *bvec, *bvprv;
        struct req_iterator iter;
+       struct scatterlist *sg;
        int nsegs, cluster;
 
        nsegs = 0;
@@ -1363,31 +1316,48 @@ int blk_rq_map_sg(struct request_queue *q, struct request *rq,
         * for each bio in rq
         */
        bvprv = NULL;
+       sg = NULL;
        rq_for_each_segment(bvec, rq, iter) {
-                       int nbytes = bvec->bv_len;
+               int nbytes = bvec->bv_len;
 
-                       if (bvprv && cluster) {
-                               if (sg[nsegs - 1].length + nbytes > q->max_segment_size)
-                                       goto new_segment;
+               if (bvprv && cluster) {
+                       if (sg->length + nbytes > q->max_segment_size)
+                               goto new_segment;
 
-                               if (!BIOVEC_PHYS_MERGEABLE(bvprv, bvec))
-                                       goto new_segment;
-                               if (!BIOVEC_SEG_BOUNDARY(q, bvprv, bvec))
-                                       goto new_segment;
+                       if (!BIOVEC_PHYS_MERGEABLE(bvprv, bvec))
+                               goto new_segment;
+                       if (!BIOVEC_SEG_BOUNDARY(q, bvprv, bvec))
+                               goto new_segment;
 
-                               sg[nsegs - 1].length += nbytes;
-                       } else {
+                       sg->length += nbytes;
+               } else {
 new_segment:
-                               memset(&sg[nsegs],0,sizeof(struct scatterlist));
-                               sg[nsegs].page = bvec->bv_page;
-                               sg[nsegs].length = nbytes;
-                               sg[nsegs].offset = bvec->bv_offset;
-
-                               nsegs++;
+                       if (!sg)
+                               sg = sglist;
+                       else {
+                               /*
+                                * If the driver previously mapped a shorter
+                                * list, we could see a termination bit
+                                * prematurely unless it fully inits the sg
+                                * table on each mapping. We KNOW that there
+                                * must be more entries here or the driver
+                                * would be buggy, so force clear the
+                                * termination bit to avoid doing a full
+                                * sg_init_table() in drivers for each command.
+                                */
+                               sg->page_link &= ~0x02;
+                               sg = sg_next(sg);
                        }
-                       bvprv = bvec;
+
+                       sg_set_page(sg, bvec->bv_page, nbytes, bvec->bv_offset);
+                       nsegs++;
+               }
+               bvprv = bvec;
        } /* segments in rq */
 
+       if (sg)
+               sg_mark_end(sg);
+
        return nsegs;
 }
 
@@ -1651,15 +1621,7 @@ static void blk_backing_dev_unplug(struct backing_dev_info *bdi,
 {
        struct request_queue *q = bdi->unplug_io_data;
 
-       /*
-        * devices don't necessarily have an ->unplug_fn defined
-        */
-       if (q->unplug_fn) {
-               blk_add_trace_pdu_int(q, BLK_TA_UNPLUG_IO, NULL,
-                                       q->rq.count[READ] + q->rq.count[WRITE]);
-
-               q->unplug_fn(q);
-       }
+       blk_unplug(q);
 }
 
 static void blk_unplug_work(struct work_struct *work)
@@ -1683,6 +1645,20 @@ static void blk_unplug_timeout(unsigned long data)
        kblockd_schedule_work(&q->unplug_work);
 }
 
+void blk_unplug(struct request_queue *q)
+{
+       /*
+        * devices don't necessarily have an ->unplug_fn defined
+        */
+       if (q->unplug_fn) {
+               blk_add_trace_pdu_int(q, BLK_TA_UNPLUG_IO, NULL,
+                                       q->rq.count[READ] + q->rq.count[WRITE]);
+
+               q->unplug_fn(q);
+       }
+}
+EXPORT_SYMBOL(blk_unplug);
+
 /**
  * blk_start_queue - restart a previously stopped queue
  * @q:    The &struct request_queue in question
@@ -1751,6 +1727,7 @@ EXPORT_SYMBOL(blk_stop_queue);
 void blk_sync_queue(struct request_queue *q)
 {
        del_timer_sync(&q->unplug_timer);
+       kblockd_flush_work(&q->unplug_work);
 }
 EXPORT_SYMBOL(blk_sync_queue);
 
@@ -1814,6 +1791,7 @@ static void blk_release_queue(struct kobject *kobj)
 
        blk_trace_shutdown(q);
 
+       bdi_destroy(&q->backing_dev_info);
        kmem_cache_free(requestq_cachep, q);
 }
 
@@ -1867,21 +1845,27 @@ static struct kobj_type queue_ktype;
 struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
 {
        struct request_queue *q;
+       int err;
 
        q = kmem_cache_alloc_node(requestq_cachep,
                                gfp_mask | __GFP_ZERO, node_id);
        if (!q)
                return NULL;
 
+       q->backing_dev_info.unplug_io_fn = blk_backing_dev_unplug;
+       q->backing_dev_info.unplug_io_data = q;
+       err = bdi_init(&q->backing_dev_info);
+       if (err) {
+               kmem_cache_free(requestq_cachep, q);
+               return NULL;
+       }
+
        init_timer(&q->unplug_timer);
 
-       snprintf(q->kobj.name, KOBJ_NAME_LEN, "%s", "queue");
+       kobject_set_name(&q->kobj, "%s", "queue");
        q->kobj.ktype = &queue_ktype;
        kobject_init(&q->kobj);
 
-       q->backing_dev_info.unplug_io_fn = blk_backing_dev_unplug;
-       q->backing_dev_info.unplug_io_data = q;
-
        mutex_init(&q->sysfs_lock);
 
        return q;
@@ -2347,7 +2331,7 @@ void blk_insert_request(struct request_queue *q, struct request *rq,
        if (blk_rq_tagged(rq))
                blk_queue_end_tag(q, rq);
 
-       drive_stat_acct(rq, rq->nr_sectors, 1);
+       drive_stat_acct(rq, 1);
        __elv_add_request(q, rq, where, 0);
        blk_start_queueing(q);
        spin_unlock_irqrestore(q->queue_lock, flags);
@@ -2422,7 +2406,7 @@ static int __blk_rq_map_user(struct request_queue *q, struct request *rq,
                return bio->bi_size;
 
        /* if it was boucned we must call the end io function */
-       bio_endio(bio, bio->bi_size, 0);
+       bio_endio(bio, 0);
        __blk_rq_unmap_user(orig_bio);
        bio_put(bio);
        return ret;
@@ -2531,7 +2515,7 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
                return PTR_ERR(bio);
 
        if (bio->bi_size != len) {
-               bio_endio(bio, bio->bi_size, 0);
+               bio_endio(bio, 0);
                bio_unmap_user(bio);
                return -EINVAL;
        }
@@ -2681,6 +2665,14 @@ int blk_execute_rq(struct request_queue *q, struct gendisk *bd_disk,
 
 EXPORT_SYMBOL(blk_execute_rq);
 
+static void bio_end_empty_barrier(struct bio *bio, int err)
+{
+       if (err)
+               clear_bit(BIO_UPTODATE, &bio->bi_flags);
+
+       complete(bio->bi_private);
+}
+
 /**
  * blkdev_issue_flush - queue a flush
  * @bdev:      blockdev to issue flush for
@@ -2693,7 +2685,10 @@ EXPORT_SYMBOL(blk_execute_rq);
  */
 int blkdev_issue_flush(struct block_device *bdev, sector_t *error_sector)
 {
+       DECLARE_COMPLETION_ONSTACK(wait);
        struct request_queue *q;
+       struct bio *bio;
+       int ret;
 
        if (bdev->bd_disk == NULL)
                return -ENXIO;
@@ -2701,15 +2696,37 @@ int blkdev_issue_flush(struct block_device *bdev, sector_t *error_sector)
        q = bdev_get_queue(bdev);
        if (!q)
                return -ENXIO;
-       if (!q->issue_flush_fn)
-               return -EOPNOTSUPP;
 
-       return q->issue_flush_fn(q, bdev->bd_disk, error_sector);
+       bio = bio_alloc(GFP_KERNEL, 0);
+       if (!bio)
+               return -ENOMEM;
+
+       bio->bi_end_io = bio_end_empty_barrier;
+       bio->bi_private = &wait;
+       bio->bi_bdev = bdev;
+       submit_bio(1 << BIO_RW_BARRIER, bio);
+
+       wait_for_completion(&wait);
+
+       /*
+        * The driver must store the error location in ->bi_sector, if
+        * it supports it. For non-stacked drivers, this should be copied
+        * from rq->sector.
+        */
+       if (error_sector)
+               *error_sector = bio->bi_sector;
+
+       ret = 0;
+       if (!bio_flagged(bio, BIO_UPTODATE))
+               ret = -EIO;
+
+       bio_put(bio);
+       return ret;
 }
 
 EXPORT_SYMBOL(blkdev_issue_flush);
 
-static void drive_stat_acct(struct request *rq, int nr_sectors, int new_io)
+static void drive_stat_acct(struct request *rq, int new_io)
 {
        int rw = rq_data_dir(rq);
 
@@ -2731,7 +2748,7 @@ static void drive_stat_acct(struct request *rq, int nr_sectors, int new_io)
  */
 static inline void add_request(struct request_queue * q, struct request * req)
 {
-       drive_stat_acct(req, req->nr_sectors, 1);
+       drive_stat_acct(req, 1);
 
        /*
         * elevator indicated where it wants this request to be
@@ -2988,7 +3005,7 @@ static int __make_request(struct request_queue *q, struct bio *bio)
                        req->biotail = bio;
                        req->nr_sectors = req->hard_nr_sectors += nr_sectors;
                        req->ioprio = ioprio_best(req->ioprio, prio);
-                       drive_stat_acct(req, nr_sectors, 0);
+                       drive_stat_acct(req, 0);
                        if (!attempt_back_merge(q, req))
                                elv_merged_request(q, req, el_ret);
                        goto out;
@@ -3015,7 +3032,7 @@ static int __make_request(struct request_queue *q, struct bio *bio)
                        req->sector = req->hard_sector = bio->bi_sector;
                        req->nr_sectors = req->hard_nr_sectors += nr_sectors;
                        req->ioprio = ioprio_best(req->ioprio, prio);
-                       drive_stat_acct(req, nr_sectors, 0);
+                       drive_stat_acct(req, 0);
                        if (!attempt_front_merge(q, req))
                                elv_merged_request(q, req, el_ret);
                        goto out;
@@ -3061,7 +3078,7 @@ out:
        return 0;
 
 end_io:
-       bio_endio(bio, nr_sectors << 9, err);
+       bio_endio(bio, err);
        return 0;
 }
 
@@ -3072,7 +3089,7 @@ static inline void blk_partition_remap(struct bio *bio)
 {
        struct block_device *bdev = bio->bi_bdev;
 
-       if (bdev != bdev->bd_contains) {
+       if (bio_sectors(bio) && bdev != bdev->bd_contains) {
                struct hd_struct *p = bdev->bd_part;
                const int rw = bio_data_dir(bio);
 
@@ -3138,6 +3155,35 @@ static inline int should_fail_request(struct bio *bio)
 
 #endif /* CONFIG_FAIL_MAKE_REQUEST */
 
+/*
+ * Check whether this bio extends beyond the end of the device.
+ */
+static inline int bio_check_eod(struct bio *bio, unsigned int nr_sectors)
+{
+       sector_t maxsector;
+
+       if (!nr_sectors)
+               return 0;
+
+       /* Test device or partition size, when known. */
+       maxsector = bio->bi_bdev->bd_inode->i_size >> 9;
+       if (maxsector) {
+               sector_t sector = bio->bi_sector;
+
+               if (maxsector < nr_sectors || maxsector - nr_sectors < sector) {
+                       /*
+                        * This may well happen - the kernel calls bread()
+                        * without checking the size of the device, e.g., when
+                        * mounting a device.
+                        */
+                       handle_bad_sector(bio);
+                       return 1;
+               }
+       }
+
+       return 0;
+}
+
 /**
  * generic_make_request: hand a buffer to its device driver for I/O
  * @bio:  The bio describing the location in memory and on the device.
@@ -3165,27 +3211,15 @@ static inline int should_fail_request(struct bio *bio)
 static inline void __generic_make_request(struct bio *bio)
 {
        struct request_queue *q;
-       sector_t maxsector;
        sector_t old_sector;
        int ret, nr_sectors = bio_sectors(bio);
        dev_t old_dev;
+       int err = -EIO;
 
        might_sleep();
-       /* Test device or partition size, when known. */
-       maxsector = bio->bi_bdev->bd_inode->i_size >> 9;
-       if (maxsector) {
-               sector_t sector = bio->bi_sector;
 
-               if (maxsector < nr_sectors || maxsector - nr_sectors < sector) {
-                       /*
-                        * This may well happen - the kernel calls bread()
-                        * without checking the size of the device, e.g., when
-                        * mounting a device.
-                        */
-                       handle_bad_sector(bio);
-                       goto end_io;
-               }
-       }
+       if (bio_check_eod(bio, nr_sectors))
+               goto end_io;
 
        /*
         * Resolve the mapping until finished. (drivers are
@@ -3208,11 +3242,11 @@ static inline void __generic_make_request(struct bio *bio)
                                bdevname(bio->bi_bdev, b),
                                (long long) bio->bi_sector);
 end_io:
-                       bio_endio(bio, bio->bi_size, -EIO);
+                       bio_endio(bio, err);
                        break;
                }
 
-               if (unlikely(bio_sectors(bio) > q->max_hw_sectors)) {
+               if (unlikely(nr_sectors > q->max_hw_sectors)) {
                        printk("bio too big device %s (%u > %u)\n", 
                                bdevname(bio->bi_bdev, b),
                                bio_sectors(bio),
@@ -3233,7 +3267,7 @@ end_io:
                blk_partition_remap(bio);
 
                if (old_sector != -1)
-                       blk_add_trace_remap(q, bio, old_dev, bio->bi_sector, 
+                       blk_add_trace_remap(q, bio, old_dev, bio->bi_sector,
                                            old_sector);
 
                blk_add_trace_bio(q, bio, BLK_TA_QUEUE);
@@ -3241,20 +3275,11 @@ end_io:
                old_sector = bio->bi_sector;
                old_dev = bio->bi_bdev->bd_dev;
 
-               maxsector = bio->bi_bdev->bd_inode->i_size >> 9;
-               if (maxsector) {
-                       sector_t sector = bio->bi_sector;
-
-                       if (maxsector < nr_sectors ||
-                                       maxsector - nr_sectors < sector) {
-                               /*
-                                * This may well happen - partitions are not
-                                * checked to make sure they are within the size
-                                * of the whole device.
-                                */
-                               handle_bad_sector(bio);
-                               goto end_io;
-                       }
+               if (bio_check_eod(bio, nr_sectors))
+                       goto end_io;
+               if (bio_empty_barrier(bio) && !q->prepare_flush_fn) {
+                       err = -EOPNOTSUPP;
+                       goto end_io;
                }
 
                ret = q->make_request_fn(q, bio);
@@ -3328,23 +3353,32 @@ void submit_bio(int rw, struct bio *bio)
 {
        int count = bio_sectors(bio);
 
-       BIO_BUG_ON(!bio->bi_size);
-       BIO_BUG_ON(!bio->bi_io_vec);
        bio->bi_rw |= rw;
-       if (rw & WRITE) {
-               count_vm_events(PGPGOUT, count);
-       } else {
-               task_io_account_read(bio->bi_size);
-               count_vm_events(PGPGIN, count);
-       }
 
-       if (unlikely(block_dump)) {
-               char b[BDEVNAME_SIZE];
-               printk(KERN_DEBUG "%s(%d): %s block %Lu on %s\n",
-                       current->comm, current->pid,
-                       (rw & WRITE) ? "WRITE" : "READ",
-                       (unsigned long long)bio->bi_sector,
-                       bdevname(bio->bi_bdev,b));
+       /*
+        * If it's a regular read/write or a barrier with data attached,
+        * go through the normal accounting stuff before submission.
+        */
+       if (!bio_empty_barrier(bio)) {
+
+               BIO_BUG_ON(!bio->bi_size);
+               BIO_BUG_ON(!bio->bi_io_vec);
+
+               if (rw & WRITE) {
+                       count_vm_events(PGPGOUT, count);
+               } else {
+                       task_io_account_read(bio->bi_size);
+                       count_vm_events(PGPGIN, count);
+               }
+
+               if (unlikely(block_dump)) {
+                       char b[BDEVNAME_SIZE];
+                       printk(KERN_DEBUG "%s(%d): %s block %Lu on %s\n",
+                       current->comm, task_pid_nr(current),
+                               (rw & WRITE) ? "WRITE" : "READ",
+                               (unsigned long long)bio->bi_sector,
+                               bdevname(bio->bi_bdev,b));
+               }
        }
 
        generic_make_request(bio);
@@ -3420,11 +3454,18 @@ static int __end_that_request_first(struct request *req, int uptodate,
        while ((bio = req->bio) != NULL) {
                int nbytes;
 
+               /*
+                * For an empty barrier request, the low level driver must
+                * store a potential error location in ->sector. We pass
+                * that back up in ->bi_sector.
+                */
+               if (blk_empty_barrier(req))
+                       bio->bi_sector = req->sector;
+
                if (nr_bytes >= bio->bi_size) {
                        req->bio = bio->bi_next;
                        nbytes = bio->bi_size;
-                       if (!ordered_bio_endio(req, bio, nbytes, error))
-                               bio_endio(bio, nbytes, error);
+                       req_bio_endio(req, bio, nbytes, error);
                        next_idx = 0;
                        bio_nbytes = 0;
                } else {
@@ -3479,8 +3520,7 @@ static int __end_that_request_first(struct request *req, int uptodate,
         * if the request wasn't completed, update state
         */
        if (bio_nbytes) {
-               if (!ordered_bio_endio(req, bio, bio_nbytes, error))
-                       bio_endio(bio, bio_nbytes, error);
+               req_bio_endio(req, bio, bio_nbytes, error);
                bio->bi_idx += next_idx;
                bio_iovec(bio)->bv_offset += nr_bytes;
                bio_iovec(bio)->bv_len -= nr_bytes;
@@ -3555,7 +3595,7 @@ static void blk_done_softirq(struct softirq_action *h)
        }
 }
 
-static int blk_cpu_notify(struct notifier_block *self, unsigned long action,
+static int __cpuinit blk_cpu_notify(struct notifier_block *self, unsigned long action,
                          void *hcpu)
 {
        /*
@@ -3576,7 +3616,7 @@ static int blk_cpu_notify(struct notifier_block *self, unsigned long action,
 }
 
 
-static struct notifier_block __devinitdata blk_cpu_notifier = {
+static struct notifier_block blk_cpu_notifier __cpuinitdata = {
        .notifier_call  = blk_cpu_notify,
 };
 
@@ -3587,7 +3627,7 @@ static struct notifier_block __devinitdata blk_cpu_notifier = {
  * Description:
  *     Ends all I/O on a request. It does not handle partial completions,
  *     unless the driver actually implements this in its completion callback
- *     through requeueing. Theh actual completion happens out-of-order,
+ *     through requeueing. The actual completion happens out-of-order,
  *     through a softirq handler. The user must have registered a completion
  *     callback through blk_queue_softirq_done().
  **/
@@ -3650,15 +3690,83 @@ void end_that_request_last(struct request *req, int uptodate)
 
 EXPORT_SYMBOL(end_that_request_last);
 
-void end_request(struct request *req, int uptodate)
+static inline void __end_request(struct request *rq, int uptodate,
+                                unsigned int nr_bytes, int dequeue)
 {
-       if (!end_that_request_first(req, uptodate, req->hard_cur_sectors)) {
-               add_disk_randomness(req->rq_disk);
-               blkdev_dequeue_request(req);
-               end_that_request_last(req, uptodate);
+       if (!end_that_request_chunk(rq, uptodate, nr_bytes)) {
+               if (dequeue)
+                       blkdev_dequeue_request(rq);
+               add_disk_randomness(rq->rq_disk);
+               end_that_request_last(rq, uptodate);
        }
 }
 
+static unsigned int rq_byte_size(struct request *rq)
+{
+       if (blk_fs_request(rq))
+               return rq->hard_nr_sectors << 9;
+
+       return rq->data_len;
+}
+
+/**
+ * end_queued_request - end all I/O on a queued request
+ * @rq:                the request being processed
+ * @uptodate:  error value or 0/1 uptodate flag
+ *
+ * Description:
+ *     Ends all I/O on a request, and removes it from the block layer queues.
+ *     Not suitable for normal IO completion, unless the driver still has
+ *     the request attached to the block layer.
+ *
+ **/
+void end_queued_request(struct request *rq, int uptodate)
+{
+       __end_request(rq, uptodate, rq_byte_size(rq), 1);
+}
+EXPORT_SYMBOL(end_queued_request);
+
+/**
+ * end_dequeued_request - end all I/O on a dequeued request
+ * @rq:                the request being processed
+ * @uptodate:  error value or 0/1 uptodate flag
+ *
+ * Description:
+ *     Ends all I/O on a request. The request must already have been
+ *     dequeued using blkdev_dequeue_request(), as is normally the case
+ *     for most drivers.
+ *
+ **/
+void end_dequeued_request(struct request *rq, int uptodate)
+{
+       __end_request(rq, uptodate, rq_byte_size(rq), 0);
+}
+EXPORT_SYMBOL(end_dequeued_request);
+
+
+/**
+ * end_request - end I/O on the current segment of the request
+ * @req:       the request being processed
+ * @uptodate:  error value or 0/1 uptodate flag
+ *
+ * Description:
+ *     Ends I/O on the current segment of a request. If that is the only
+ *     remaining segment, the request is also completed and freed.
+ *
+ *     This is a remnant of how older block drivers handled IO completions.
+ *     Modern drivers typically end IO on the full request in one go, unless
+ *     they have a residual value to account for. For that case this function
+ *     isn't really useful, unless the residual just happens to be the
+ *     full current segment. In other words, don't use this function in new
+ *     code. Either use end_request_completely(), or the
+ *     end_that_request_chunk() (along with end_that_request_last()) for
+ *     partial completions.
+ *
+ **/
+void end_request(struct request *req, int uptodate)
+{
+       __end_request(req, uptodate, req->hard_cur_sectors << 9, 1);
+}
 EXPORT_SYMBOL(end_request);
 
 static void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
@@ -3951,7 +4059,6 @@ queue_max_sectors_store(struct request_queue *q, const char *page, size_t count)
                        max_hw_sectors_kb = q->max_hw_sectors >> 1,
                        page_kb = 1 << (PAGE_CACHE_SHIFT - 10);
        ssize_t ret = queue_var_store(&max_sectors_kb, page, count);
-       int ra_kb;
 
        if (max_sectors_kb > max_hw_sectors_kb || max_sectors_kb < page_kb)
                return -EINVAL;
@@ -3960,14 +4067,6 @@ queue_max_sectors_store(struct request_queue *q, const char *page, size_t count)
         * values synchronously:
         */
        spin_lock_irq(q->queue_lock);
-       /*
-        * Trim readahead window as well, if necessary:
-        */
-       ra_kb = q->backing_dev_info.ra_pages << (PAGE_CACHE_SHIFT - 10);
-       if (ra_kb > max_sectors_kb)
-               q->backing_dev_info.ra_pages =
-                               max_sectors_kb >> (PAGE_CACHE_SHIFT - 10);
-
        q->max_sectors = max_sectors_kb << 1;
        spin_unlock_irq(q->queue_lock);
 
@@ -4083,7 +4182,7 @@ int blk_register_queue(struct gendisk *disk)
        if (!q || !q->request_fn)
                return -ENXIO;
 
-       q->kobj.parent = kobject_get(&disk->kobj);
+       q->kobj.parent = kobject_get(&disk->dev.kobj);
 
        ret = kobject_add(&q->kobj);
        if (ret < 0)
@@ -4110,6 +4209,6 @@ void blk_unregister_queue(struct gendisk *disk)
 
                kobject_uevent(&q->kobj, KOBJ_REMOVE);
                kobject_del(&q->kobj);
-               kobject_put(&disk->kobj);
+               kobject_put(&disk->dev.kobj);
        }
 }