blk-flush: fix invalid BUG_ON in blk_insert_flush
[linux-2.6.git] / block / blk-lib.c
index 0dc4388..2b461b4 100644 (file)
@@ -9,18 +9,20 @@
 
 #include "blk.h"
 
-static void blkdev_discard_end_io(struct bio *bio, int err)
-{
-       if (err) {
-               if (err == -EOPNOTSUPP)
-                       set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
-               clear_bit(BIO_UPTODATE, &bio->bi_flags);
-       }
+struct bio_batch {
+       atomic_t                done;
+       unsigned long           flags;
+       struct completion       *wait;
+};
 
-       if (bio->bi_private)
-               complete(bio->bi_private);
-       __free_page(bio_page(bio));
+static void bio_batch_end_io(struct bio *bio, int err)
+{
+       struct bio_batch *bb = bio->bi_private;
 
+       if (err && (err != -EOPNOTSUPP))
+               clear_bit(BIO_UPTODATE, &bb->flags);
+       if (atomic_dec_and_test(&bb->done))
+               complete(bb->wait);
        bio_put(bio);
 }
 
@@ -40,10 +42,10 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
 {
        DECLARE_COMPLETION_ONSTACK(wait);
        struct request_queue *q = bdev_get_queue(bdev);
-       int type = flags & BLKDEV_IFL_BARRIER ?
-               DISCARD_BARRIER : DISCARD_NOBARRIER;
+       int type = REQ_WRITE | REQ_DISCARD;
+       unsigned int max_discard_sectors;
+       struct bio_batch bb;
        struct bio *bio;
-       struct page *page;
        int ret = 0;
 
        if (!q)
@@ -52,36 +54,42 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
        if (!blk_queue_discard(q))
                return -EOPNOTSUPP;
 
-       while (nr_sects && !ret) {
-               unsigned int sector_size = q->limits.logical_block_size;
-               unsigned int max_discard_sectors =
-                       min(q->limits.max_discard_sectors, UINT_MAX >> 9);
+       /*
+        * Ensure that max_discard_sectors is of the proper
+        * granularity
+        */
+       max_discard_sectors = min(q->limits.max_discard_sectors, UINT_MAX >> 9);
+       if (unlikely(!max_discard_sectors)) {
+               /* Avoid infinite loop below. Being cautious never hurts. */
+               return -EOPNOTSUPP;
+       } else if (q->limits.discard_granularity) {
+               unsigned int disc_sects = q->limits.discard_granularity >> 9;
+
+               max_discard_sectors &= ~(disc_sects - 1);
+       }
+
+       if (flags & BLKDEV_DISCARD_SECURE) {
+               if (!blk_queue_secdiscard(q))
+                       return -EOPNOTSUPP;
+               type |= REQ_SECURE;
+       }
 
+       atomic_set(&bb.done, 1);
+       bb.flags = 1 << BIO_UPTODATE;
+       bb.wait = &wait;
+
+       while (nr_sects) {
                bio = bio_alloc(gfp_mask, 1);
-               if (!bio)
-                       goto out;
+               if (!bio) {
+                       ret = -ENOMEM;
+                       break;
+               }
+
                bio->bi_sector = sector;
-               bio->bi_end_io = blkdev_discard_end_io;
+               bio->bi_end_io = bio_batch_end_io;
                bio->bi_bdev = bdev;
-               if (flags & BLKDEV_IFL_WAIT)
-                       bio->bi_private = &wait;
-
-               /*
-                * Add a zeroed one-sector payload as that's what
-                * our current implementations need.  If we'll ever need
-                * more the interface will need revisiting.
-                */
-               page = alloc_page(gfp_mask | __GFP_ZERO);
-               if (!page)
-                       goto out_free_bio;
-               if (bio_add_pc_page(q, bio, page, sector_size, 0) < sector_size)
-                       goto out_free_page;
-
-               /*
-                * And override the bio size - the way discard works we
-                * touch many more blocks on disk than the actual payload
-                * length.
-                */
+               bio->bi_private = &bb;
+
                if (nr_sects > max_discard_sectors) {
                        bio->bi_size = max_discard_sectors << 9;
                        nr_sects -= max_discard_sectors;
@@ -91,24 +99,80 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
                        nr_sects = 0;
                }
 
-               bio_get(bio);
+               atomic_inc(&bb.done);
                submit_bio(type, bio);
+       }
 
-               if (flags & BLKDEV_IFL_WAIT)
-                       wait_for_completion(&wait);
+       /* Wait for bios in-flight */
+       if (!atomic_dec_and_test(&bb.done))
+               wait_for_completion(&wait);
+
+       if (!test_bit(BIO_UPTODATE, &bb.flags))
+               ret = -EIO;
 
-               if (bio_flagged(bio, BIO_EOPNOTSUPP))
-                       ret = -EOPNOTSUPP;
-               else if (!bio_flagged(bio, BIO_UPTODATE))
-                       ret = -EIO;
-               bio_put(bio);
-       }
        return ret;
-out_free_page:
-       __free_page(page);
-out_free_bio:
-       bio_put(bio);
-out:
-       return -ENOMEM;
 }
 EXPORT_SYMBOL(blkdev_issue_discard);
+
+/**
+ * blkdev_issue_zeroout - generate number of zero filed write bios
+ * @bdev:      blockdev to issue
+ * @sector:    start sector
+ * @nr_sects:  number of sectors to write
+ * @gfp_mask:  memory allocation flags (for bio_alloc)
+ *
+ * Description:
+ *  Generate and issue number of bios with zerofiled pages.
+ */
+
+int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
+                       sector_t nr_sects, gfp_t gfp_mask)
+{
+       int ret;
+       struct bio *bio;
+       struct bio_batch bb;
+       unsigned int sz;
+       DECLARE_COMPLETION_ONSTACK(wait);
+
+       atomic_set(&bb.done, 1);
+       bb.flags = 1 << BIO_UPTODATE;
+       bb.wait = &wait;
+
+       ret = 0;
+       while (nr_sects != 0) {
+               bio = bio_alloc(gfp_mask,
+                               min(nr_sects, (sector_t)BIO_MAX_PAGES));
+               if (!bio) {
+                       ret = -ENOMEM;
+                       break;
+               }
+
+               bio->bi_sector = sector;
+               bio->bi_bdev   = bdev;
+               bio->bi_end_io = bio_batch_end_io;
+               bio->bi_private = &bb;
+
+               while (nr_sects != 0) {
+                       sz = min((sector_t) PAGE_SIZE >> 9 , nr_sects);
+                       ret = bio_add_page(bio, ZERO_PAGE(0), sz << 9, 0);
+                       nr_sects -= ret >> 9;
+                       sector += ret >> 9;
+                       if (ret < (sz << 9))
+                               break;
+               }
+               ret = 0;
+               atomic_inc(&bb.done);
+               submit_bio(WRITE, bio);
+       }
+
+       /* Wait for bios in-flight */
+       if (!atomic_dec_and_test(&bb.done))
+               wait_for_completion(&wait);
+
+       if (!test_bit(BIO_UPTODATE, &bb.flags))
+               /* One of bios in the batch was completed with error.*/
+               ret = -EIO;
+
+       return ret;
+}
+EXPORT_SYMBOL(blkdev_issue_zeroout);