Fix a direct I/O locking issue revealed by the new mutex code.
[linux-2.6.git] / fs / direct-io.c
index 5a674a0..27f3e78 100644 (file)
@@ -56,7 +56,7 @@
  * lock_type is DIO_LOCKING for regular files on direct-IO-naive filesystems.
  * This determines whether we need to do the fancy locking which prevents
  * direct-IO from being able to read uninitialised disk blocks.  If its zero
- * (blockdev) this locking is not done, and if it is DIO_OWN_LOCKING i_sem is
+ * (blockdev) this locking is not done, and if it is DIO_OWN_LOCKING i_mutex is
  * not held for the entire direct write (taken briefly, initially, during a
  * direct read though, but its never held for the duration of a direct-IO).
  */
@@ -66,6 +66,7 @@ struct dio {
        struct bio *bio;                /* bio under assembly */
        struct inode *inode;
        int rw;
+       loff_t i_size;                  /* i_size when submitted */
        int lock_type;                  /* doesn't change */
        unsigned blkbits;               /* doesn't change */
        unsigned blkfactor;             /* When we're using an alignment which
@@ -161,6 +162,7 @@ static int dio_refill_pages(struct dio *dio)
        up_read(&current->mm->mmap_sem);
 
        if (ret < 0 && dio->blocks_available && (dio->rw == WRITE)) {
+               struct page *page = ZERO_PAGE(dio->curr_user_address);
                /*
                 * A memory fault, but the filesystem has some outstanding
                 * mapped blocks.  We need to use those blocks up to avoid
@@ -168,7 +170,8 @@ static int dio_refill_pages(struct dio *dio)
                 */
                if (dio->page_errors == 0)
                        dio->page_errors = ret;
-               dio->pages[0] = ZERO_PAGE(dio->curr_user_address);
+               page_cache_get(page);
+               dio->pages[0] = page;
                dio->head = 0;
                dio->tail = 1;
                ret = 0;
@@ -214,7 +217,7 @@ static struct page *dio_get_page(struct dio *dio)
 static void dio_complete(struct dio *dio, loff_t offset, ssize_t bytes)
 {
        if (dio->end_io && dio->result)
-               dio->end_io(dio->inode, offset, bytes, dio->map_bh.b_private);
+               dio->end_io(dio->iocb, offset, bytes, dio->map_bh.b_private);
        if (dio->lock_type == DIO_LOCKING)
                up_read(&dio->inode->i_alloc_sem);
 }
@@ -230,17 +233,29 @@ static void finished_one_bio(struct dio *dio)
        spin_lock_irqsave(&dio->bio_lock, flags);
        if (dio->bio_count == 1) {
                if (dio->is_async) {
+                       ssize_t transferred;
+                       loff_t offset;
+
                        /*
                         * Last reference to the dio is going away.
                         * Drop spinlock and complete the DIO.
                         */
                        spin_unlock_irqrestore(&dio->bio_lock, flags);
-                       dio_complete(dio, dio->block_in_file << dio->blkbits,
-                                       dio->result);
+
+                       /* Check for short read case */
+                       transferred = dio->result;
+                       offset = dio->iocb->ki_pos;
+
+                       if ((dio->rw == READ) &&
+                           ((offset + transferred) > dio->i_size))
+                               transferred = dio->i_size - offset;
+
+                       dio_complete(dio, offset, transferred);
+
                        /* Complete AIO later if falling back to buffered i/o */
                        if (dio->result == dio->size ||
                                ((dio->rw == READ) && dio->result)) {
-                               aio_complete(dio->iocb, dio->result, 0);
+                               aio_complete(dio->iocb, transferred, 0);
                                kfree(dio);
                                return;
                        } else {
@@ -842,6 +857,7 @@ do_holes:
                        /* Handle holes */
                        if (!buffer_mapped(map_bh)) {
                                char *kaddr;
+                               loff_t i_size_aligned;
 
                                /* AKPM: eargh, -ENOTBLK is a hack */
                                if (dio->rw == WRITE) {
@@ -849,8 +865,14 @@ do_holes:
                                        return -ENOTBLK;
                                }
 
+                               /*
+                                * Be sure to account for a partial block as the
+                                * last block in the file
+                                */
+                               i_size_aligned = ALIGN(i_size_read(dio->inode),
+                                                       1 << blkbits);
                                if (dio->block_in_file >=
-                                       i_size_read(dio->inode)>>blkbits) {
+                                               i_size_aligned >> blkbits) {
                                        /* We hit eof */
                                        page_cache_release(page);
                                        goto out;
@@ -915,7 +937,7 @@ out:
 }
 
 /*
- * Releases both i_sem and i_alloc_sem
+ * Releases both i_mutex and i_alloc_sem
  */
 static ssize_t
 direct_io_worker(int rw, struct kiocb *iocb, struct inode *inode, 
@@ -951,6 +973,7 @@ direct_io_worker(int rw, struct kiocb *iocb, struct inode *inode,
        dio->page_errors = 0;
        dio->result = 0;
        dio->iocb = iocb;
+       dio->i_size = i_size_read(inode);
 
        /*
         * BIO completion state.
@@ -1046,11 +1069,11 @@ direct_io_worker(int rw, struct kiocb *iocb, struct inode *inode,
 
        /*
         * All block lookups have been performed. For READ requests
-        * we can let i_sem go now that its achieved its purpose
+        * we can let i_mutex go now that its achieved its purpose
         * of protecting us from looking up uninitialized blocks.
         */
        if ((rw == READ) && (dio->lock_type == DIO_LOCKING))
-               up(&dio->inode->i_sem);
+               mutex_unlock(&dio->inode->i_mutex);
 
        /*
         * OK, all BIOs are submitted, so we can decrement bio_count to truly
@@ -1129,18 +1152,19 @@ direct_io_worker(int rw, struct kiocb *iocb, struct inode *inode,
  * The locking rules are governed by the dio_lock_type parameter.
  *
  * DIO_NO_LOCKING (no locking, for raw block device access)
- * For writes, i_sem is not held on entry; it is never taken.
+ * For writes, i_mutex is not held on entry; it is never taken.
  *
  * DIO_LOCKING (simple locking for regular files)
- * For writes we are called under i_sem and return with i_sem held, even though
- * it is internally dropped.
- * For reads, i_sem is not held on entry, but it is taken and dropped before
+ * For writes we are called under i_mutex and return with i_mutex held, even
+ * though it is internally dropped.
+ * For reads, i_mutex is not held on entry, but it is taken and dropped before
  * returning.
  *
  * DIO_OWN_LOCKING (filesystem provides synchronisation and handling of
  *     uninitialised data, allowing parallel direct readers and writers)
- * For writes we are called without i_sem, return without it, never touch it.
- * For reads, i_sem is held on entry and will be released before returning.
+ * For writes we are called without i_mutex, return without it, never touch it.
+ * For reads we are called under i_mutex and return with i_mutex held, even
+ * though it may be internally dropped.
  *
  * Additional i_alloc_sem locking requirements described inline below.
  */
@@ -1159,7 +1183,8 @@ __blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
        ssize_t retval = -EINVAL;
        loff_t end = offset;
        struct dio *dio;
-       int reader_with_isem = (rw == READ && dio_lock_type == DIO_OWN_LOCKING);
+       int release_i_mutex = 0;
+       int acquire_i_mutex = 0;
 
        if (rw & WRITE)
                current->flags |= PF_SYNCWRITE;
@@ -1198,11 +1223,10 @@ __blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
         * For block device access DIO_NO_LOCKING is used,
         *      neither readers nor writers do any locking at all
         * For regular files using DIO_LOCKING,
-        *      readers need to grab i_sem and i_alloc_sem
-        *      writers need to grab i_alloc_sem only (i_sem is already held)
+        *      readers need to grab i_mutex and i_alloc_sem
+        *      writers need to grab i_alloc_sem only (i_mutex is already held)
         * For regular files using DIO_OWN_LOCKING,
         *      neither readers nor writers take any locks here
-        *      (i_sem is already held and release for writers here)
         */
        dio->lock_type = dio_lock_type;
        if (dio_lock_type != DIO_NO_LOCKING) {
@@ -1212,8 +1236,8 @@ __blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
 
                        mapping = iocb->ki_filp->f_mapping;
                        if (dio_lock_type != DIO_OWN_LOCKING) {
-                               down(&inode->i_sem);
-                               reader_with_isem = 1;
+                               mutex_lock(&inode->i_mutex);
+                               release_i_mutex = 1;
                        }
 
                        retval = filemap_write_and_wait_range(mapping, offset,
@@ -1224,8 +1248,8 @@ __blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
                        }
 
                        if (dio_lock_type == DIO_OWN_LOCKING) {
-                               up(&inode->i_sem);
-                               reader_with_isem = 0;
+                               mutex_unlock(&inode->i_mutex);
+                               acquire_i_mutex = 1;
                        }
                }
 
@@ -1246,11 +1270,13 @@ __blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
                                nr_segs, blkbits, get_blocks, end_io, dio);
 
        if (rw == READ && dio_lock_type == DIO_LOCKING)
-               reader_with_isem = 0;
+               release_i_mutex = 0;
 
 out:
-       if (reader_with_isem)
-               up(&inode->i_sem);
+       if (release_i_mutex)
+               mutex_unlock(&inode->i_mutex);
+       else if (acquire_i_mutex)
+               mutex_lock(&inode->i_mutex);
        if (rw & WRITE)
                current->flags &= ~PF_SYNCWRITE;
        return retval;