NeilBrown <neilb@suse.de>
[linux-2.6.git] / mm / filemap.c
index f560b41..adbac10 100644 (file)
@@ -9,12 +9,13 @@
  * most "normal" filesystems (but you don't /have/ to use this:
  * the NFS filesystem used to do this differently, for example)
  */
-#include <linux/config.h>
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/compiler.h>
 #include <linux/fs.h>
+#include <linux/uaccess.h>
 #include <linux/aio.h>
+#include <linux/capability.h>
 #include <linux/kernel_stat.h>
 #include <linux/mm.h>
 #include <linux/swap.h>
 #include <linux/blkdev.h>
 #include <linux/security.h>
 #include <linux/syscalls.h>
+#include <linux/cpuset.h>
 #include "filemap.h"
+#include "internal.h"
+
 /*
  * FIXME: remove all knowledge of the buffer layer from the core VM
  */
 #include <linux/buffer_head.h> /* for generic_osync_inode */
 
-#include <asm/uaccess.h>
 #include <asm/mman.h>
 
 static ssize_t
@@ -61,21 +64,21 @@ generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
  *      ->swap_lock            (exclusive_swap_page, others)
  *        ->mapping->tree_lock
  *
- *  ->i_sem
+ *  ->i_mutex
  *    ->i_mmap_lock            (truncate->unmap_mapping_range)
  *
  *  ->mmap_sem
  *    ->i_mmap_lock
- *      ->page_table_lock      (various places, mainly in mmap.c)
+ *      ->page_table_lock or pte_lock  (various, mainly in memory.c)
  *        ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock)
  *
  *  ->mmap_sem
  *    ->lock_page              (access_process_vm)
  *
- *  ->mmap_sem
- *    ->i_sem                  (msync)
+ *  ->i_mutex                  (generic_file_buffered_write)
+ *    ->mmap_sem               (fault_in_pages_readable->do_page_fault)
  *
- *  ->i_sem
+ *  ->i_mutex
  *    ->i_alloc_sem             (various)
  *
  *  ->inode_lock
@@ -86,13 +89,14 @@ generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
  *    ->anon_vma.lock          (vma_adjust)
  *
  *  ->anon_vma.lock
- *    ->page_table_lock                (anon_vma_prepare and various)
+ *    ->page_table_lock or pte_lock    (anon_vma_prepare and various)
  *
- *  ->page_table_lock
+ *  ->page_table_lock or pte_lock
  *    ->swap_lock              (try_to_unmap_one)
  *    ->private_lock           (try_to_unmap_one)
  *    ->tree_lock              (try_to_unmap_one)
  *    ->zone.lru_lock          (follow_page->mark_page_accessed)
+ *    ->zone.lru_lock          (check_pte_range->isolate_lru_page)
  *    ->private_lock           (page_remove_rmap->set_page_dirty)
  *    ->tree_lock              (page_remove_rmap->set_page_dirty)
  *    ->inode_lock             (page_remove_rmap->set_page_dirty)
@@ -115,7 +119,8 @@ void __remove_from_page_cache(struct page *page)
        radix_tree_delete(&mapping->page_tree, page->index);
        page->mapping = NULL;
        mapping->nrpages--;
-       pagecache_acct(-1);
+       __dec_zone_page_state(page, NR_FILE_PAGES);
+       BUG_ON(page_mapped(page));
 }
 
 void remove_from_page_cache(struct page *page)
@@ -134,7 +139,7 @@ static int sync_page(void *word)
        struct address_space *mapping;
        struct page *page;
 
-       page = container_of((page_flags_t *)word, struct page, flags);
+       page = container_of((unsigned long *)word, struct page, flags);
 
        /*
         * page_mapping() is being called without PG_locked held.
@@ -166,27 +171,29 @@ static int sync_page(void *word)
 }
 
 /**
- * filemap_fdatawrite_range - start writeback against all of a mapping's
- * dirty pages that lie within the byte offsets <start, end>
+ * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
  * @mapping:   address space structure to write
  * @start:     offset in bytes where the range starts
- * @end:       offset in bytes where the range ends
+ * @end:       offset in bytes where the range ends (inclusive)
  * @sync_mode: enable synchronous operation
  *
+ * Start writeback against all of a mapping's dirty pages that lie
+ * within the byte offsets <start, end> inclusive.
+ *
  * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
- * opposed to a regular memory * cleansing writeback.  The difference between
+ * opposed to a regular memory cleansing writeback.  The difference between
  * these two operations is that if a dirty page/buffer is encountered, it must
  * be waited upon, and not just skipped over.
  */
-static int __filemap_fdatawrite_range(struct address_space *mapping,
-       loff_t start, loff_t end, int sync_mode)
+int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
+                               loff_t end, int sync_mode)
 {
        int ret;
        struct writeback_control wbc = {
                .sync_mode = sync_mode,
                .nr_to_write = mapping->nrpages * 2,
-               .start = start,
-               .end = end,
+               .range_start = start,
+               .range_end = end,
        };
 
        if (!mapping_cap_writeback_dirty(mapping))
@@ -199,7 +206,7 @@ static int __filemap_fdatawrite_range(struct address_space *mapping,
 static inline int __filemap_fdatawrite(struct address_space *mapping,
        int sync_mode)
 {
-       return __filemap_fdatawrite_range(mapping, 0, 0, sync_mode);
+       return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
 }
 
 int filemap_fdatawrite(struct address_space *mapping)
@@ -208,13 +215,16 @@ int filemap_fdatawrite(struct address_space *mapping)
 }
 EXPORT_SYMBOL(filemap_fdatawrite);
 
-static int filemap_fdatawrite_range(struct address_space *mapping,
-       loff_t start, loff_t end)
+static int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
+                               loff_t end)
 {
        return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
 }
 
-/*
+/**
+ * filemap_flush - mostly a non-blocking flush
+ * @mapping:   target address_space
+ *
  * This is a mostly non-blocking flush.  Not suitable for data-integrity
  * purposes - I/O may not be started against all dirty pages.
  */
@@ -224,11 +234,16 @@ int filemap_flush(struct address_space *mapping)
 }
 EXPORT_SYMBOL(filemap_flush);
 
-/*
+/**
+ * wait_on_page_writeback_range - wait for writeback to complete
+ * @mapping:   target address_space
+ * @start:     beginning page index
+ * @end:       ending page index
+ *
  * Wait for writeback to complete against pages indexed by start->end
  * inclusive
  */
-static int wait_on_page_writeback_range(struct address_space *mapping,
+int wait_on_page_writeback_range(struct address_space *mapping,
                                pgoff_t start, pgoff_t end)
 {
        struct pagevec pvec;
@@ -271,16 +286,22 @@ static int wait_on_page_writeback_range(struct address_space *mapping,
        return ret;
 }
 
-/*
+/**
+ * sync_page_range - write and wait on all pages in the passed range
+ * @inode:     target inode
+ * @mapping:   target address_space
+ * @pos:       beginning offset in pages to write
+ * @count:     number of bytes to write
+ *
  * Write and wait upon all the pages in the passed range.  This is a "data
  * integrity" operation.  It waits upon in-flight writeout before starting and
  * waiting upon new writeout.  If there was an IO error, return it.
  *
- * We need to re-take i_sem during the generic_osync_inode list walk because
+ * We need to re-take i_mutex during the generic_osync_inode list walk because
  * it is otherwise livelockable.
  */
 int sync_page_range(struct inode *inode, struct address_space *mapping,
-                       loff_t pos, size_t count)
+                       loff_t pos, loff_t count)
 {
        pgoff_t start = pos >> PAGE_CACHE_SHIFT;
        pgoff_t end = (pos + count - 1) >> PAGE_CACHE_SHIFT;
@@ -290,9 +311,9 @@ int sync_page_range(struct inode *inode, struct address_space *mapping,
                return 0;
        ret = filemap_fdatawrite_range(mapping, pos, pos + count - 1);
        if (ret == 0) {
-               down(&inode->i_sem);
+               mutex_lock(&inode->i_mutex);
                ret = generic_osync_inode(inode, mapping, OSYNC_METADATA);
-               up(&inode->i_sem);
+               mutex_unlock(&inode->i_mutex);
        }
        if (ret == 0)
                ret = wait_on_page_writeback_range(mapping, start, end);
@@ -300,14 +321,19 @@ int sync_page_range(struct inode *inode, struct address_space *mapping,
 }
 EXPORT_SYMBOL(sync_page_range);
 
-/*
- * Note: Holding i_sem across sync_page_range_nolock is not a good idea
+/**
+ * sync_page_range_nolock
+ * @inode:     target inode
+ * @mapping:   target address_space
+ * @pos:       beginning offset in pages to write
+ * @count:     number of bytes to write
+ *
+ * Note: Holding i_mutex across sync_page_range_nolock() is not a good idea
  * as it forces O_SYNC writers to different parts of the same file
  * to be serialised right until io completion.
  */
-static int sync_page_range_nolock(struct inode *inode,
-                                 struct address_space *mapping,
-                                 loff_t pos, size_t count)
+int sync_page_range_nolock(struct inode *inode, struct address_space *mapping,
+                          loff_t pos, loff_t count)
 {
        pgoff_t start = pos >> PAGE_CACHE_SHIFT;
        pgoff_t end = (pos + count - 1) >> PAGE_CACHE_SHIFT;
@@ -322,12 +348,14 @@ static int sync_page_range_nolock(struct inode *inode,
                ret = wait_on_page_writeback_range(mapping, start, end);
        return ret;
 }
+EXPORT_SYMBOL(sync_page_range_nolock);
 
 /**
- * filemap_fdatawait - walk the list of under-writeback pages of the given
- *     address space and wait for all of them.
- *
+ * filemap_fdatawait - wait for all under-writeback pages to complete
  * @mapping: address space structure to wait for
+ *
+ * Walk the list of under-writeback pages of the given address space
+ * and wait for all of them.
  */
 int filemap_fdatawait(struct address_space *mapping)
 {
@@ -343,34 +371,65 @@ EXPORT_SYMBOL(filemap_fdatawait);
 
 int filemap_write_and_wait(struct address_space *mapping)
 {
-       int retval = 0;
+       int err = 0;
 
        if (mapping->nrpages) {
-               retval = filemap_fdatawrite(mapping);
-               if (retval == 0)
-                       retval = filemap_fdatawait(mapping);
+               err = filemap_fdatawrite(mapping);
+               /*
+                * Even if the above returned error, the pages may be
+                * written partially (e.g. -ENOSPC), so we wait for it.
+                * But the -EIO is special case, it may indicate the worst
+                * thing (e.g. bug) happened, so we avoid waiting for it.
+                */
+               if (err != -EIO) {
+                       int err2 = filemap_fdatawait(mapping);
+                       if (!err)
+                               err = err2;
+               }
        }
-       return retval;
+       return err;
 }
+EXPORT_SYMBOL(filemap_write_and_wait);
 
+/**
+ * filemap_write_and_wait_range - write out & wait on a file range
+ * @mapping:   the address_space for the pages
+ * @lstart:    offset in bytes where the range starts
+ * @lend:      offset in bytes where the range ends (inclusive)
+ *
+ * Write out and wait upon file offsets lstart->lend, inclusive.
+ *
+ * Note that `lend' is inclusive (describes the last byte to be written) so
+ * that this function can be used to write to the very end-of-file (end = -1).
+ */
 int filemap_write_and_wait_range(struct address_space *mapping,
                                 loff_t lstart, loff_t lend)
 {
-       int retval = 0;
+       int err = 0;
 
        if (mapping->nrpages) {
-               retval = __filemap_fdatawrite_range(mapping, lstart, lend,
-                                                   WB_SYNC_ALL);
-               if (retval == 0)
-                       retval = wait_on_page_writeback_range(mapping,
-                                                   lstart >> PAGE_CACHE_SHIFT,
-                                                   lend >> PAGE_CACHE_SHIFT);
+               err = __filemap_fdatawrite_range(mapping, lstart, lend,
+                                                WB_SYNC_ALL);
+               /* See comment of filemap_write_and_wait() */
+               if (err != -EIO) {
+                       int err2 = wait_on_page_writeback_range(mapping,
+                                               lstart >> PAGE_CACHE_SHIFT,
+                                               lend >> PAGE_CACHE_SHIFT);
+                       if (!err)
+                               err = err2;
+               }
        }
-       return retval;
+       return err;
 }
 
-/*
- * This function is used to add newly allocated pagecache pages:
+/**
+ * add_to_page_cache - add newly allocated pagecache pages
+ * @page:      page to add
+ * @mapping:   the page's address_space
+ * @offset:    page index
+ * @gfp_mask:  page allocation mode
+ *
+ * This function is used to add newly allocated pagecache pages;
  * the page is new, so we can just run SetPageLocked() against it.
  * The other page state flags were set by rmqueue().
  *
@@ -390,14 +449,13 @@ int add_to_page_cache(struct page *page, struct address_space *mapping,
                        page->mapping = mapping;
                        page->index = offset;
                        mapping->nrpages++;
-                       pagecache_acct(1);
+                       __inc_zone_page_state(page, NR_FILE_PAGES);
                }
                write_unlock_irq(&mapping->tree_lock);
                radix_tree_preload_end();
        }
        return error;
 }
-
 EXPORT_SYMBOL(add_to_page_cache);
 
 int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
@@ -409,6 +467,24 @@ int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
        return ret;
 }
 
+#ifdef CONFIG_NUMA
+struct page *__page_cache_alloc(gfp_t gfp)
+{
+       if (cpuset_do_page_mem_spread()) {
+               int n = cpuset_mem_spread_node();
+               return alloc_pages_node(n, gfp, 0);
+       }
+       return alloc_pages(gfp, 0);
+}
+EXPORT_SYMBOL(__page_cache_alloc);
+#endif
+
+static int __sleep_on_page_lock(void *word)
+{
+       io_schedule();
+       return 0;
+}
+
 /*
  * In order to wait for pages to become available there must be
  * waitqueues associated with pages. By using a hash table of
@@ -442,8 +518,7 @@ void fastcall wait_on_page_bit(struct page *page, int bit_nr)
 EXPORT_SYMBOL(wait_on_page_bit);
 
 /**
- * unlock_page() - unlock a locked page
- *
+ * unlock_page - unlock a locked page
  * @page: the page
  *
  * Unlocks the page and wakes up sleepers in ___wait_on_page_locked().
@@ -466,8 +541,9 @@ void fastcall unlock_page(struct page *page)
 }
 EXPORT_SYMBOL(unlock_page);
 
-/*
- * End writeback against a page.
+/**
+ * end_page_writeback - end writeback against a page
+ * @page: the page
  */
 void end_page_writeback(struct page *page)
 {
@@ -480,10 +556,11 @@ void end_page_writeback(struct page *page)
 }
 EXPORT_SYMBOL(end_page_writeback);
 
-/*
- * Get a lock on the page, assuming we need to sleep to get it.
+/**
+ * __lock_page - get a lock on the page, assuming we need to sleep to get it
+ * @page: the page to lock
  *
- * Ugly: running sync_page() in state TASK_UNINTERRUPTIBLE is scary.  If some
+ * Ugly. Running sync_page() in state TASK_UNINTERRUPTIBLE is scary.  If some
  * random driver's requestfn sets TASK_RUNNING, we could busywait.  However
  * chances are that on the second loop, the block layer's plug list is empty,
  * so sync_page() will then return in state TASK_UNINTERRUPTIBLE.
@@ -498,43 +575,39 @@ void fastcall __lock_page(struct page *page)
 EXPORT_SYMBOL(__lock_page);
 
 /*
- * a rather lightweight function, finding and getting a reference to a
- * hashed page atomically.
+ * Variant of lock_page that does not require the caller to hold a reference
+ * on the page's mapping.
  */
-struct page * find_get_page(struct address_space *mapping, unsigned long offset)
+void fastcall __lock_page_nosync(struct page *page)
 {
-       struct page *page;
-
-       read_lock_irq(&mapping->tree_lock);
-       page = radix_tree_lookup(&mapping->page_tree, offset);
-       if (page)
-               page_cache_get(page);
-       read_unlock_irq(&mapping->tree_lock);
-       return page;
+       DEFINE_WAIT_BIT(wait, &page->flags, PG_locked);
+       __wait_on_bit_lock(page_waitqueue(page), &wait, __sleep_on_page_lock,
+                                                       TASK_UNINTERRUPTIBLE);
 }
 
-EXPORT_SYMBOL(find_get_page);
-
-/*
- * Same as above, but trylock it instead of incrementing the count.
+/**
+ * find_get_page - find and get a page reference
+ * @mapping: the address_space to search
+ * @offset: the page index
+ *
+ * Is there a pagecache struct page at the given (mapping, offset) tuple?
+ * If yes, increment its refcount and return it; if no, return NULL.
  */
-struct page *find_trylock_page(struct address_space *mapping, unsigned long offset)
+struct page * find_get_page(struct address_space *mapping, unsigned long offset)
 {
        struct page *page;
 
        read_lock_irq(&mapping->tree_lock);
        page = radix_tree_lookup(&mapping->page_tree, offset);
-       if (page && TestSetPageLocked(page))
-               page = NULL;
+       if (page)
+               page_cache_get(page);
        read_unlock_irq(&mapping->tree_lock);
        return page;
 }
-
-EXPORT_SYMBOL(find_trylock_page);
+EXPORT_SYMBOL(find_get_page);
 
 /**
  * find_lock_page - locate, pin and lock a pagecache page
- *
  * @mapping: the address_space to search
  * @offset: the page index
  *
@@ -555,11 +628,12 @@ repeat:
                page_cache_get(page);
                if (TestSetPageLocked(page)) {
                        read_unlock_irq(&mapping->tree_lock);
-                       lock_page(page);
+                       __lock_page(page);
                        read_lock_irq(&mapping->tree_lock);
 
                        /* Has the page been truncated while we slept? */
-                       if (page->mapping != mapping || page->index != offset) {
+                       if (unlikely(page->mapping != mapping ||
+                                    page->index != offset)) {
                                unlock_page(page);
                                page_cache_release(page);
                                goto repeat;
@@ -569,12 +643,10 @@ repeat:
        read_unlock_irq(&mapping->tree_lock);
        return page;
 }
-
 EXPORT_SYMBOL(find_lock_page);
 
 /**
  * find_or_create_page - locate or add a pagecache page
- *
  * @mapping: the page's address_space
  * @index: the page's index into the mapping
  * @gfp_mask: page allocation mode
@@ -599,7 +671,8 @@ repeat:
        page = find_lock_page(mapping, index);
        if (!page) {
                if (!cached_page) {
-                       cached_page = alloc_page(gfp_mask);
+                       cached_page =
+                               __page_cache_alloc(gfp_mask);
                        if (!cached_page)
                                return NULL;
                }
@@ -615,7 +688,6 @@ repeat:
                page_cache_release(cached_page);
        return page;
 }
-
 EXPORT_SYMBOL(find_or_create_page);
 
 /**
@@ -649,9 +721,49 @@ unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
        return ret;
 }
 
-/*
+/**
+ * find_get_pages_contig - gang contiguous pagecache lookup
+ * @mapping:   The address_space to search
+ * @index:     The starting page index
+ * @nr_pages:  The maximum number of pages
+ * @pages:     Where the resulting pages are placed
+ *
+ * find_get_pages_contig() works exactly like find_get_pages(), except
+ * that the returned number of pages are guaranteed to be contiguous.
+ *
+ * find_get_pages_contig() returns the number of pages which were found.
+ */
+unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
+                              unsigned int nr_pages, struct page **pages)
+{
+       unsigned int i;
+       unsigned int ret;
+
+       read_lock_irq(&mapping->tree_lock);
+       ret = radix_tree_gang_lookup(&mapping->page_tree,
+                               (void **)pages, index, nr_pages);
+       for (i = 0; i < ret; i++) {
+               if (pages[i]->mapping == NULL || pages[i]->index != index)
+                       break;
+
+               page_cache_get(pages[i]);
+               index++;
+       }
+       read_unlock_irq(&mapping->tree_lock);
+       return i;
+}
+EXPORT_SYMBOL(find_get_pages_contig);
+
+/**
+ * find_get_pages_tag - find and return pages that match @tag
+ * @mapping:   the address_space to search
+ * @index:     the starting page index
+ * @tag:       the tag index
+ * @nr_pages:  the maximum number of pages
+ * @pages:     where the resulting pages are placed
+ *
  * Like find_get_pages, except we only return pages which are tagged with
- * `tag'.   We update *index to index the next page for the traversal.
+ * @tag.   We update @index to index the next page for the traversal.
  */
 unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
                        int tag, unsigned int nr_pages, struct page **pages)
@@ -669,9 +781,14 @@ unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
        read_unlock_irq(&mapping->tree_lock);
        return ret;
 }
+EXPORT_SYMBOL(find_get_pages_tag);
 
-/*
- * Same as grab_cache_page, but do not wait if the page is unavailable.
+/**
+ * grab_cache_page_nowait - returns locked page at given index in given cache
+ * @mapping: target address_space
+ * @index: the page index
+ *
+ * Same as grab_cache_page(), but do not wait if the page is unavailable.
  * This is intended for speculative data generators, where the data can
  * be regenerated if the page couldn't be grabbed.  This routine should
  * be safe to call while holding the lock for another page.
@@ -683,7 +800,6 @@ struct page *
 grab_cache_page_nowait(struct address_space *mapping, unsigned long index)
 {
        struct page *page = find_get_page(mapping, index);
-       gfp_t gfp_mask;
 
        if (page) {
                if (!TestSetPageLocked(page))
@@ -691,27 +807,56 @@ grab_cache_page_nowait(struct address_space *mapping, unsigned long index)
                page_cache_release(page);
                return NULL;
        }
-       gfp_mask = mapping_gfp_mask(mapping) & ~__GFP_FS;
-       page = alloc_pages(gfp_mask, 0);
-       if (page && add_to_page_cache_lru(page, mapping, index, gfp_mask)) {
+       page = __page_cache_alloc(mapping_gfp_mask(mapping) & ~__GFP_FS);
+       if (page && add_to_page_cache_lru(page, mapping, index, GFP_KERNEL)) {
                page_cache_release(page);
                page = NULL;
        }
        return page;
 }
-
 EXPORT_SYMBOL(grab_cache_page_nowait);
 
 /*
+ * CD/DVDs are error prone. When a medium error occurs, the driver may fail
+ * a _large_ part of the i/o request. Imagine the worst scenario:
+ *
+ *      ---R__________________________________________B__________
+ *         ^ reading here                             ^ bad block(assume 4k)
+ *
+ * read(R) => miss => readahead(R...B) => media error => frustrating retries
+ * => failing the whole request => read(R) => read(R+1) =>
+ * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) =>
+ * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) =>
+ * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ......
+ *
+ * It is going insane. Fix it by quickly scaling down the readahead size.
+ */
+static void shrink_readahead_size_eio(struct file *filp,
+                                       struct file_ra_state *ra)
+{
+       if (!ra->ra_pages)
+               return;
+
+       ra->ra_pages /= 4;
+}
+
+/**
+ * do_generic_mapping_read - generic file read routine
+ * @mapping:   address_space to be read
+ * @_ra:       file's readahead state
+ * @filp:      the file to read
+ * @ppos:      current file position
+ * @desc:      read_descriptor
+ * @actor:     read method
+ *
  * This is a generic file read routine, and uses the
- * mapping->a_ops->readpage() function for the actual low-level
- * stuff.
+ * mapping->a_ops->readpage() function for the actual low-level stuff.
  *
  * This is really ugly. But the goto's actually try to clarify some
  * of the logic when it comes to error handling etc.
  *
- * Note the struct file* is only passed for the use of readpage.  It may be
- * NULL.
+ * Note the struct file* is only passed for the use of readpage.
+ * It may be NULL.
  */
 void do_generic_mapping_read(struct address_space *mapping,
                             struct file_ra_state *_ra,
@@ -727,6 +872,7 @@ void do_generic_mapping_read(struct address_space *mapping,
        unsigned long last_index;
        unsigned long next_index;
        unsigned long prev_index;
+       unsigned int prev_offset;
        loff_t isize;
        struct page *cached_page;
        int error;
@@ -735,7 +881,8 @@ void do_generic_mapping_read(struct address_space *mapping,
        cached_page = NULL;
        index = *ppos >> PAGE_CACHE_SHIFT;
        next_index = index;
-       prev_index = ra.prev_page;
+       prev_index = ra.prev_index;
+       prev_offset = ra.prev_offset;
        last_index = (*ppos + desc->count + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
        offset = *ppos & ~PAGE_CACHE_MASK;
 
@@ -783,10 +930,10 @@ page_ok:
                        flush_dcache_page(page);
 
                /*
-                * When (part of) the same page is read multiple times
-                * in succession, only mark it as accessed the first time.
+                * When a sequential read accesses a page several times,
+                * only mark it as accessed the first time.
                 */
-               if (prev_index != index)
+               if (prev_index != index || offset != prev_offset)
                        mark_page_accessed(page);
                prev_index = index;
 
@@ -804,6 +951,8 @@ page_ok:
                offset += ret;
                index += offset >> PAGE_CACHE_SHIFT;
                offset &= ~PAGE_CACHE_MASK;
+               prev_offset = offset;
+               ra.prev_offset = offset;
 
                page_cache_release(page);
                if (ret == nr && desc->count)
@@ -814,7 +963,7 @@ page_not_up_to_date:
                /* Get exclusive access to the page ... */
                lock_page(page);
 
-               /* Did it get unhashed before we got the lock? */
+               /* Did it get truncated before we got the lock? */
                if (!page->mapping) {
                        unlock_page(page);
                        page_cache_release(page);
@@ -831,8 +980,13 @@ readpage:
                /* Start the actual read. The read will unlock the page. */
                error = mapping->a_ops->readpage(filp, page);
 
-               if (unlikely(error))
+               if (unlikely(error)) {
+                       if (error == AOP_TRUNCATED_PAGE) {
+                               page_cache_release(page);
+                               goto find_page;
+                       }
                        goto readpage_error;
+               }
 
                if (!PageUptodate(page)) {
                        lock_page(page);
@@ -847,6 +1001,7 @@ readpage:
                                }
                                unlock_page(page);
                                error = -EIO;
+                               shrink_readahead_size_eio(filp, &ra);
                                goto readpage_error;
                        }
                        unlock_page(page);
@@ -919,7 +1074,6 @@ out:
        if (filp)
                file_accessed(filp);
 }
-
 EXPORT_SYMBOL(do_generic_mapping_read);
 
 int file_read_actor(read_descriptor_t *desc, struct page *page,
@@ -961,41 +1115,72 @@ success:
 }
 
 /*
- * This is the "read()" routine for all filesystems
- * that can use the page cache directly.
+ * Performs necessary checks before doing a write
+ * @iov:       io vector request
+ * @nr_segs:   number of segments in the iovec
+ * @count:     number of bytes to write
+ * @access_flags: type of access: %VERIFY_READ or %VERIFY_WRITE
+ *
+ * Adjust number of segments and amount of bytes to write (nr_segs should be
+ * properly initialized first). Returns appropriate error code that caller
+ * should return or zero in case that write should be allowed.
  */
-ssize_t
-__generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
-               unsigned long nr_segs, loff_t *ppos)
+int generic_segment_checks(const struct iovec *iov,
+                       unsigned long *nr_segs, size_t *count, int access_flags)
 {
-       struct file *filp = iocb->ki_filp;
-       ssize_t retval;
-       unsigned long seg;
-       size_t count;
-
-       count = 0;
-       for (seg = 0; seg < nr_segs; seg++) {
+       unsigned long   seg;
+       size_t cnt = 0;
+       for (seg = 0; seg < *nr_segs; seg++) {
                const struct iovec *iv = &iov[seg];
 
                /*
                 * If any segment has a negative length, or the cumulative
                 * length ever wraps negative then return -EINVAL.
                 */
-               count += iv->iov_len;
-               if (unlikely((ssize_t)(count|iv->iov_len) < 0))
+               cnt += iv->iov_len;
+               if (unlikely((ssize_t)(cnt|iv->iov_len) < 0))
                        return -EINVAL;
-               if (access_ok(VERIFY_WRITE, iv->iov_base, iv->iov_len))
+               if (access_ok(access_flags, iv->iov_base, iv->iov_len))
                        continue;
                if (seg == 0)
                        return -EFAULT;
-               nr_segs = seg;
-               count -= iv->iov_len;   /* This segment is no good */
+               *nr_segs = seg;
+               cnt -= iv->iov_len;     /* This segment is no good */
                break;
        }
+       *count = cnt;
+       return 0;
+}
+EXPORT_SYMBOL(generic_segment_checks);
+
+/**
+ * generic_file_aio_read - generic filesystem read routine
+ * @iocb:      kernel I/O control block
+ * @iov:       io vector request
+ * @nr_segs:   number of segments in the iovec
+ * @pos:       current file position
+ *
+ * This is the "read()" routine for all filesystems
+ * that can use the page cache directly.
+ */
+ssize_t
+generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
+               unsigned long nr_segs, loff_t pos)
+{
+       struct file *filp = iocb->ki_filp;
+       ssize_t retval;
+       unsigned long seg;
+       size_t count;
+       loff_t *ppos = &iocb->ki_pos;
+
+       count = 0;
+       retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
+       if (retval)
+               return retval;
 
        /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
        if (filp->f_flags & O_DIRECT) {
-               loff_t pos = *ppos, size;
+               loff_t size;
                struct address_space *mapping;
                struct inode *inode;
 
@@ -1008,13 +1193,13 @@ __generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
                if (pos < size) {
                        retval = generic_file_direct_IO(READ, iocb,
                                                iov, pos, nr_segs);
-                       if (retval > 0 && !is_sync_kiocb(iocb))
-                               retval = -EIOCBQUEUED;
                        if (retval > 0)
                                *ppos = pos + retval;
                }
-               file_accessed(filp);
-               goto out;
+               if (likely(retval != 0)) {
+                       file_accessed(filp);
+                       goto out;
+               }
        }
 
        retval = 0;
@@ -1030,45 +1215,19 @@ __generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
                        desc.error = 0;
                        do_generic_file_read(filp,ppos,&desc,file_read_actor);
                        retval += desc.written;
-                       if (!retval) {
-                               retval = desc.error;
+                       if (desc.error) {
+                               retval = retval ?: desc.error;
                                break;
                        }
+                       if (desc.count > 0)
+                               break;
                }
        }
 out:
        return retval;
 }
-
-EXPORT_SYMBOL(__generic_file_aio_read);
-
-ssize_t
-generic_file_aio_read(struct kiocb *iocb, char __user *buf, size_t count, loff_t pos)
-{
-       struct iovec local_iov = { .iov_base = buf, .iov_len = count };
-
-       BUG_ON(iocb->ki_pos != pos);
-       return __generic_file_aio_read(iocb, &local_iov, 1, &iocb->ki_pos);
-}
-
 EXPORT_SYMBOL(generic_file_aio_read);
 
-ssize_t
-generic_file_read(struct file *filp, char __user *buf, size_t count, loff_t *ppos)
-{
-       struct iovec local_iov = { .iov_base = buf, .iov_len = count };
-       struct kiocb kiocb;
-       ssize_t ret;
-
-       init_sync_kiocb(&kiocb, filp);
-       ret = __generic_file_aio_read(&kiocb, &local_iov, 1, ppos);
-       if (-EIOCBQUEUED == ret)
-               ret = wait_on_sync_kiocb(&kiocb);
-       return ret;
-}
-
-EXPORT_SYMBOL(generic_file_read);
-
 int file_send_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size)
 {
        ssize_t written;
@@ -1089,27 +1248,6 @@ int file_send_actor(read_descriptor_t * desc, struct page *page, unsigned long o
        return written;
 }
 
-ssize_t generic_file_sendfile(struct file *in_file, loff_t *ppos,
-                        size_t count, read_actor_t actor, void *target)
-{
-       read_descriptor_t desc;
-
-       if (!count)
-               return 0;
-
-       desc.written = 0;
-       desc.count = count;
-       desc.arg.data = target;
-       desc.error = 0;
-
-       do_generic_file_read(in_file, ppos, &desc, actor);
-       if (desc.written)
-               return desc.written;
-       return desc.error;
-}
-
-EXPORT_SYMBOL(generic_file_sendfile);
-
 static ssize_t
 do_readahead(struct address_space *mapping, struct file *filp,
             unsigned long index, unsigned long nr)
@@ -1143,40 +1281,47 @@ asmlinkage ssize_t sys_readahead(int fd, loff_t offset, size_t count)
 }
 
 #ifdef CONFIG_MMU
-/*
+static int FASTCALL(page_cache_read(struct file * file, unsigned long offset));
+/**
+ * page_cache_read - adds requested page to the page cache if not already there
+ * @file:      file to read
+ * @offset:    page index
+ *
  * This adds the requested page to the page cache if it isn't already there,
  * and schedules an I/O to read in its contents from disk.
  */
-static int FASTCALL(page_cache_read(struct file * file, unsigned long offset));
 static int fastcall page_cache_read(struct file * file, unsigned long offset)
 {
        struct address_space *mapping = file->f_mapping;
        struct page *page; 
-       int error;
+       int ret;
 
-       page = page_cache_alloc_cold(mapping);
-       if (!page)
-               return -ENOMEM;
+       do {
+               page = page_cache_alloc_cold(mapping);
+               if (!page)
+                       return -ENOMEM;
+
+               ret = add_to_page_cache_lru(page, mapping, offset, GFP_KERNEL);
+               if (ret == 0)
+                       ret = mapping->a_ops->readpage(file, page);
+               else if (ret == -EEXIST)
+                       ret = 0; /* losing race to add is OK */
 
-       error = add_to_page_cache_lru(page, mapping, offset, GFP_KERNEL);
-       if (!error) {
-               error = mapping->a_ops->readpage(file, page);
                page_cache_release(page);
-               return error;
-       }
 
-       /*
-        * We arrive here in the unlikely event that someone 
-        * raced with us and added our page to the cache first
-        * or we are out of memory for radix-tree nodes.
-        */
-       page_cache_release(page);
-       return error == -EEXIST ? 0 : error;
+       } while (ret == AOP_TRUNCATED_PAGE);
+               
+       return ret;
 }
 
 #define MMAP_LOTSAMISS  (100)
 
-/*
+/**
+ * filemap_nopage - read in file data for page fault handling
+ * @area:      the applicable vm_area
+ * @address:   target address to read in
+ * @type:      returned with VM_FAULT_{MINOR,MAJOR} if not %NULL
+ *
  * filemap_nopage() is invoked via the vma operations vector for a
  * mapped memory region to read in file data during a page fault.
  *
@@ -1243,7 +1388,7 @@ retry_find:
                 */
                if (!did_readaround) {
                        majmin = VM_FAULT_MAJOR;
-                       inc_page_state(pgmajfault);
+                       count_vm_event(PGMAJFAULT);
                }
                did_readaround = 1;
                ra_pages = max_sane_readahead(file->f_ra.ra_pages);
@@ -1284,7 +1429,7 @@ outside_data_content:
         * accessible..
         */
        if (area->vm_mm == current->mm)
-               return NULL;
+               return NOPAGE_SIGBUS;
        /* Fall through to the non-read-ahead case */
 no_cached_page:
        /*
@@ -1292,7 +1437,6 @@ no_cached_page:
         * effect.
         */
        error = page_cache_read(file, pgoff);
-       grab_swap_token();
 
        /*
         * The page we want has now been added to the page cache.
@@ -1309,32 +1453,12 @@ no_cached_page:
         */
        if (error == -ENOMEM)
                return NOPAGE_OOM;
-       return NULL;
+       return NOPAGE_SIGBUS;
 
 page_not_uptodate:
        if (!did_readaround) {
                majmin = VM_FAULT_MAJOR;
-               inc_page_state(pgmajfault);
-       }
-       lock_page(page);
-
-       /* Did it get unhashed while we waited for it? */
-       if (!page->mapping) {
-               unlock_page(page);
-               page_cache_release(page);
-               goto retry_all;
-       }
-
-       /* Did somebody else get it up-to-date? */
-       if (PageUptodate(page)) {
-               unlock_page(page);
-               goto success;
-       }
-
-       if (!mapping->a_ops->readpage(file, page)) {
-               wait_on_page_locked(page);
-               if (PageUptodate(page))
-                       goto success;
+               count_vm_event(PGMAJFAULT);
        }
 
        /*
@@ -1358,20 +1482,24 @@ page_not_uptodate:
                goto success;
        }
        ClearPageError(page);
-       if (!mapping->a_ops->readpage(file, page)) {
+       error = mapping->a_ops->readpage(file, page);
+       if (!error) {
                wait_on_page_locked(page);
                if (PageUptodate(page))
                        goto success;
+       } else if (error == AOP_TRUNCATED_PAGE) {
+               page_cache_release(page);
+               goto retry_find;
        }
 
        /*
         * Things didn't work out. Return zero to tell the
         * mm layer so, possibly freeing the page cache page first.
         */
+       shrink_readahead_size_eio(file, ra);
        page_cache_release(page);
-       return NULL;
+       return NOPAGE_SIGBUS;
 }
-
 EXPORT_SYMBOL(filemap_nopage);
 
 static struct page * filemap_getpage(struct file *file, unsigned long pgoff,
@@ -1432,7 +1560,7 @@ no_cached_page:
 page_not_uptodate:
        lock_page(page);
 
-       /* Did it get unhashed while we waited for it? */
+       /* Did it get truncated while we waited for it? */
        if (!page->mapping) {
                unlock_page(page);
                goto err;
@@ -1444,10 +1572,14 @@ page_not_uptodate:
                goto success;
        }
 
-       if (!mapping->a_ops->readpage(file, page)) {
+       error = mapping->a_ops->readpage(file, page);
+       if (!error) {
                wait_on_page_locked(page);
                if (PageUptodate(page))
                        goto success;
+       } else if (error == AOP_TRUNCATED_PAGE) {
+               page_cache_release(page);
+               goto retry_find;
        }
 
        /*
@@ -1470,10 +1602,14 @@ page_not_uptodate:
        }
 
        ClearPageError(page);
-       if (!mapping->a_ops->readpage(file, page)) {
+       error = mapping->a_ops->readpage(file, page);
+       if (!error) {
                wait_on_page_locked(page);
                if (PageUptodate(page))
                        goto success;
+       } else if (error == AOP_TRUNCATED_PAGE) {
+               page_cache_release(page);
+               goto retry_find;
        }
 
        /*
@@ -1537,6 +1673,7 @@ repeat:
 
        return 0;
 }
+EXPORT_SYMBOL(filemap_populate);
 
 struct vm_operations_struct generic_file_vm_ops = {
        .nopage         = filemap_nopage,
@@ -1555,7 +1692,6 @@ int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
        vma->vm_ops = &generic_file_vm_ops;
        return 0;
 }
-EXPORT_SYMBOL(filemap_populate);
 
 /*
  * This is for filesystems which do not implement ->writepage.
@@ -1580,7 +1716,7 @@ int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
 EXPORT_SYMBOL(generic_file_mmap);
 EXPORT_SYMBOL(generic_file_readonly_mmap);
 
-static inline struct page *__read_cache_page(struct address_space *mapping,
+static struct page *__read_cache_page(struct address_space *mapping,
                                unsigned long index,
                                int (*filler)(void *,struct page*),
                                void *data)
@@ -1618,10 +1754,10 @@ repeat:
 }
 
 /*
- * Read into the page cache. If a page already exists,
- * and PageUptodate() is not set, try to fill the page.
+ * Same as read_cache_page, but don't wait for page to become unlocked
+ * after submitting it to the filler.
  */
-struct page *read_cache_page(struct address_space *mapping,
+struct page *read_cache_page_async(struct address_space *mapping,
                                unsigned long index,
                                int (*filler)(void *,struct page*),
                                void *data)
@@ -1632,8 +1768,7 @@ struct page *read_cache_page(struct address_space *mapping,
 retry:
        page = __read_cache_page(mapping, index, filler, data);
        if (IS_ERR(page))
-               goto out;
-       mark_page_accessed(page);
+               return page;
        if (PageUptodate(page))
                goto out;
 
@@ -1650,12 +1785,44 @@ retry:
        err = filler(data, page);
        if (err < 0) {
                page_cache_release(page);
-               page = ERR_PTR(err);
+               return ERR_PTR(err);
        }
- out:
+out:
+       mark_page_accessed(page);
        return page;
 }
+EXPORT_SYMBOL(read_cache_page_async);
 
+/**
+ * read_cache_page - read into page cache, fill it if needed
+ * @mapping:   the page's address_space
+ * @index:     the page index
+ * @filler:    function to perform the read
+ * @data:      destination for read data
+ *
+ * Read into the page cache. If a page already exists, and PageUptodate() is
+ * not set, try to fill the page then wait for it to become unlocked.
+ *
+ * If the page does not get brought uptodate, return -EIO.
+ */
+struct page *read_cache_page(struct address_space *mapping,
+                               unsigned long index,
+                               int (*filler)(void *,struct page*),
+                               void *data)
+{
+       struct page *page;
+
+       page = read_cache_page_async(mapping, index, filler, data);
+       if (IS_ERR(page))
+               goto out;
+       wait_on_page_locked(page);
+       if (!PageUptodate(page)) {
+               page_cache_release(page);
+               page = ERR_PTR(-EIO);
+       }
+ out:
+       return page;
+}
 EXPORT_SYMBOL(read_cache_page);
 
 /*
@@ -1698,11 +1865,10 @@ repeat:
  *     if suid or (sgid and xgrp)
  *             remove privs
  */
-int remove_suid(struct dentry *dentry)
+int should_remove_suid(struct dentry *dentry)
 {
        mode_t mode = dentry->d_inode->i_mode;
        int kill = 0;
-       int result = 0;
 
        /* suid always must be killed */
        if (unlikely(mode & S_ISUID))
@@ -1715,18 +1881,34 @@ int remove_suid(struct dentry *dentry)
        if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
                kill |= ATTR_KILL_SGID;
 
-       if (unlikely(kill && !capable(CAP_FSETID))) {
-               struct iattr newattrs;
+       if (unlikely(kill && !capable(CAP_FSETID)))
+               return kill;
 
-               newattrs.ia_valid = ATTR_FORCE | kill;
-               result = notify_change(dentry, &newattrs);
-       }
-       return result;
+       return 0;
+}
+EXPORT_SYMBOL(should_remove_suid);
+
+int __remove_suid(struct dentry *dentry, int kill)
+{
+       struct iattr newattrs;
+
+       newattrs.ia_valid = ATTR_FORCE | kill;
+       return notify_change(dentry, &newattrs);
+}
+
+int remove_suid(struct dentry *dentry)
+{
+       int kill = should_remove_suid(dentry);
+
+       if (unlikely(kill))
+               return __remove_suid(dentry, kill);
+
+       return 0;
 }
 EXPORT_SYMBOL(remove_suid);
 
 size_t
-__filemap_copy_from_user_iovec(char *vaddr, 
+__filemap_copy_from_user_iovec_inatomic(char *vaddr,
                        const struct iovec *iov, size_t base, size_t bytes)
 {
        size_t copied = 0, left = 0;
@@ -1736,18 +1918,14 @@ __filemap_copy_from_user_iovec(char *vaddr,
                int copy = min(bytes, iov->iov_len - base);
 
                base = 0;
-               left = __copy_from_user_inatomic(vaddr, buf, copy);
+               left = __copy_from_user_inatomic_nocache(vaddr, buf, copy);
                copied += copy;
                bytes -= copy;
                vaddr += copy;
                iov++;
 
-               if (unlikely(left)) {
-                       /* zero the rest of the target like __copy_from_user */
-                       if (bytes)
-                               memset(vaddr, 0, bytes);
+               if (unlikely(left))
                        break;
-               }
        }
        return copied - left;
 }
@@ -1755,7 +1933,7 @@ __filemap_copy_from_user_iovec(char *vaddr,
 /*
  * Performs necessary checks before doing a write
  *
- * Can adjust writing position aor amount of bytes to write.
+ * Can adjust writing position or amount of bytes to write.
  * Returns appropriate error code that caller should return or
  * zero in case that write should be allowed.
  */
@@ -1816,6 +1994,7 @@ inline int generic_write_checks(struct file *file, loff_t *pos, size_t *count, i
                if (unlikely(*pos + *count > inode->i_sb->s_maxbytes))
                        *count = inode->i_sb->s_maxbytes - *pos;
        } else {
+#ifdef CONFIG_BLOCK
                loff_t isize;
                if (bdev_read_only(I_BDEV(inode)))
                        return -EPERM;
@@ -1827,6 +2006,9 @@ inline int generic_write_checks(struct file *file, loff_t *pos, size_t *count, i
 
                if (*pos + *count > isize)
                        *count = isize - *pos;
+#else
+               return -EPERM;
+#endif
        }
        return 0;
 }
@@ -1858,16 +2040,15 @@ generic_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
        /*
         * Sync the fs metadata but not the minor inode changes and
         * of course not the data as we did direct DMA for the IO.
-        * i_sem is held, which protects generic_osync_inode() from
-        * livelocking.
+        * i_mutex is held, which protects generic_osync_inode() from
+        * livelocking.  AIO O_DIRECT ops attempt to sync metadata here.
         */
-       if (written >= 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
+       if ((written >= 0 || written == -EIOCBQUEUED) &&
+           ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
                int err = generic_osync_inode(inode, mapping, OSYNC_METADATA);
                if (err < 0)
                        written = err;
        }
-       if (written == count && !is_sync_kiocb(iocb))
-               written = -EIOCBQUEUED;
        return written;
 }
 EXPORT_SYMBOL(generic_file_direct_write);
@@ -1879,7 +2060,7 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
 {
        struct file *file = iocb->ki_filp;
        struct address_space * mapping = file->f_mapping;
-       struct address_space_operations *a_ops = mapping->a_ops;
+       const struct address_space_operations *a_ops = mapping->a_ops;
        struct inode    *inode = mapping->host;
        long            status = 0;
        struct page     *page;
@@ -1905,41 +2086,61 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
        do {
                unsigned long index;
                unsigned long offset;
-               unsigned long maxlen;
                size_t copied;
 
                offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */
                index = pos >> PAGE_CACHE_SHIFT;
                bytes = PAGE_CACHE_SIZE - offset;
-               if (bytes > count)
-                       bytes = count;
 
-               /*
-                * Bring in the user page that we will copy from _first_.
-                * Otherwise there's a nasty deadlock on copying from the
-                * same page as we're writing to, without it being marked
-                * up-to-date.
+               /* Limit the size of the copy to the caller's write size */
+               bytes = min(bytes, count);
+
+               /* We only need to worry about prefaulting when writes are from
+                * user-space.  NFSd uses vfs_writev with several non-aligned
+                * segments in the vector, and limiting to one segment a time is
+                * a noticeable performance for re-write
                 */
-               maxlen = cur_iov->iov_len - iov_base;
-               if (maxlen > bytes)
-                       maxlen = bytes;
-               fault_in_pages_readable(buf, maxlen);
+               if (!segment_eq(get_fs(), KERNEL_DS)) {
+                       /*
+                        * Limit the size of the copy to that of the current
+                        * segment, because fault_in_pages_readable() doesn't
+                        * know how to walk segments.
+                        */
+                       bytes = min(bytes, cur_iov->iov_len - iov_base);
 
+                       /*
+                        * Bring in the user page that we will copy from
+                        * _first_.  Otherwise there's a nasty deadlock on
+                        * copying from the same page as we're writing to,
+                        * without it being marked up-to-date.
+                        */
+                       fault_in_pages_readable(buf, bytes);
+               }
                page = __grab_cache_page(mapping,index,&cached_page,&lru_pvec);
                if (!page) {
                        status = -ENOMEM;
                        break;
                }
 
+               if (unlikely(bytes == 0)) {
+                       status = 0;
+                       copied = 0;
+                       goto zero_length_segment;
+               }
+
                status = a_ops->prepare_write(file, page, offset, offset+bytes);
                if (unlikely(status)) {
                        loff_t isize = i_size_read(inode);
+
+                       if (status != AOP_TRUNCATED_PAGE)
+                               unlock_page(page);
+                       page_cache_release(page);
+                       if (status == AOP_TRUNCATED_PAGE)
+                               continue;
                        /*
                         * prepare_write() may have instantiated a few blocks
                         * outside i_size.  Trim these off again.
                         */
-                       unlock_page(page);
-                       page_cache_release(page);
                        if (pos + bytes > isize)
                                vmtruncate(inode, isize);
                        break;
@@ -1952,7 +2153,12 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
                                                cur_iov, iov_base, bytes);
                flush_dcache_page(page);
                status = a_ops->commit_write(file, page, offset, offset+bytes);
-               if (likely(copied > 0)) {
+               if (status == AOP_TRUNCATED_PAGE) {
+                       page_cache_release(page);
+                       continue;
+               }
+zero_length_segment:
+               if (likely(copied >= 0)) {
                        if (!status)
                                status = copied;
 
@@ -2021,30 +2227,14 @@ __generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
        size_t ocount;          /* original count */
        size_t count;           /* after file limit checks */
        struct inode    *inode = mapping->host;
-       unsigned long   seg;
        loff_t          pos;
        ssize_t         written;
        ssize_t         err;
 
        ocount = 0;
-       for (seg = 0; seg < nr_segs; seg++) {
-               const struct iovec *iv = &iov[seg];
-
-               /*
-                * If any segment has a negative length, or the cumulative
-                * length ever wraps negative then return -EINVAL.
-                */
-               ocount += iv->iov_len;
-               if (unlikely((ssize_t)(ocount|iv->iov_len) < 0))
-                       return -EINVAL;
-               if (access_ok(VERIFY_READ, iv->iov_base, iv->iov_len))
-                       continue;
-               if (seg == 0)
-                       return -EFAULT;
-               nr_segs = seg;
-               ocount -= iv->iov_len;  /* This segment is no good */
-               break;
-       }
+       err = generic_segment_checks(iov, &nr_segs, &ocount, VERIFY_READ);
+       if (err)
+               return err;
 
        count = ocount;
        pos = *ppos;
@@ -2062,16 +2252,19 @@ __generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
        if (count == 0)
                goto out;
 
-       err = remove_suid(file->f_dentry);
+       err = remove_suid(file->f_path.dentry);
        if (err)
                goto out;
 
-       inode_update_time(inode, 1);
+       file_update_time(file);
 
        /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
        if (unlikely(file->f_flags & O_DIRECT)) {
-               written = generic_file_direct_write(iocb, iov,
-                               &nr_segs, pos, ppos, count, ocount);
+               loff_t endbyte;
+               ssize_t written_buffered;
+
+               written = generic_file_direct_write(iocb, iov, &nr_segs, pos,
+                                                       ppos, count, ocount);
                if (written < 0 || written == count)
                        goto out;
                /*
@@ -2080,30 +2273,66 @@ __generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
                 */
                pos += written;
                count -= written;
-       }
+               written_buffered = generic_file_buffered_write(iocb, iov,
+                                               nr_segs, pos, ppos, count,
+                                               written);
+               /*
+                * If generic_file_buffered_write() retuned a synchronous error
+                * then we want to return the number of bytes which were
+                * direct-written, or the error code if that was zero.  Note
+                * that this differs from normal direct-io semantics, which
+                * will return -EFOO even if some bytes were written.
+                */
+               if (written_buffered < 0) {
+                       err = written_buffered;
+                       goto out;
+               }
 
-       written = generic_file_buffered_write(iocb, iov, nr_segs,
-                       pos, ppos, count, written);
+               /*
+                * We need to ensure that the page cache pages are written to
+                * disk and invalidated to preserve the expected O_DIRECT
+                * semantics.
+                */
+               endbyte = pos + written_buffered - written - 1;
+               err = do_sync_mapping_range(file->f_mapping, pos, endbyte,
+                                           SYNC_FILE_RANGE_WAIT_BEFORE|
+                                           SYNC_FILE_RANGE_WRITE|
+                                           SYNC_FILE_RANGE_WAIT_AFTER);
+               if (err == 0) {
+                       written = written_buffered;
+                       invalidate_mapping_pages(mapping,
+                                                pos >> PAGE_CACHE_SHIFT,
+                                                endbyte >> PAGE_CACHE_SHIFT);
+               } else {
+                       /*
+                        * We don't know how much we wrote, so just return
+                        * the number of bytes which were direct-written
+                        */
+               }
+       } else {
+               written = generic_file_buffered_write(iocb, iov, nr_segs,
+                               pos, ppos, count, written);
+       }
 out:
        current->backing_dev_info = NULL;
        return written ? written : err;
 }
-EXPORT_SYMBOL(generic_file_aio_write_nolock);
 
-ssize_t
-generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
-                               unsigned long nr_segs, loff_t *ppos)
+ssize_t generic_file_aio_write_nolock(struct kiocb *iocb,
+               const struct iovec *iov, unsigned long nr_segs, loff_t pos)
 {
        struct file *file = iocb->ki_filp;
        struct address_space *mapping = file->f_mapping;
        struct inode *inode = mapping->host;
        ssize_t ret;
-       loff_t pos = *ppos;
 
-       ret = __generic_file_aio_write_nolock(iocb, iov, nr_segs, ppos);
+       BUG_ON(iocb->ki_pos != pos);
+
+       ret = __generic_file_aio_write_nolock(iocb, iov, nr_segs,
+                       &iocb->ki_pos);
 
        if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
-               int err;
+               ssize_t err;
 
                err = sync_page_range_nolock(inode, mapping, pos, ret);
                if (err < 0)
@@ -2111,52 +2340,22 @@ generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
        }
        return ret;
 }
+EXPORT_SYMBOL(generic_file_aio_write_nolock);
 
-static ssize_t
-__generic_file_write_nolock(struct file *file, const struct iovec *iov,
-                               unsigned long nr_segs, loff_t *ppos)
-{
-       struct kiocb kiocb;
-       ssize_t ret;
-
-       init_sync_kiocb(&kiocb, file);
-       ret = __generic_file_aio_write_nolock(&kiocb, iov, nr_segs, ppos);
-       if (ret == -EIOCBQUEUED)
-               ret = wait_on_sync_kiocb(&kiocb);
-       return ret;
-}
-
-ssize_t
-generic_file_write_nolock(struct file *file, const struct iovec *iov,
-                               unsigned long nr_segs, loff_t *ppos)
-{
-       struct kiocb kiocb;
-       ssize_t ret;
-
-       init_sync_kiocb(&kiocb, file);
-       ret = generic_file_aio_write_nolock(&kiocb, iov, nr_segs, ppos);
-       if (-EIOCBQUEUED == ret)
-               ret = wait_on_sync_kiocb(&kiocb);
-       return ret;
-}
-EXPORT_SYMBOL(generic_file_write_nolock);
-
-ssize_t generic_file_aio_write(struct kiocb *iocb, const char __user *buf,
-                              size_t count, loff_t pos)
+ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
+               unsigned long nr_segs, loff_t pos)
 {
        struct file *file = iocb->ki_filp;
        struct address_space *mapping = file->f_mapping;
        struct inode *inode = mapping->host;
        ssize_t ret;
-       struct iovec local_iov = { .iov_base = (void __user *)buf,
-                                       .iov_len = count };
 
        BUG_ON(iocb->ki_pos != pos);
 
-       down(&inode->i_sem);
-       ret = __generic_file_aio_write_nolock(iocb, &local_iov, 1,
-                                               &iocb->ki_pos);
-       up(&inode->i_sem);
+       mutex_lock(&inode->i_mutex);
+       ret = __generic_file_aio_write_nolock(iocb, iov, nr_segs,
+                       &iocb->ki_pos);
+       mutex_unlock(&inode->i_mutex);
 
        if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
                ssize_t err;
@@ -2169,68 +2368,8 @@ ssize_t generic_file_aio_write(struct kiocb *iocb, const char __user *buf,
 }
 EXPORT_SYMBOL(generic_file_aio_write);
 
-ssize_t generic_file_write(struct file *file, const char __user *buf,
-                          size_t count, loff_t *ppos)
-{
-       struct address_space *mapping = file->f_mapping;
-       struct inode *inode = mapping->host;
-       ssize_t ret;
-       struct iovec local_iov = { .iov_base = (void __user *)buf,
-                                       .iov_len = count };
-
-       down(&inode->i_sem);
-       ret = __generic_file_write_nolock(file, &local_iov, 1, ppos);
-       up(&inode->i_sem);
-
-       if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
-               ssize_t err;
-
-               err = sync_page_range(inode, mapping, *ppos - ret, ret);
-               if (err < 0)
-                       ret = err;
-       }
-       return ret;
-}
-EXPORT_SYMBOL(generic_file_write);
-
-ssize_t generic_file_readv(struct file *filp, const struct iovec *iov,
-                       unsigned long nr_segs, loff_t *ppos)
-{
-       struct kiocb kiocb;
-       ssize_t ret;
-
-       init_sync_kiocb(&kiocb, filp);
-       ret = __generic_file_aio_read(&kiocb, iov, nr_segs, ppos);
-       if (-EIOCBQUEUED == ret)
-               ret = wait_on_sync_kiocb(&kiocb);
-       return ret;
-}
-EXPORT_SYMBOL(generic_file_readv);
-
-ssize_t generic_file_writev(struct file *file, const struct iovec *iov,
-                       unsigned long nr_segs, loff_t *ppos)
-{
-       struct address_space *mapping = file->f_mapping;
-       struct inode *inode = mapping->host;
-       ssize_t ret;
-
-       down(&inode->i_sem);
-       ret = __generic_file_write_nolock(file, iov, nr_segs, ppos);
-       up(&inode->i_sem);
-
-       if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
-               int err;
-
-               err = sync_page_range(inode, mapping, *ppos - ret, ret);
-               if (err < 0)
-                       ret = err;
-       }
-       return ret;
-}
-EXPORT_SYMBOL(generic_file_writev);
-
 /*
- * Called under i_sem for writes to S_ISREG files.   Returns -EIO if something
+ * Called under i_mutex for writes to S_ISREG files.   Returns -EIO if something
  * went wrong during pagecache shootdown.
  */
 static ssize_t
@@ -2240,7 +2379,8 @@ generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
        struct file *file = iocb->ki_filp;
        struct address_space *mapping = file->f_mapping;
        ssize_t retval;
-       size_t write_len = 0;
+       size_t write_len;
+       pgoff_t end = 0; /* silence gcc */
 
        /*
         * If it's a write, unmap all mmappings of the file up-front.  This
@@ -2249,22 +2389,75 @@ generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
         */
        if (rw == WRITE) {
                write_len = iov_length(iov, nr_segs);
+               end = (offset + write_len - 1) >> PAGE_CACHE_SHIFT;
                if (mapping_mapped(mapping))
                        unmap_mapping_range(mapping, offset, write_len, 0);
        }
 
        retval = filemap_write_and_wait(mapping);
-       if (retval == 0) {
-               retval = mapping->a_ops->direct_IO(rw, iocb, iov,
-                                               offset, nr_segs);
-               if (rw == WRITE && mapping->nrpages) {
-                       pgoff_t end = (offset + write_len - 1)
-                                               >> PAGE_CACHE_SHIFT;
-                       int err = invalidate_inode_pages2_range(mapping,
+       if (retval)
+               goto out;
+
+       /*
+        * After a write we want buffered reads to be sure to go to disk to get
+        * the new data.  We invalidate clean cached page from the region we're
+        * about to write.  We do this *before* the write so that we can return
+        * -EIO without clobbering -EIOCBQUEUED from ->direct_IO().
+        */
+       if (rw == WRITE && mapping->nrpages) {
+               retval = invalidate_inode_pages2_range(mapping,
                                        offset >> PAGE_CACHE_SHIFT, end);
-                       if (err)
-                               retval = err;
-               }
+               if (retval)
+                       goto out;
+       }
+
+       retval = mapping->a_ops->direct_IO(rw, iocb, iov, offset, nr_segs);
+       if (retval)
+               goto out;
+
+       /*
+        * Finally, try again to invalidate clean pages which might have been
+        * faulted in by get_user_pages() if the source of the write was an
+        * mmap()ed region of the file we're writing.  That's a pretty crazy
+        * thing to do, so we don't support it 100%.  If this invalidation
+        * fails and we have -EIOCBQUEUED we ignore the failure.
+        */
+       if (rw == WRITE && mapping->nrpages) {
+               int err = invalidate_inode_pages2_range(mapping,
+                                             offset >> PAGE_CACHE_SHIFT, end);
+               if (err && retval >= 0)
+                       retval = err;
        }
+out:
        return retval;
 }
+
+/**
+ * try_to_release_page() - release old fs-specific metadata on a page
+ *
+ * @page: the page which the kernel is trying to free
+ * @gfp_mask: memory allocation flags (and I/O mode)
+ *
+ * The address_space is to try to release any data against the page
+ * (presumably at page->private).  If the release was successful, return `1'.
+ * Otherwise return zero.
+ *
+ * The @gfp_mask argument specifies whether I/O may be performed to release
+ * this page (__GFP_IO), and whether the call may block (__GFP_WAIT).
+ *
+ * NOTE: @gfp_mask may go away, and this function may become non-blocking.
+ */
+int try_to_release_page(struct page *page, gfp_t gfp_mask)
+{
+       struct address_space * const mapping = page->mapping;
+
+       BUG_ON(!PageLocked(page));
+       if (PageWriteback(page))
+               return 0;
+
+       if (mapping && mapping->a_ops->releasepage)
+               return mapping->a_ops->releasepage(page, gfp_mask);
+       return try_to_free_buffers(page);
+}
+
+EXPORT_SYMBOL(try_to_release_page);