mm: trim more holes
[linux-2.6.git] / mm / filemap.c
index f30ef28..73b98c6 100644 (file)
@@ -120,6 +120,7 @@ void __remove_from_page_cache(struct page *page)
        page->mapping = NULL;
        mapping->nrpages--;
        __dec_zone_page_state(page, NR_FILE_PAGES);
+       BUG_ON(page_mapped(page));
 }
 
 void remove_from_page_cache(struct page *page)
@@ -327,7 +328,7 @@ EXPORT_SYMBOL(sync_page_range);
  * @pos:       beginning offset in pages to write
  * @count:     number of bytes to write
  *
- * Note: Holding i_mutex across sync_page_range_nolock is not a good idea
+ * Note: Holding i_mutex across sync_page_range_nolock() is not a good idea
  * as it forces O_SYNC writers to different parts of the same file
  * to be serialised right until io completion.
  */
@@ -592,7 +593,7 @@ void fastcall __lock_page_nosync(struct page *page)
  * Is there a pagecache struct page at the given (mapping, offset) tuple?
  * If yes, increment its refcount and return it; if no, return NULL.
  */
-struct page * find_get_page(struct address_space *mapping, unsigned long offset)
+struct page * find_get_page(struct address_space *mapping, pgoff_t offset)
 {
        struct page *page;
 
@@ -616,30 +617,31 @@ EXPORT_SYMBOL(find_get_page);
  * Returns zero if the page was not present. find_lock_page() may sleep.
  */
 struct page *find_lock_page(struct address_space *mapping,
-                               unsigned long offset)
+                               pgoff_t offset)
 {
        struct page *page;
 
-       read_lock_irq(&mapping->tree_lock);
 repeat:
+       read_lock_irq(&mapping->tree_lock);
        page = radix_tree_lookup(&mapping->page_tree, offset);
        if (page) {
                page_cache_get(page);
                if (TestSetPageLocked(page)) {
                        read_unlock_irq(&mapping->tree_lock);
                        __lock_page(page);
-                       read_lock_irq(&mapping->tree_lock);
 
                        /* Has the page been truncated while we slept? */
-                       if (unlikely(page->mapping != mapping ||
-                                    page->index != offset)) {
+                       if (unlikely(page->mapping != mapping)) {
                                unlock_page(page);
                                page_cache_release(page);
                                goto repeat;
                        }
+                       VM_BUG_ON(page->index != offset);
+                       goto out;
                }
        }
        read_unlock_irq(&mapping->tree_lock);
+out:
        return page;
 }
 EXPORT_SYMBOL(find_lock_page);
@@ -662,7 +664,7 @@ EXPORT_SYMBOL(find_lock_page);
  * memory exhaustion.
  */
 struct page *find_or_create_page(struct address_space *mapping,
-               unsigned long index, gfp_t gfp_mask)
+               pgoff_t index, gfp_t gfp_mask)
 {
        struct page *page, *cached_page = NULL;
        int err;
@@ -670,7 +672,8 @@ repeat:
        page = find_lock_page(mapping, index);
        if (!page) {
                if (!cached_page) {
-                       cached_page = alloc_page(gfp_mask);
+                       cached_page =
+                               __page_cache_alloc(gfp_mask);
                        if (!cached_page)
                                return NULL;
                }
@@ -750,6 +753,7 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
        read_unlock_irq(&mapping->tree_lock);
        return i;
 }
+EXPORT_SYMBOL(find_get_pages_contig);
 
 /**
  * find_get_pages_tag - find and return pages that match @tag
@@ -778,13 +782,14 @@ unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
        read_unlock_irq(&mapping->tree_lock);
        return ret;
 }
+EXPORT_SYMBOL(find_get_pages_tag);
 
 /**
  * grab_cache_page_nowait - returns locked page at given index in given cache
  * @mapping: target address_space
  * @index: the page index
  *
- * Same as grab_cache_page, but do not wait if the page is unavailable.
+ * Same as grab_cache_page(), but do not wait if the page is unavailable.
  * This is intended for speculative data generators, where the data can
  * be regenerated if the page couldn't be grabbed.  This routine should
  * be safe to call while holding the lock for another page.
@@ -793,7 +798,7 @@ unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
  * and deadlock against the caller's locked page.
  */
 struct page *
-grab_cache_page_nowait(struct address_space *mapping, unsigned long index)
+grab_cache_page_nowait(struct address_space *mapping, pgoff_t index)
 {
        struct page *page = find_get_page(mapping, index);
 
@@ -855,67 +860,80 @@ static void shrink_readahead_size_eio(struct file *filp,
  * It may be NULL.
  */
 void do_generic_mapping_read(struct address_space *mapping,
-                            struct file_ra_state *_ra,
+                            struct file_ra_state *ra,
                             struct file *filp,
                             loff_t *ppos,
                             read_descriptor_t *desc,
                             read_actor_t actor)
 {
        struct inode *inode = mapping->host;
-       unsigned long index;
-       unsigned long end_index;
-       unsigned long offset;
-       unsigned long last_index;
-       unsigned long next_index;
-       unsigned long prev_index;
-       loff_t isize;
+       pgoff_t index;
+       pgoff_t last_index;
+       pgoff_t prev_index;
+       unsigned long offset;      /* offset into pagecache page */
+       unsigned int prev_offset;
        struct page *cached_page;
        int error;
-       struct file_ra_state ra = *_ra;
 
        cached_page = NULL;
        index = *ppos >> PAGE_CACHE_SHIFT;
-       next_index = index;
-       prev_index = ra.prev_page;
+       prev_index = ra->prev_pos >> PAGE_CACHE_SHIFT;
+       prev_offset = ra->prev_pos & (PAGE_CACHE_SIZE-1);
        last_index = (*ppos + desc->count + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
        offset = *ppos & ~PAGE_CACHE_MASK;
 
-       isize = i_size_read(inode);
-       if (!isize)
-               goto out;
-
-       end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
        for (;;) {
                struct page *page;
+               pgoff_t end_index;
+               loff_t isize;
                unsigned long nr, ret;
 
+               cond_resched();
+find_page:
+               page = find_get_page(mapping, index);
+               if (!page) {
+                       page_cache_sync_readahead(mapping,
+                                       ra, filp,
+                                       index, last_index - index);
+                       page = find_get_page(mapping, index);
+                       if (unlikely(page == NULL))
+                               goto no_cached_page;
+               }
+               if (PageReadahead(page)) {
+                       page_cache_async_readahead(mapping,
+                                       ra, filp, page,
+                                       index, last_index - index);
+               }
+               if (!PageUptodate(page))
+                       goto page_not_up_to_date;
+page_ok:
+               /*
+                * i_size must be checked after we know the page is Uptodate.
+                *
+                * Checking i_size after the check allows us to calculate
+                * the correct value for "nr", which means the zero-filled
+                * part of the page is not copied back to userspace (unless
+                * another truncate extends the file - this is desired though).
+                */
+
+               isize = i_size_read(inode);
+               end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
+               if (unlikely(!isize || index > end_index)) {
+                       page_cache_release(page);
+                       goto out;
+               }
+
                /* nr is the maximum number of bytes to copy from this page */
                nr = PAGE_CACHE_SIZE;
-               if (index >= end_index) {
-                       if (index > end_index)
-                               goto out;
+               if (index == end_index) {
                        nr = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
                        if (nr <= offset) {
+                               page_cache_release(page);
                                goto out;
                        }
                }
                nr = nr - offset;
 
-               cond_resched();
-               if (index == next_index)
-                       next_index = page_cache_readahead(mapping, &ra, filp,
-                                       index, last_index - index);
-
-find_page:
-               page = find_get_page(mapping, index);
-               if (unlikely(page == NULL)) {
-                       handle_ra_miss(mapping, &ra, index);
-                       goto no_cached_page;
-               }
-               if (!PageUptodate(page))
-                       goto page_not_up_to_date;
-page_ok:
-
                /* If users can be writing to this page using arbitrary
                 * virtual addresses, take care about potential aliasing
                 * before reading the page on the kernel side.
@@ -924,10 +942,10 @@ page_ok:
                        flush_dcache_page(page);
 
                /*
-                * When (part of) the same page is read multiple times
-                * in succession, only mark it as accessed the first time.
+                * When a sequential read accesses a page several times,
+                * only mark it as accessed the first time.
                 */
-               if (prev_index != index)
+               if (prev_index != index || offset != prev_offset)
                        mark_page_accessed(page);
                prev_index = index;
 
@@ -945,6 +963,7 @@ page_ok:
                offset += ret;
                index += offset >> PAGE_CACHE_SHIFT;
                offset &= ~PAGE_CACHE_MASK;
+               prev_offset = offset;
 
                page_cache_release(page);
                if (ret == nr && desc->count)
@@ -993,37 +1012,12 @@ readpage:
                                }
                                unlock_page(page);
                                error = -EIO;
-                               shrink_readahead_size_eio(filp, &ra);
+                               shrink_readahead_size_eio(filp, ra);
                                goto readpage_error;
                        }
                        unlock_page(page);
                }
 
-               /*
-                * i_size must be checked after we have done ->readpage.
-                *
-                * Checking i_size after the readpage allows us to calculate
-                * the correct value for "nr", which means the zero-filled
-                * part of the page is not copied back to userspace (unless
-                * another truncate extends the file - this is desired though).
-                */
-               isize = i_size_read(inode);
-               end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
-               if (unlikely(!isize || index > end_index)) {
-                       page_cache_release(page);
-                       goto out;
-               }
-
-               /* nr is the maximum number of bytes to copy from this page */
-               nr = PAGE_CACHE_SIZE;
-               if (index == end_index) {
-                       nr = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
-                       if (nr <= offset) {
-                               page_cache_release(page);
-                               goto out;
-                       }
-               }
-               nr = nr - offset;
                goto page_ok;
 
 readpage_error:
@@ -1058,9 +1052,11 @@ no_cached_page:
        }
 
 out:
-       *_ra = ra;
+       ra->prev_pos = prev_index;
+       ra->prev_pos <<= PAGE_CACHE_SHIFT;
+       ra->prev_pos |= prev_offset;
 
-       *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
+       *ppos = ((loff_t)index << PAGE_CACHE_SHIFT) + offset;
        if (cached_page)
                page_cache_release(cached_page);
        if (filp)
@@ -1106,6 +1102,45 @@ success:
        return size;
 }
 
+/*
+ * Performs necessary checks before doing a write
+ * @iov:       io vector request
+ * @nr_segs:   number of segments in the iovec
+ * @count:     number of bytes to write
+ * @access_flags: type of access: %VERIFY_READ or %VERIFY_WRITE
+ *
+ * Adjust number of segments and amount of bytes to write (nr_segs should be
+ * properly initialized first). Returns appropriate error code that caller
+ * should return or zero in case that write should be allowed.
+ */
+int generic_segment_checks(const struct iovec *iov,
+                       unsigned long *nr_segs, size_t *count, int access_flags)
+{
+       unsigned long   seg;
+       size_t cnt = 0;
+       for (seg = 0; seg < *nr_segs; seg++) {
+               const struct iovec *iv = &iov[seg];
+
+               /*
+                * If any segment has a negative length, or the cumulative
+                * length ever wraps negative then return -EINVAL.
+                */
+               cnt += iv->iov_len;
+               if (unlikely((ssize_t)(cnt|iv->iov_len) < 0))
+                       return -EINVAL;
+               if (access_ok(access_flags, iv->iov_base, iv->iov_len))
+                       continue;
+               if (seg == 0)
+                       return -EFAULT;
+               *nr_segs = seg;
+               cnt -= iv->iov_len;     /* This segment is no good */
+               break;
+       }
+       *count = cnt;
+       return 0;
+}
+EXPORT_SYMBOL(generic_segment_checks);
+
 /**
  * generic_file_aio_read - generic filesystem read routine
  * @iocb:      kernel I/O control block
@@ -1127,24 +1162,9 @@ generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
        loff_t *ppos = &iocb->ki_pos;
 
        count = 0;
-       for (seg = 0; seg < nr_segs; seg++) {
-               const struct iovec *iv = &iov[seg];
-
-               /*
-                * If any segment has a negative length, or the cumulative
-                * length ever wraps negative then return -EINVAL.
-                */
-               count += iv->iov_len;
-               if (unlikely((ssize_t)(count|iv->iov_len) < 0))
-                       return -EINVAL;
-               if (access_ok(VERIFY_WRITE, iv->iov_base, iv->iov_len))
-                       continue;
-               if (seg == 0)
-                       return -EFAULT;
-               nr_segs = seg;
-               count -= iv->iov_len;   /* This segment is no good */
-               break;
-       }
+       retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
+       if (retval)
+               return retval;
 
        /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
        if (filp->f_flags & O_DIRECT) {
@@ -1187,6 +1207,8 @@ generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
                                retval = retval ?: desc.error;
                                break;
                        }
+                       if (desc.count > 0)
+                               break;
                }
        }
 out:
@@ -1194,49 +1216,9 @@ out:
 }
 EXPORT_SYMBOL(generic_file_aio_read);
 
-int file_send_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size)
-{
-       ssize_t written;
-       unsigned long count = desc->count;
-       struct file *file = desc->arg.data;
-
-       if (size > count)
-               size = count;
-
-       written = file->f_op->sendpage(file, page, offset,
-                                      size, &file->f_pos, size<count);
-       if (written < 0) {
-               desc->error = written;
-               written = 0;
-       }
-       desc->count = count - written;
-       desc->written += written;
-       return written;
-}
-
-ssize_t generic_file_sendfile(struct file *in_file, loff_t *ppos,
-                        size_t count, read_actor_t actor, void *target)
-{
-       read_descriptor_t desc;
-
-       if (!count)
-               return 0;
-
-       desc.written = 0;
-       desc.count = count;
-       desc.arg.data = target;
-       desc.error = 0;
-
-       do_generic_file_read(in_file, ppos, &desc, actor);
-       if (desc.written)
-               return desc.written;
-       return desc.error;
-}
-EXPORT_SYMBOL(generic_file_sendfile);
-
 static ssize_t
 do_readahead(struct address_space *mapping, struct file *filp,
-            unsigned long index, unsigned long nr)
+            pgoff_t index, unsigned long nr)
 {
        if (!mapping || !mapping->a_ops || !mapping->a_ops->readpage)
                return -EINVAL;
@@ -1256,8 +1238,8 @@ asmlinkage ssize_t sys_readahead(int fd, loff_t offset, size_t count)
        if (file) {
                if (file->f_mode & FMODE_READ) {
                        struct address_space *mapping = file->f_mapping;
-                       unsigned long start = offset >> PAGE_CACHE_SHIFT;
-                       unsigned long end = (offset + count - 1) >> PAGE_CACHE_SHIFT;
+                       pgoff_t start = offset >> PAGE_CACHE_SHIFT;
+                       pgoff_t end = (offset + count - 1) >> PAGE_CACHE_SHIFT;
                        unsigned long len = end - start + 1;
                        ret = do_readahead(mapping, file, start, len);
                }
@@ -1267,7 +1249,6 @@ asmlinkage ssize_t sys_readahead(int fd, loff_t offset, size_t count)
 }
 
 #ifdef CONFIG_MMU
-static int FASTCALL(page_cache_read(struct file * file, unsigned long offset));
 /**
  * page_cache_read - adds requested page to the page cache if not already there
  * @file:      file to read
@@ -1276,7 +1257,7 @@ static int FASTCALL(page_cache_read(struct file * file, unsigned long offset));
  * This adds the requested page to the page cache if it isn't already there,
  * and schedules an I/O to read in its contents from disk.
  */
-static int fastcall page_cache_read(struct file * file, unsigned long offset)
+static int fastcall page_cache_read(struct file * file, pgoff_t offset)
 {
        struct address_space *mapping = file->f_mapping;
        struct page *page; 
@@ -1303,69 +1284,69 @@ static int fastcall page_cache_read(struct file * file, unsigned long offset)
 #define MMAP_LOTSAMISS  (100)
 
 /**
- * filemap_nopage - read in file data for page fault handling
- * @area:      the applicable vm_area
- * @address:   target address to read in
- * @type:      returned with VM_FAULT_{MINOR,MAJOR} if not %NULL
+ * filemap_fault - read in file data for page fault handling
+ * @vma:       vma in which the fault was taken
+ * @vmf:       struct vm_fault containing details of the fault
  *
- * filemap_nopage() is invoked via the vma operations vector for a
+ * filemap_fault() is invoked via the vma operations vector for a
  * mapped memory region to read in file data during a page fault.
  *
  * The goto's are kind of ugly, but this streamlines the normal case of having
  * it in the page cache, and handles the special cases reasonably without
  * having a lot of duplicated code.
  */
-struct page *filemap_nopage(struct vm_area_struct *area,
-                               unsigned long address, int *type)
+int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
        int error;
-       struct file *file = area->vm_file;
+       struct file *file = vma->vm_file;
        struct address_space *mapping = file->f_mapping;
        struct file_ra_state *ra = &file->f_ra;
        struct inode *inode = mapping->host;
        struct page *page;
-       unsigned long size, pgoff;
-       int did_readaround = 0, majmin = VM_FAULT_MINOR;
-
-       pgoff = ((address-area->vm_start) >> PAGE_CACHE_SHIFT) + area->vm_pgoff;
+       unsigned long size;
+       int did_readaround = 0;
+       int ret = 0;
 
-retry_all:
        size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
-       if (pgoff >= size)
+       if (vmf->pgoff >= size)
                goto outside_data_content;
 
        /* If we don't want any read-ahead, don't bother */
-       if (VM_RandomReadHint(area))
+       if (VM_RandomReadHint(vma))
                goto no_cached_page;
 
        /*
-        * The readahead code wants to be told about each and every page
-        * so it can build and shrink its windows appropriately
-        *
-        * For sequential accesses, we use the generic readahead logic.
-        */
-       if (VM_SequentialReadHint(area))
-               page_cache_readahead(mapping, ra, file, pgoff, 1);
-
-       /*
         * Do we have something in the page cache already?
         */
 retry_find:
-       page = find_get_page(mapping, pgoff);
+       page = find_lock_page(mapping, vmf->pgoff);
+       /*
+        * For sequential accesses, we use the generic readahead logic.
+        */
+       if (VM_SequentialReadHint(vma)) {
+               if (!page) {
+                       page_cache_sync_readahead(mapping, ra, file,
+                                                          vmf->pgoff, 1);
+                       page = find_lock_page(mapping, vmf->pgoff);
+                       if (!page)
+                               goto no_cached_page;
+               }
+               if (PageReadahead(page)) {
+                       page_cache_async_readahead(mapping, ra, file, page,
+                                                          vmf->pgoff, 1);
+               }
+       }
+
        if (!page) {
                unsigned long ra_pages;
 
-               if (VM_SequentialReadHint(area)) {
-                       handle_ra_miss(mapping, ra, pgoff);
-                       goto no_cached_page;
-               }
                ra->mmap_miss++;
 
                /*
                 * Do we miss much more than hit in this file? If so,
                 * stop bothering with read-ahead. It will only hurt.
                 */
-               if (ra->mmap_miss > ra->mmap_hit + MMAP_LOTSAMISS)
+               if (ra->mmap_miss > MMAP_LOTSAMISS)
                        goto no_cached_page;
 
                /*
@@ -1373,7 +1354,7 @@ retry_find:
                 * check did_readaround, as this is an inner loop.
                 */
                if (!did_readaround) {
-                       majmin = VM_FAULT_MAJOR;
+                       ret = VM_FAULT_MAJOR;
                        count_vm_event(PGMAJFAULT);
                }
                did_readaround = 1;
@@ -1381,48 +1362,56 @@ retry_find:
                if (ra_pages) {
                        pgoff_t start = 0;
 
-                       if (pgoff > ra_pages / 2)
-                               start = pgoff - ra_pages / 2;
+                       if (vmf->pgoff > ra_pages / 2)
+                               start = vmf->pgoff - ra_pages / 2;
                        do_page_cache_readahead(mapping, file, start, ra_pages);
                }
-               page = find_get_page(mapping, pgoff);
+               page = find_lock_page(mapping, vmf->pgoff);
                if (!page)
                        goto no_cached_page;
        }
 
        if (!did_readaround)
-               ra->mmap_hit++;
+               ra->mmap_miss--;
 
        /*
-        * Ok, found a page in the page cache, now we need to check
-        * that it's up-to-date.
+        * We have a locked page in the page cache, now we need to check
+        * that it's up-to-date. If not, it is going to be due to an error.
         */
-       if (!PageUptodate(page))
+       if (unlikely(!PageUptodate(page)))
                goto page_not_uptodate;
 
-success:
+       /* Must recheck i_size under page lock */
+       size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
+       if (unlikely(vmf->pgoff >= size)) {
+               unlock_page(page);
+               page_cache_release(page);
+               goto outside_data_content;
+       }
+
        /*
         * Found the page and have a reference on it.
         */
        mark_page_accessed(page);
-       if (type)
-               *type = majmin;
-       return page;
+       ra->prev_pos = (loff_t)page->index << PAGE_CACHE_SHIFT;
+       vmf->page = page;
+       return ret | VM_FAULT_LOCKED;
 
 outside_data_content:
        /*
         * An external ptracer can access pages that normally aren't
         * accessible..
         */
-       if (area->vm_mm == current->mm)
-               return NOPAGE_SIGBUS;
+       if (vma->vm_mm == current->mm)
+               return VM_FAULT_SIGBUS;
+
        /* Fall through to the non-read-ahead case */
 no_cached_page:
        /*
         * We're only likely to ever get here if MADV_RANDOM is in
         * effect.
         */
-       error = page_cache_read(file, pgoff);
+       error = page_cache_read(file, vmf->pgoff);
 
        /*
         * The page we want has now been added to the page cache.
@@ -1438,38 +1427,15 @@ no_cached_page:
         * to schedule I/O.
         */
        if (error == -ENOMEM)
-               return NOPAGE_OOM;
-       return NOPAGE_SIGBUS;
+               return VM_FAULT_OOM;
+       return VM_FAULT_SIGBUS;
 
 page_not_uptodate:
+       /* IO error path */
        if (!did_readaround) {
-               majmin = VM_FAULT_MAJOR;
+               ret = VM_FAULT_MAJOR;
                count_vm_event(PGMAJFAULT);
        }
-       lock_page(page);
-
-       /* Did it get unhashed while we waited for it? */
-       if (!page->mapping) {
-               unlock_page(page);
-               page_cache_release(page);
-               goto retry_all;
-       }
-
-       /* Did somebody else get it up-to-date? */
-       if (PageUptodate(page)) {
-               unlock_page(page);
-               goto success;
-       }
-
-       error = mapping->a_ops->readpage(file, page);
-       if (!error) {
-               wait_on_page_locked(page);
-               if (PageUptodate(page))
-                       goto success;
-       } else if (error == AOP_TRUNCATED_PAGE) {
-               page_cache_release(page);
-               goto retry_find;
-       }
 
        /*
         * Umm, take care of errors if the page isn't up-to-date.
@@ -1477,217 +1443,21 @@ page_not_uptodate:
         * because there really aren't any performance issues here
         * and we need to check for errors.
         */
-       lock_page(page);
-
-       /* Somebody truncated the page on us? */
-       if (!page->mapping) {
-               unlock_page(page);
-               page_cache_release(page);
-               goto retry_all;
-       }
-
-       /* Somebody else successfully read it in? */
-       if (PageUptodate(page)) {
-               unlock_page(page);
-               goto success;
-       }
        ClearPageError(page);
        error = mapping->a_ops->readpage(file, page);
-       if (!error) {
-               wait_on_page_locked(page);
-               if (PageUptodate(page))
-                       goto success;
-       } else if (error == AOP_TRUNCATED_PAGE) {
-               page_cache_release(page);
-               goto retry_find;
-       }
-
-       /*
-        * Things didn't work out. Return zero to tell the
-        * mm layer so, possibly freeing the page cache page first.
-        */
-       shrink_readahead_size_eio(file, ra);
        page_cache_release(page);
-       return NOPAGE_SIGBUS;
-}
-EXPORT_SYMBOL(filemap_nopage);
-
-static struct page * filemap_getpage(struct file *file, unsigned long pgoff,
-                                       int nonblock)
-{
-       struct address_space *mapping = file->f_mapping;
-       struct page *page;
-       int error;
 
-       /*
-        * Do we have something in the page cache already?
-        */
-retry_find:
-       page = find_get_page(mapping, pgoff);
-       if (!page) {
-               if (nonblock)
-                       return NULL;
-               goto no_cached_page;
-       }
-
-       /*
-        * Ok, found a page in the page cache, now we need to check
-        * that it's up-to-date.
-        */
-       if (!PageUptodate(page)) {
-               if (nonblock) {
-                       page_cache_release(page);
-                       return NULL;
-               }
-               goto page_not_uptodate;
-       }
-
-success:
-       /*
-        * Found the page and have a reference on it.
-        */
-       mark_page_accessed(page);
-       return page;
-
-no_cached_page:
-       error = page_cache_read(file, pgoff);
-
-       /*
-        * The page we want has now been added to the page cache.
-        * In the unlikely event that someone removed it in the
-        * meantime, we'll just come back here and read it again.
-        */
-       if (error >= 0)
+       if (!error || error == AOP_TRUNCATED_PAGE)
                goto retry_find;
 
-       /*
-        * An error return from page_cache_read can result if the
-        * system is low on memory, or a problem occurs while trying
-        * to schedule I/O.
-        */
-       return NULL;
-
-page_not_uptodate:
-       lock_page(page);
-
-       /* Did it get truncated while we waited for it? */
-       if (!page->mapping) {
-               unlock_page(page);
-               goto err;
-       }
-
-       /* Did somebody else get it up-to-date? */
-       if (PageUptodate(page)) {
-               unlock_page(page);
-               goto success;
-       }
-
-       error = mapping->a_ops->readpage(file, page);
-       if (!error) {
-               wait_on_page_locked(page);
-               if (PageUptodate(page))
-                       goto success;
-       } else if (error == AOP_TRUNCATED_PAGE) {
-               page_cache_release(page);
-               goto retry_find;
-       }
-
-       /*
-        * Umm, take care of errors if the page isn't up-to-date.
-        * Try to re-read it _once_. We do this synchronously,
-        * because there really aren't any performance issues here
-        * and we need to check for errors.
-        */
-       lock_page(page);
-
-       /* Somebody truncated the page on us? */
-       if (!page->mapping) {
-               unlock_page(page);
-               goto err;
-       }
-       /* Somebody else successfully read it in? */
-       if (PageUptodate(page)) {
-               unlock_page(page);
-               goto success;
-       }
-
-       ClearPageError(page);
-       error = mapping->a_ops->readpage(file, page);
-       if (!error) {
-               wait_on_page_locked(page);
-               if (PageUptodate(page))
-                       goto success;
-       } else if (error == AOP_TRUNCATED_PAGE) {
-               page_cache_release(page);
-               goto retry_find;
-       }
-
-       /*
-        * Things didn't work out. Return zero to tell the
-        * mm layer so, possibly freeing the page cache page first.
-        */
-err:
-       page_cache_release(page);
-
-       return NULL;
-}
-
-int filemap_populate(struct vm_area_struct *vma, unsigned long addr,
-               unsigned long len, pgprot_t prot, unsigned long pgoff,
-               int nonblock)
-{
-       struct file *file = vma->vm_file;
-       struct address_space *mapping = file->f_mapping;
-       struct inode *inode = mapping->host;
-       unsigned long size;
-       struct mm_struct *mm = vma->vm_mm;
-       struct page *page;
-       int err;
-
-       if (!nonblock)
-               force_page_cache_readahead(mapping, vma->vm_file,
-                                       pgoff, len >> PAGE_CACHE_SHIFT);
-
-repeat:
-       size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
-       if (pgoff + (len >> PAGE_CACHE_SHIFT) > size)
-               return -EINVAL;
-
-       page = filemap_getpage(file, pgoff, nonblock);
-
-       /* XXX: This is wrong, a filesystem I/O error may have happened. Fix that as
-        * done in shmem_populate calling shmem_getpage */
-       if (!page && !nonblock)
-               return -ENOMEM;
-
-       if (page) {
-               err = install_page(mm, vma, addr, page, prot);
-               if (err) {
-                       page_cache_release(page);
-                       return err;
-               }
-       } else if (vma->vm_flags & VM_NONLINEAR) {
-               /* No page was found just because we can't read it in now (being
-                * here implies nonblock != 0), but the page may exist, so set
-                * the PTE to fault it in later. */
-               err = install_file_pte(mm, vma, addr, pgoff, prot);
-               if (err)
-                       return err;
-       }
-
-       len -= PAGE_SIZE;
-       addr += PAGE_SIZE;
-       pgoff++;
-       if (len)
-               goto repeat;
-
-       return 0;
+       /* Things didn't work out. Return zero to tell the mm layer so. */
+       shrink_readahead_size_eio(file, ra);
+       return VM_FAULT_SIGBUS;
 }
-EXPORT_SYMBOL(filemap_populate);
+EXPORT_SYMBOL(filemap_fault);
 
 struct vm_operations_struct generic_file_vm_ops = {
-       .nopage         = filemap_nopage,
-       .populate       = filemap_populate,
+       .fault          = filemap_fault,
 };
 
 /* This is used for a general mmap of a disk file */
@@ -1700,6 +1470,7 @@ int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
                return -ENOEXEC;
        file_accessed(file);
        vma->vm_ops = &generic_file_vm_ops;
+       vma->vm_flags |= VM_CAN_NONLINEAR;
        return 0;
 }
 
@@ -1726,8 +1497,8 @@ int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
 EXPORT_SYMBOL(generic_file_mmap);
 EXPORT_SYMBOL(generic_file_readonly_mmap);
 
-static inline struct page *__read_cache_page(struct address_space *mapping,
-                               unsigned long index,
+static struct page *__read_cache_page(struct address_space *mapping,
+                               pgoff_t index,
                                int (*filler)(void *,struct page*),
                                void *data)
 {
@@ -1763,18 +1534,12 @@ repeat:
        return page;
 }
 
-/**
- * read_cache_page - read into page cache, fill it if needed
- * @mapping:   the page's address_space
- * @index:     the page index
- * @filler:    function to perform the read
- * @data:      destination for read data
- *
- * Read into the page cache. If a page already exists,
- * and PageUptodate() is not set, try to fill the page.
+/*
+ * Same as read_cache_page, but don't wait for page to become unlocked
+ * after submitting it to the filler.
  */
-struct page *read_cache_page(struct address_space *mapping,
-                               unsigned long index,
+struct page *read_cache_page_async(struct address_space *mapping,
+                               pgoff_t index,
                                int (*filler)(void *,struct page*),
                                void *data)
 {
@@ -1784,8 +1549,7 @@ struct page *read_cache_page(struct address_space *mapping,
 retry:
        page = __read_cache_page(mapping, index, filler, data);
        if (IS_ERR(page))
-               goto out;
-       mark_page_accessed(page);
+               return page;
        if (PageUptodate(page))
                goto out;
 
@@ -1802,7 +1566,40 @@ retry:
        err = filler(data, page);
        if (err < 0) {
                page_cache_release(page);
-               page = ERR_PTR(err);
+               return ERR_PTR(err);
+       }
+out:
+       mark_page_accessed(page);
+       return page;
+}
+EXPORT_SYMBOL(read_cache_page_async);
+
+/**
+ * read_cache_page - read into page cache, fill it if needed
+ * @mapping:   the page's address_space
+ * @index:     the page index
+ * @filler:    function to perform the read
+ * @data:      destination for read data
+ *
+ * Read into the page cache. If a page already exists, and PageUptodate() is
+ * not set, try to fill the page then wait for it to become unlocked.
+ *
+ * If the page does not get brought uptodate, return -EIO.
+ */
+struct page *read_cache_page(struct address_space *mapping,
+                               pgoff_t index,
+                               int (*filler)(void *,struct page*),
+                               void *data)
+{
+       struct page *page;
+
+       page = read_cache_page_async(mapping, index, filler, data);
+       if (IS_ERR(page))
+               goto out;
+       wait_on_page_locked(page);
+       if (!PageUptodate(page)) {
+               page_cache_release(page);
+               page = ERR_PTR(-EIO);
        }
  out:
        return page;
@@ -1951,7 +1748,6 @@ inline int generic_write_checks(struct file *file, loff_t *pos, size_t *count, i
        if (unlikely(*pos + *count > MAX_NON_LFS &&
                                !(file->f_flags & O_LARGEFILE))) {
                if (*pos >= MAX_NON_LFS) {
-                       send_sig(SIGXFSZ, current, 0);
                        return -EFBIG;
                }
                if (*count > MAX_NON_LFS - (unsigned long)*pos) {
@@ -1969,7 +1765,6 @@ inline int generic_write_checks(struct file *file, loff_t *pos, size_t *count, i
        if (likely(!isblk)) {
                if (unlikely(*pos >= inode->i_sb->s_maxbytes)) {
                        if (*count || *pos > inode->i_sb->s_maxbytes) {
-                               send_sig(SIGXFSZ, current, 0);
                                return -EFBIG;
                        }
                        /* zero-length writes at ->s_maxbytes are OK */
@@ -2043,16 +1838,15 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
                size_t count, ssize_t written)
 {
        struct file *file = iocb->ki_filp;
-       struct address_space * mapping = file->f_mapping;
+       struct address_space *mapping = file->f_mapping;
        const struct address_space_operations *a_ops = mapping->a_ops;
        struct inode    *inode = mapping->host;
        long            status = 0;
        struct page     *page;
        struct page     *cached_page = NULL;
-       size_t          bytes;
        struct pagevec  lru_pvec;
        const struct iovec *cur_iov = iov; /* current iovec */
-       size_t          iov_base = 0;      /* offset in the current iovec */
+       size_t          iov_offset = 0;    /* offset in the current iovec */
        char __user     *buf;
 
        pagevec_init(&lru_pvec, 0);
@@ -2063,36 +1857,36 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
        if (likely(nr_segs == 1))
                buf = iov->iov_base + written;
        else {
-               filemap_set_next_iovec(&cur_iov, &iov_base, written);
-               buf = cur_iov->iov_base + iov_base;
+               filemap_set_next_iovec(&cur_iov, &iov_offset, written);
+               buf = cur_iov->iov_base + iov_offset;
        }
 
        do {
-               unsigned long index;
-               unsigned long offset;
-               size_t copied;
+               pgoff_t index;          /* Pagecache index for current page */
+               unsigned long offset;   /* Offset into pagecache page */
+               unsigned long maxlen;   /* Bytes remaining in current iovec */
+               size_t bytes;           /* Bytes to write to page */
+               size_t copied;          /* Bytes copied from user */
 
-               offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */
+               offset = (pos & (PAGE_CACHE_SIZE - 1));
                index = pos >> PAGE_CACHE_SHIFT;
                bytes = PAGE_CACHE_SIZE - offset;
+               if (bytes > count)
+                       bytes = count;
 
-               /* Limit the size of the copy to the caller's write size */
-               bytes = min(bytes, count);
-
-               /*
-                * Limit the size of the copy to that of the current segment,
-                * because fault_in_pages_readable() doesn't know how to walk
-                * segments.
-                */
-               bytes = min(bytes, cur_iov->iov_len - iov_base);
+               maxlen = cur_iov->iov_len - iov_offset;
+               if (maxlen > bytes)
+                       maxlen = bytes;
 
+#ifndef CONFIG_DEBUG_VM
                /*
                 * Bring in the user page that we will copy from _first_.
                 * Otherwise there's a nasty deadlock on copying from the
                 * same page as we're writing to, without it being marked
                 * up-to-date.
                 */
-               fault_in_pages_readable(buf, bytes);
+               fault_in_pages_readable(buf, maxlen);
+#endif
 
                page = __grab_cache_page(mapping,index,&cached_page,&lru_pvec);
                if (!page) {
@@ -2100,72 +1894,65 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
                        break;
                }
 
-               if (unlikely(bytes == 0)) {
-                       status = 0;
-                       copied = 0;
-                       goto zero_length_segment;
-               }
-
                status = a_ops->prepare_write(file, page, offset, offset+bytes);
-               if (unlikely(status)) {
-                       loff_t isize = i_size_read(inode);
+               if (unlikely(status))
+                       goto fs_write_aop_error;
 
-                       if (status != AOP_TRUNCATED_PAGE)
-                               unlock_page(page);
-                       page_cache_release(page);
-                       if (status == AOP_TRUNCATED_PAGE)
-                               continue;
-                       /*
-                        * prepare_write() may have instantiated a few blocks
-                        * outside i_size.  Trim these off again.
-                        */
-                       if (pos + bytes > isize)
-                               vmtruncate(inode, isize);
-                       break;
-               }
                if (likely(nr_segs == 1))
                        copied = filemap_copy_from_user(page, offset,
                                                        buf, bytes);
                else
                        copied = filemap_copy_from_user_iovec(page, offset,
-                                               cur_iov, iov_base, bytes);
+                                               cur_iov, iov_offset, bytes);
                flush_dcache_page(page);
                status = a_ops->commit_write(file, page, offset, offset+bytes);
-               if (status == AOP_TRUNCATED_PAGE) {
-                       page_cache_release(page);
-                       continue;
+               if (unlikely(status < 0 || status == AOP_TRUNCATED_PAGE))
+                       goto fs_write_aop_error;
+               if (unlikely(copied != bytes)) {
+                       status = -EFAULT;
+                       goto fs_write_aop_error;
                }
-zero_length_segment:
-               if (likely(copied >= 0)) {
-                       if (!status)
-                               status = copied;
-
-                       if (status >= 0) {
-                               written += status;
-                               count -= status;
-                               pos += status;
-                               buf += status;
-                               if (unlikely(nr_segs > 1)) {
-                                       filemap_set_next_iovec(&cur_iov,
-                                                       &iov_base, status);
-                                       if (count)
-                                               buf = cur_iov->iov_base +
-                                                       iov_base;
-                               } else {
-                                       iov_base += status;
-                               }
+               if (unlikely(status > 0)) /* filesystem did partial write */
+                       copied = status;
+
+               if (likely(copied > 0)) {
+                       written += copied;
+                       count -= copied;
+                       pos += copied;
+                       buf += copied;
+                       if (unlikely(nr_segs > 1)) {
+                               filemap_set_next_iovec(&cur_iov,
+                                               &iov_offset, copied);
+                               if (count)
+                                       buf = cur_iov->iov_base + iov_offset;
+                       } else {
+                               iov_offset += copied;
                        }
                }
-               if (unlikely(copied != bytes))
-                       if (status >= 0)
-                               status = -EFAULT;
                unlock_page(page);
                mark_page_accessed(page);
                page_cache_release(page);
-               if (status < 0)
-                       break;
                balance_dirty_pages_ratelimited(mapping);
                cond_resched();
+               continue;
+
+fs_write_aop_error:
+               if (status != AOP_TRUNCATED_PAGE)
+                       unlock_page(page);
+               page_cache_release(page);
+
+               /*
+                * prepare_write() may have instantiated a few blocks
+                * outside i_size.  Trim these off again. Don't need
+                * i_size_read because we hold i_mutex.
+                */
+               if (pos + bytes > inode->i_size)
+                       vmtruncate(inode, inode->i_size);
+               if (status == AOP_TRUNCATED_PAGE)
+                       continue;
+               else
+                       break;
+
        } while (count);
        *ppos = pos;
 
@@ -2205,30 +1992,14 @@ __generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
        size_t ocount;          /* original count */
        size_t count;           /* after file limit checks */
        struct inode    *inode = mapping->host;
-       unsigned long   seg;
        loff_t          pos;
        ssize_t         written;
        ssize_t         err;
 
        ocount = 0;
-       for (seg = 0; seg < nr_segs; seg++) {
-               const struct iovec *iv = &iov[seg];
-
-               /*
-                * If any segment has a negative length, or the cumulative
-                * length ever wraps negative then return -EINVAL.
-                */
-               ocount += iv->iov_len;
-               if (unlikely((ssize_t)(ocount|iv->iov_len) < 0))
-                       return -EINVAL;
-               if (access_ok(VERIFY_READ, iv->iov_base, iv->iov_len))
-                       continue;
-               if (seg == 0)
-                       return -EFAULT;
-               nr_segs = seg;
-               ocount -= iv->iov_len;  /* This segment is no good */
-               break;
-       }
+       err = generic_segment_checks(iov, &nr_segs, &ocount, VERIFY_READ);
+       if (err)
+               return err;
 
        count = ocount;
        pos = *ppos;
@@ -2288,10 +2059,10 @@ __generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
                 * semantics.
                 */
                endbyte = pos + written_buffered - written - 1;
-               err = do_sync_file_range(file, pos, endbyte,
-                                        SYNC_FILE_RANGE_WAIT_BEFORE|
-                                        SYNC_FILE_RANGE_WRITE|
-                                        SYNC_FILE_RANGE_WAIT_AFTER);
+               err = do_sync_mapping_range(file->f_mapping, pos, endbyte,
+                                           SYNC_FILE_RANGE_WAIT_BEFORE|
+                                           SYNC_FILE_RANGE_WRITE|
+                                           SYNC_FILE_RANGE_WAIT_AFTER);
                if (err == 0) {
                        written = written_buffered;
                        invalidate_mapping_pages(mapping,
@@ -2373,7 +2144,8 @@ generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
        struct file *file = iocb->ki_filp;
        struct address_space *mapping = file->f_mapping;
        ssize_t retval;
-       size_t write_len = 0;
+       size_t write_len;
+       pgoff_t end = 0; /* silence gcc */
 
        /*
         * If it's a write, unmap all mmappings of the file up-front.  This
@@ -2382,23 +2154,46 @@ generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
         */
        if (rw == WRITE) {
                write_len = iov_length(iov, nr_segs);
+               end = (offset + write_len - 1) >> PAGE_CACHE_SHIFT;
                if (mapping_mapped(mapping))
                        unmap_mapping_range(mapping, offset, write_len, 0);
        }
 
        retval = filemap_write_and_wait(mapping);
-       if (retval == 0) {
-               retval = mapping->a_ops->direct_IO(rw, iocb, iov,
-                                               offset, nr_segs);
-               if (rw == WRITE && mapping->nrpages) {
-                       pgoff_t end = (offset + write_len - 1)
-                                               >> PAGE_CACHE_SHIFT;
-                       int err = invalidate_inode_pages2_range(mapping,
+       if (retval)
+               goto out;
+
+       /*
+        * After a write we want buffered reads to be sure to go to disk to get
+        * the new data.  We invalidate clean cached page from the region we're
+        * about to write.  We do this *before* the write so that we can return
+        * -EIO without clobbering -EIOCBQUEUED from ->direct_IO().
+        */
+       if (rw == WRITE && mapping->nrpages) {
+               retval = invalidate_inode_pages2_range(mapping,
                                        offset >> PAGE_CACHE_SHIFT, end);
-                       if (err)
-                               retval = err;
-               }
+               if (retval)
+                       goto out;
+       }
+
+       retval = mapping->a_ops->direct_IO(rw, iocb, iov, offset, nr_segs);
+       if (retval)
+               goto out;
+
+       /*
+        * Finally, try again to invalidate clean pages which might have been
+        * faulted in by get_user_pages() if the source of the write was an
+        * mmap()ed region of the file we're writing.  That's a pretty crazy
+        * thing to do, so we don't support it 100%.  If this invalidation
+        * fails and we have -EIOCBQUEUED we ignore the failure.
+        */
+       if (rw == WRITE && mapping->nrpages) {
+               int err = invalidate_inode_pages2_range(mapping,
+                                             offset >> PAGE_CACHE_SHIFT, end);
+               if (err && retval >= 0)
+                       retval = err;
        }
+out:
        return retval;
 }