tmpfs: copy truncate_inode_pages_range
[linux-2.6.git] / mm / truncate.c
index 7c617b5..232eb27 100644 (file)
@@ -19,6 +19,7 @@
 #include <linux/task_io_accounting_ops.h>
 #include <linux/buffer_head.h> /* grr. try_to_release_page,
                                   do_invalidatepage */
+#include <linux/cleancache.h>
 #include "internal.h"
 
 
@@ -51,6 +52,7 @@ void do_invalidatepage(struct page *page, unsigned long offset)
 static inline void truncate_partial_page(struct page *page, unsigned partial)
 {
        zero_user_segment(page, partial, PAGE_CACHE_SIZE);
+       cleancache_flush_page(page->mapping, page);
        if (page_has_private(page))
                do_invalidatepage(page, partial);
 }
@@ -197,9 +199,6 @@ int invalidate_inode_page(struct page *page)
  * The first pass will remove most pages, so the search cost of the second pass
  * is low.
  *
- * When looking at page->index outside the page lock we need to be careful to
- * copy it into a local to avoid races (it could change at any time).
- *
  * We pass down the cache-hot hint to the page freeing code.  Even if the
  * mapping is large, it is probably the case that the final pages are the most
  * recently touched, and freeing happens in ascending file offset order.
@@ -208,12 +207,13 @@ void truncate_inode_pages_range(struct address_space *mapping,
                                loff_t lstart, loff_t lend)
 {
        const pgoff_t start = (lstart + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
-       pgoff_t end;
        const unsigned partial = lstart & (PAGE_CACHE_SIZE - 1);
        struct pagevec pvec;
-       pgoff_t next;
+       pgoff_t index;
+       pgoff_t end;
        int i;
 
+       cleancache_flush_inode(mapping);
        if (mapping->nrpages == 0)
                return;
 
@@ -221,24 +221,21 @@ void truncate_inode_pages_range(struct address_space *mapping,
        end = (lend >> PAGE_CACHE_SHIFT);
 
        pagevec_init(&pvec, 0);
-       next = start;
-       while (next <= end &&
-              pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
+       index = start;
+       while (index <= end && pagevec_lookup(&pvec, mapping, index,
+                       min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
                mem_cgroup_uncharge_start();
                for (i = 0; i < pagevec_count(&pvec); i++) {
                        struct page *page = pvec.pages[i];
-                       pgoff_t page_index = page->index;
 
-                       if (page_index > end) {
-                               next = page_index;
+                       /* We rely upon deletion not changing page->index */
+                       index = page->index;
+                       if (index > end)
                                break;
-                       }
 
-                       if (page_index > next)
-                               next = page_index;
-                       next++;
                        if (!trylock_page(page))
                                continue;
+                       WARN_ON(page->index != index);
                        if (PageWriteback(page)) {
                                unlock_page(page);
                                continue;
@@ -249,6 +246,7 @@ void truncate_inode_pages_range(struct address_space *mapping,
                pagevec_release(&pvec);
                mem_cgroup_uncharge_end();
                cond_resched();
+               index++;
        }
 
        if (partial) {
@@ -261,16 +259,17 @@ void truncate_inode_pages_range(struct address_space *mapping,
                }
        }
 
-       next = start;
+       index = start;
        for ( ; ; ) {
                cond_resched();
-               if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
-                       if (next == start)
+               if (!pagevec_lookup(&pvec, mapping, index,
+                       min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
+                       if (index == start)
                                break;
-                       next = start;
+                       index = start;
                        continue;
                }
-               if (pvec.pages[0]->index > end) {
+               if (index == start && pvec.pages[0]->index > end) {
                        pagevec_release(&pvec);
                        break;
                }
@@ -278,19 +277,22 @@ void truncate_inode_pages_range(struct address_space *mapping,
                for (i = 0; i < pagevec_count(&pvec); i++) {
                        struct page *page = pvec.pages[i];
 
-                       if (page->index > end)
+                       /* We rely upon deletion not changing page->index */
+                       index = page->index;
+                       if (index > end)
                                break;
+
                        lock_page(page);
+                       WARN_ON(page->index != index);
                        wait_on_page_writeback(page);
                        truncate_inode_page(mapping, page);
-                       if (page->index > next)
-                               next = page->index;
-                       next++;
                        unlock_page(page);
                }
                pagevec_release(&pvec);
                mem_cgroup_uncharge_end();
+               index++;
        }
+       cleancache_flush_inode(mapping);
 }
 EXPORT_SYMBOL(truncate_inode_pages_range);
 
@@ -300,6 +302,11 @@ EXPORT_SYMBOL(truncate_inode_pages_range);
  * @lstart: offset from which to truncate
  *
  * Called under (and serialised by) inode->i_mutex.
+ *
+ * Note: When this function returns, there can be a page in the process of
+ * deletion (inside __delete_from_page_cache()) in the specified range.  Thus
+ * mapping->nrpages can be non-zero when this function returns even after
+ * truncation of the whole mapping.
  */
 void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
 {
@@ -321,48 +328,45 @@ EXPORT_SYMBOL(truncate_inode_pages);
  * pagetables.
  */
 unsigned long invalidate_mapping_pages(struct address_space *mapping,
-                                      pgoff_t start, pgoff_t end)
+               pgoff_t start, pgoff_t end)
 {
        struct pagevec pvec;
-       pgoff_t next = start;
-       unsigned long ret = 0;
+       pgoff_t index = start;
+       unsigned long ret;
+       unsigned long count = 0;
        int i;
 
        pagevec_init(&pvec, 0);
-       while (next <= end &&
-                       pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
+       while (index <= end && pagevec_lookup(&pvec, mapping, index,
+                       min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
                mem_cgroup_uncharge_start();
                for (i = 0; i < pagevec_count(&pvec); i++) {
                        struct page *page = pvec.pages[i];
-                       pgoff_t index;
-                       int lock_failed;
 
-                       lock_failed = !trylock_page(page);
-
-                       /*
-                        * We really shouldn't be looking at the ->index of an
-                        * unlocked page.  But we're not allowed to lock these
-                        * pages.  So we rely upon nobody altering the ->index
-                        * of this (pinned-by-us) page.
-                        */
+                       /* We rely upon deletion not changing page->index */
                        index = page->index;
-                       if (index > next)
-                               next = index;
-                       next++;
-                       if (lock_failed)
-                               continue;
-
-                       ret += invalidate_inode_page(page);
+                       if (index > end)
+                               break;
 
+                       if (!trylock_page(page))
+                               continue;
+                       WARN_ON(page->index != index);
+                       ret = invalidate_inode_page(page);
                        unlock_page(page);
-                       if (next > end)
-                               break;
+                       /*
+                        * Invalidation is a hint that the page is no longer
+                        * of interest and try to speed up its reclaim.
+                        */
+                       if (!ret)
+                               deactivate_page(page);
+                       count += ret;
                }
                pagevec_release(&pvec);
                mem_cgroup_uncharge_end();
                cond_resched();
+               index++;
        }
-       return ret;
+       return count;
 }
 EXPORT_SYMBOL(invalidate_mapping_pages);
 
@@ -388,7 +392,7 @@ invalidate_complete_page2(struct address_space *mapping, struct page *page)
 
        clear_page_mlock(page);
        BUG_ON(page_has_private(page));
-       __remove_from_page_cache(page);
+       __delete_from_page_cache(page);
        spin_unlock_irq(&mapping->tree_lock);
        mem_cgroup_uncharge_cache_page(page);
 
@@ -426,36 +430,32 @@ int invalidate_inode_pages2_range(struct address_space *mapping,
                                  pgoff_t start, pgoff_t end)
 {
        struct pagevec pvec;
-       pgoff_t next;
+       pgoff_t index;
        int i;
        int ret = 0;
        int ret2 = 0;
        int did_range_unmap = 0;
-       int wrapped = 0;
 
+       cleancache_flush_inode(mapping);
        pagevec_init(&pvec, 0);
-       next = start;
-       while (next <= end && !wrapped &&
-               pagevec_lookup(&pvec, mapping, next,
-                       min(end - next, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
+       index = start;
+       while (index <= end && pagevec_lookup(&pvec, mapping, index,
+                       min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
                mem_cgroup_uncharge_start();
                for (i = 0; i < pagevec_count(&pvec); i++) {
                        struct page *page = pvec.pages[i];
-                       pgoff_t page_index;
+
+                       /* We rely upon deletion not changing page->index */
+                       index = page->index;
+                       if (index > end)
+                               break;
 
                        lock_page(page);
+                       WARN_ON(page->index != index);
                        if (page->mapping != mapping) {
                                unlock_page(page);
                                continue;
                        }
-                       page_index = page->index;
-                       next = page_index + 1;
-                       if (next == 0)
-                               wrapped = 1;
-                       if (page_index > end) {
-                               unlock_page(page);
-                               break;
-                       }
                        wait_on_page_writeback(page);
                        if (page_mapped(page)) {
                                if (!did_range_unmap) {
@@ -463,9 +463,9 @@ int invalidate_inode_pages2_range(struct address_space *mapping,
                                         * Zap the rest of the file in one hit.
                                         */
                                        unmap_mapping_range(mapping,
-                                          (loff_t)page_index<<PAGE_CACHE_SHIFT,
-                                          (loff_t)(end - page_index + 1)
-                                                       << PAGE_CACHE_SHIFT,
+                                          (loff_t)index << PAGE_CACHE_SHIFT,
+                                          (loff_t)(1 + end - index)
+                                                        << PAGE_CACHE_SHIFT,
                                            0);
                                        did_range_unmap = 1;
                                } else {
@@ -473,8 +473,8 @@ int invalidate_inode_pages2_range(struct address_space *mapping,
                                         * Just zap this page
                                         */
                                        unmap_mapping_range(mapping,
-                                         (loff_t)page_index<<PAGE_CACHE_SHIFT,
-                                         PAGE_CACHE_SIZE, 0);
+                                          (loff_t)index << PAGE_CACHE_SHIFT,
+                                          PAGE_CACHE_SIZE, 0);
                                }
                        }
                        BUG_ON(page_mapped(page));
@@ -490,7 +490,9 @@ int invalidate_inode_pages2_range(struct address_space *mapping,
                pagevec_release(&pvec);
                mem_cgroup_uncharge_end();
                cond_resched();
+               index++;
        }
+       cleancache_flush_inode(mapping);
        return ret;
 }
 EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);
@@ -513,8 +515,8 @@ EXPORT_SYMBOL_GPL(invalidate_inode_pages2);
 /**
  * truncate_pagecache - unmap and remove pagecache that has been truncated
  * @inode: inode
- * @old: old file offset
- * @new: new file offset
+ * @oldsize: old file size
+ * @newsize: new file size
  *
  * inode's new i_size must already be written before truncate_pagecache
  * is called.
@@ -526,9 +528,10 @@ EXPORT_SYMBOL_GPL(invalidate_inode_pages2);
  * situations such as writepage being called for a page that has already
  * had its underlying blocks deallocated.
  */
-void truncate_pagecache(struct inode *inode, loff_t old, loff_t new)
+void truncate_pagecache(struct inode *inode, loff_t oldsize, loff_t newsize)
 {
        struct address_space *mapping = inode->i_mapping;
+       loff_t holebegin = round_up(newsize, PAGE_SIZE);
 
        /*
         * unmap_mapping_range is called twice, first simply for
@@ -539,9 +542,9 @@ void truncate_pagecache(struct inode *inode, loff_t old, loff_t new)
         * truncate_inode_pages finishes, hence the second
         * unmap_mapping_range call must be made for correctness.
         */
-       unmap_mapping_range(mapping, new + PAGE_SIZE - 1, 0, 1);
-       truncate_inode_pages(mapping, new);
-       unmap_mapping_range(mapping, new + PAGE_SIZE - 1, 0, 1);
+       unmap_mapping_range(mapping, holebegin, 0, 1);
+       truncate_inode_pages(mapping, newsize);
+       unmap_mapping_range(mapping, holebegin, 0, 1);
 }
 EXPORT_SYMBOL(truncate_pagecache);
 
@@ -571,22 +574,47 @@ EXPORT_SYMBOL(truncate_setsize);
 /**
  * vmtruncate - unmap mappings "freed" by truncate() syscall
  * @inode: inode of the file used
- * @offset: file offset to start truncating
+ * @newsize: file offset to start truncating
  *
  * This function is deprecated and truncate_setsize or truncate_pagecache
  * should be used instead, together with filesystem specific block truncation.
  */
-int vmtruncate(struct inode *inode, loff_t offset)
+int vmtruncate(struct inode *inode, loff_t newsize)
 {
        int error;
 
-       error = inode_newsize_ok(inode, offset);
+       error = inode_newsize_ok(inode, newsize);
        if (error)
                return error;
 
-       truncate_setsize(inode, offset);
+       truncate_setsize(inode, newsize);
        if (inode->i_op->truncate)
                inode->i_op->truncate(inode);
        return 0;
 }
 EXPORT_SYMBOL(vmtruncate);
+
+int vmtruncate_range(struct inode *inode, loff_t lstart, loff_t lend)
+{
+       struct address_space *mapping = inode->i_mapping;
+       loff_t holebegin = round_up(lstart, PAGE_SIZE);
+       loff_t holelen = 1 + lend - holebegin;
+
+       /*
+        * If the underlying filesystem is not going to provide
+        * a way to truncate a range of blocks (punch a hole) -
+        * we should return failure right now.
+        */
+       if (!inode->i_op->truncate_range)
+               return -ENOSYS;
+
+       mutex_lock(&inode->i_mutex);
+       inode_dio_wait(inode);
+       unmap_mapping_range(mapping, holebegin, holelen, 1);
+       inode->i_op->truncate_range(inode, lstart, lend);
+       /* unmap again to remove racily COWed private pages */
+       unmap_mapping_range(mapping, holebegin, holelen, 1);
+       mutex_unlock(&inode->i_mutex);
+
+       return 0;
+}