Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs-2.6

[linux-2.6.git] / mm / shmem.c

diff --git a/mm/shmem.c b/mm/shmem.c
index 1529d8fdc23f003ab7c2a0b06209950ff64ff015..dfaa0f4e9789f5e8f395ac1dee7975c1abffdc1d 100644 (file)
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -28,6 +28,7 @@
 #include <linux/file.h>
 #include <linux/mm.h>
 #include <linux/module.h>
+#include <linux/percpu_counter.h>
 #include <linux/swap.h>
 
 static struct vfsmount *shm_mnt;
@@ -233,10 +234,10 @@ static void shmem_free_blocks(struct inode *inode, long pages)
 {
        struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
        if (sbinfo->max_blocks) {
-               spin_lock(&sbinfo->stat_lock);
-               sbinfo->free_blocks += pages;
+               percpu_counter_add(&sbinfo->used_blocks, -pages);
+               spin_lock(&inode->i_lock);
                inode->i_blocks -= pages*BLOCKS_PER_PAGE;
-               spin_unlock(&sbinfo->stat_lock);
+               spin_unlock(&inode->i_lock);
        }
 }
 
@@ -416,19 +417,17 @@ static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long
                if (sgp == SGP_READ)
                        return shmem_swp_map(ZERO_PAGE(0));
                /*
-                * Test free_blocks against 1 not 0, since we have 1 data
+                * Test used_blocks against 1 less max_blocks, since we have 1 data
                 * page (and perhaps indirect index pages) yet to allocate:
                 * a waste to allocate index if we cannot allocate data.
                 */
                if (sbinfo->max_blocks) {
-                       spin_lock(&sbinfo->stat_lock);
-                       if (sbinfo->free_blocks <= 1) {
-                               spin_unlock(&sbinfo->stat_lock);
+                       if (percpu_counter_compare(&sbinfo->used_blocks, (sbinfo->max_blocks - 1)) > 0)
                                return ERR_PTR(-ENOSPC);
-                       }
-                       sbinfo->free_blocks--;
+                       percpu_counter_inc(&sbinfo->used_blocks);
+                       spin_lock(&inode->i_lock);
                        inode->i_blocks += BLOCKS_PER_PAGE;
-                       spin_unlock(&sbinfo->stat_lock);
+                       spin_unlock(&inode->i_lock);
                }
 
                spin_unlock(&info->lock);
@@ -1224,6 +1223,7 @@ static int shmem_getpage(struct inode *inode, unsigned long idx,
        struct shmem_sb_info *sbinfo;
        struct page *filepage = *pagep;
        struct page *swappage;
+       struct page *prealloc_page = NULL;
        swp_entry_t *entry;
        swp_entry_t swap;
        gfp_t gfp;
@@ -1248,7 +1248,6 @@ repeat:
                filepage = find_lock_page(mapping, idx);
        if (filepage && PageUptodate(filepage))
                goto done;
-       error = 0;
        gfp = mapping_gfp_mask(mapping);
        if (!filepage) {
                /*
@@ -1259,7 +1258,19 @@ repeat:
                if (error)
                        goto failed;
                radix_tree_preload_end();
+               if (sgp != SGP_READ && !prealloc_page) {
+                       /* We don't care if this fails */
+                       prealloc_page = shmem_alloc_page(gfp, info, idx);
+                       if (prealloc_page) {
+                               if (mem_cgroup_cache_charge(prealloc_page,
+                                               current->mm, GFP_KERNEL)) {
+                                       page_cache_release(prealloc_page);
+                                       prealloc_page = NULL;
+                               }
+                       }
+               }
        }
+       error = 0;
 
        spin_lock(&info->lock);
        shmem_recalc_inode(inode);
@@ -1388,17 +1399,16 @@ repeat:
                shmem_swp_unmap(entry);
                sbinfo = SHMEM_SB(inode->i_sb);
                if (sbinfo->max_blocks) {
-                       spin_lock(&sbinfo->stat_lock);
-                       if (sbinfo->free_blocks == 0 ||
+                       if ((percpu_counter_compare(&sbinfo->used_blocks, sbinfo->max_blocks) > 0) ||
                            shmem_acct_block(info->flags)) {
-                               spin_unlock(&sbinfo->stat_lock);
                                spin_unlock(&info->lock);
                                error = -ENOSPC;
                                goto failed;
                        }
-                       sbinfo->free_blocks--;
+                       percpu_counter_inc(&sbinfo->used_blocks);
+                       spin_lock(&inode->i_lock);
                        inode->i_blocks += BLOCKS_PER_PAGE;
-                       spin_unlock(&sbinfo->stat_lock);
+                       spin_unlock(&inode->i_lock);
                } else if (shmem_acct_block(info->flags)) {
                        spin_unlock(&info->lock);
                        error = -ENOSPC;
@@ -1408,28 +1418,38 @@ repeat:
                if (!filepage) {
                        int ret;
 
-                       spin_unlock(&info->lock);
-                       filepage = shmem_alloc_page(gfp, info, idx);
-                       if (!filepage) {
-                               shmem_unacct_blocks(info->flags, 1);
-                               shmem_free_blocks(inode, 1);
-                               error = -ENOMEM;
-                               goto failed;
-                       }
-                       SetPageSwapBacked(filepage);
+                       if (!prealloc_page) {
+                               spin_unlock(&info->lock);
+                               filepage = shmem_alloc_page(gfp, info, idx);
+                               if (!filepage) {
+                                       shmem_unacct_blocks(info->flags, 1);
+                                       shmem_free_blocks(inode, 1);
+                                       error = -ENOMEM;
+                                       goto failed;
+                               }
+                               SetPageSwapBacked(filepage);
 
-                       /* Precharge page while we can wait, compensate after */
-                       error = mem_cgroup_cache_charge(filepage, current->mm,
-                                       GFP_KERNEL);
-                       if (error) {
-                               page_cache_release(filepage);
-                               shmem_unacct_blocks(info->flags, 1);
-                               shmem_free_blocks(inode, 1);
-                               filepage = NULL;
-                               goto failed;
+                               /*
+                                * Precharge page while we can wait, compensate
+                                * after
+                                */
+                               error = mem_cgroup_cache_charge(filepage,
+                                       current->mm, GFP_KERNEL);
+                               if (error) {
+                                       page_cache_release(filepage);
+                                       shmem_unacct_blocks(info->flags, 1);
+                                       shmem_free_blocks(inode, 1);
+                                       filepage = NULL;
+                                       goto failed;
+                               }
+
+                               spin_lock(&info->lock);
+                       } else {
+                               filepage = prealloc_page;
+                               prealloc_page = NULL;
+                               SetPageSwapBacked(filepage);
                        }
 
-                       spin_lock(&info->lock);
                        entry = shmem_swp_alloc(info, idx, sgp);
                        if (IS_ERR(entry))
                                error = PTR_ERR(entry);
@@ -1470,13 +1490,19 @@ repeat:
        }
 done:
        *pagep = filepage;
-       return 0;
+       error = 0;
+       goto out;
 
 failed:
        if (*pagep != filepage) {
                unlock_page(filepage);
                page_cache_release(filepage);
        }
+out:
+       if (prealloc_page) {
+               mem_cgroup_uncharge_cache_page(prealloc_page);
+               page_cache_release(prealloc_page);
+       }
        return error;
 }
 
@@ -1792,17 +1818,16 @@ static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
        buf->f_type = TMPFS_MAGIC;
        buf->f_bsize = PAGE_CACHE_SIZE;
        buf->f_namelen = NAME_MAX;
-       spin_lock(&sbinfo->stat_lock);
        if (sbinfo->max_blocks) {
                buf->f_blocks = sbinfo->max_blocks;
-               buf->f_bavail = buf->f_bfree = sbinfo->free_blocks;
+               buf->f_bavail = buf->f_bfree =
+                               sbinfo->max_blocks - percpu_counter_sum(&sbinfo->used_blocks);
        }
        if (sbinfo->max_inodes) {
                buf->f_files = sbinfo->max_inodes;
                buf->f_ffree = sbinfo->free_inodes;
        }
        /* else leave those fields 0 like simple_statfs */
-       spin_unlock(&sbinfo->stat_lock);
        return 0;
 }
 
@@ -2243,7 +2268,6 @@ static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
 {
        struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
        struct shmem_sb_info config = *sbinfo;
-       unsigned long blocks;
        unsigned long inodes;
        int error = -EINVAL;
 
@@ -2251,9 +2275,8 @@ static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
                return error;
 
        spin_lock(&sbinfo->stat_lock);
-       blocks = sbinfo->max_blocks - sbinfo->free_blocks;
        inodes = sbinfo->max_inodes - sbinfo->free_inodes;
-       if (config.max_blocks < blocks)
+       if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0)
                goto out;
        if (config.max_inodes < inodes)
                goto out;
@@ -2270,7 +2293,6 @@ static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
 
        error = 0;
        sbinfo->max_blocks  = config.max_blocks;
-       sbinfo->free_blocks = config.max_blocks - blocks;
        sbinfo->max_inodes  = config.max_inodes;
        sbinfo->free_inodes = config.max_inodes - inodes;
 
@@ -2345,7 +2367,7 @@ int shmem_fill_super(struct super_block *sb, void *data, int silent)
 #endif
 
        spin_lock_init(&sbinfo->stat_lock);
-       sbinfo->free_blocks = sbinfo->max_blocks;
+       percpu_counter_init(&sbinfo->used_blocks, 0);
        sbinfo->free_inodes = sbinfo->max_inodes;
 
        sb->s_maxbytes = SHMEM_MAX_BYTES;