const: mark struct vm_struct_operations
[linux-2.6.git] / mm / shmem.c
index 7519988..356dd99 100644 (file)
 #include <linux/init.h>
 #include <linux/vfs.h>
 #include <linux/mount.h>
+#include <linux/pagemap.h>
 #include <linux/file.h>
 #include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/swap.h>
+#include <linux/ima.h>
 
 static struct vfsmount *shm_mnt;
 
@@ -42,13 +44,11 @@ static struct vfsmount *shm_mnt;
 #include <linux/exportfs.h>
 #include <linux/generic_acl.h>
 #include <linux/mman.h>
-#include <linux/pagemap.h>
 #include <linux/string.h>
 #include <linux/slab.h>
 #include <linux/backing-dev.h>
 #include <linux/shmem_fs.h>
 #include <linux/writeback.h>
-#include <linux/vfs.h>
 #include <linux/blkdev.h>
 #include <linux/security.h>
 #include <linux/swapops.h>
@@ -59,19 +59,33 @@ static struct vfsmount *shm_mnt;
 #include <linux/highmem.h>
 #include <linux/seq_file.h>
 #include <linux/magic.h>
-#include <linux/ima.h>
 
 #include <asm/uaccess.h>
 #include <asm/div64.h>
 #include <asm/pgtable.h>
 
+/*
+ * The maximum size of a shmem/tmpfs file is limited by the maximum size of
+ * its triple-indirect swap vector - see illustration at shmem_swp_entry().
+ *
+ * With 4kB page size, maximum file size is just over 2TB on a 32-bit kernel,
+ * but one eighth of that on a 64-bit kernel.  With 8kB page size, maximum
+ * file size is just over 4TB on a 64-bit kernel, but 16TB on a 32-bit kernel,
+ * MAX_LFS_FILESIZE being then more restrictive than swap vector layout.
+ *
+ * We use / and * instead of shifts in the definitions below, so that the swap
+ * vector can be tested with small even values (e.g. 20) for ENTRIES_PER_PAGE.
+ */
 #define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long))
-#define ENTRIES_PER_PAGEPAGE (ENTRIES_PER_PAGE*ENTRIES_PER_PAGE)
-#define BLOCKS_PER_PAGE  (PAGE_CACHE_SIZE/512)
+#define ENTRIES_PER_PAGEPAGE ((unsigned long long)ENTRIES_PER_PAGE*ENTRIES_PER_PAGE)
 
-#define SHMEM_MAX_INDEX  (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
-#define SHMEM_MAX_BYTES  ((unsigned long long)SHMEM_MAX_INDEX << PAGE_CACHE_SHIFT)
+#define SHMSWP_MAX_INDEX (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
+#define SHMSWP_MAX_BYTES (SHMSWP_MAX_INDEX << PAGE_CACHE_SHIFT)
 
+#define SHMEM_MAX_BYTES  min_t(unsigned long long, SHMSWP_MAX_BYTES, MAX_LFS_FILESIZE)
+#define SHMEM_MAX_INDEX  ((unsigned long)((SHMEM_MAX_BYTES+1) >> PAGE_CACHE_SHIFT))
+
+#define BLOCKS_PER_PAGE  (PAGE_CACHE_SIZE/512)
 #define VM_ACCT(size)    (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
 
 /* info->flags needs VM_flags to handle pagein/truncate races efficiently */
@@ -170,13 +184,13 @@ static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
  */
 static inline int shmem_acct_size(unsigned long flags, loff_t size)
 {
-       return (flags & VM_ACCOUNT) ?
-               security_vm_enough_memory_kern(VM_ACCT(size)) : 0;
+       return (flags & VM_NORESERVE) ?
+               0 : security_vm_enough_memory_kern(VM_ACCT(size));
 }
 
 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
 {
-       if (flags & VM_ACCOUNT)
+       if (!(flags & VM_NORESERVE))
                vm_unacct_memory(VM_ACCT(size));
 }
 
@@ -188,13 +202,13 @@ static inline void shmem_unacct_size(unsigned long flags, loff_t size)
  */
 static inline int shmem_acct_block(unsigned long flags)
 {
-       return (flags & VM_ACCOUNT) ?
-               0 : security_vm_enough_memory_kern(VM_ACCT(PAGE_CACHE_SIZE));
+       return (flags & VM_NORESERVE) ?
+               security_vm_enough_memory_kern(VM_ACCT(PAGE_CACHE_SIZE)) : 0;
 }
 
 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
 {
-       if (!(flags & VM_ACCOUNT))
+       if (flags & VM_NORESERVE)
                vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
 }
 
@@ -204,7 +218,7 @@ static const struct file_operations shmem_file_operations;
 static const struct inode_operations shmem_inode_operations;
 static const struct inode_operations shmem_dir_inode_operations;
 static const struct inode_operations shmem_special_inode_operations;
-static struct vm_operations_struct shmem_vm_ops;
+static const struct vm_operations_struct shmem_vm_ops;
 
 static struct backing_dev_info shmem_backing_dev_info  __read_mostly = {
        .ra_pages       = 0,    /* No readahead */
@@ -1032,8 +1046,9 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc)
         * sync from ever calling shmem_writepage; but a stacking filesystem
         * may use the ->writepage of its underlying filesystem, in which case
         * tmpfs should write out to swap only in response to memory pressure,
-        * and not for pdflush or sync.  However, in those cases, we do still
-        * want to check if there's a redundant swappage to be discarded.
+        * and not for the writeback threads or sync.  However, in those cases,
+        * we do still want to check if there's a redundant swappage to be
+        * discarded.
         */
        if (wbc->for_reclaim)
                swap = get_swap_page();
@@ -1068,8 +1083,7 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc)
                swap_duplicate(swap);
                BUG_ON(page_mapped(page));
                page_cache_release(page);       /* pagecache ref */
-               set_page_dirty(page);
-               unlock_page(page);
+               swap_writepage(page, wbc);
                if (inode) {
                        mutex_lock(&shmem_swaplist_mutex);
                        /* move instead of add in case we're racing */
@@ -1083,7 +1097,11 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc)
        shmem_swp_unmap(entry);
 unlock:
        spin_unlock(&info->lock);
-       swap_free(swap);
+       /*
+        * add_to_swap_cache() doesn't return -EEXIST, so we can safely
+        * clear SWAP_HAS_CACHE flag.
+        */
+       swapcache_free(swap, NULL);
 redirty:
        set_page_dirty(page);
        if (wbc->for_reclaim)
@@ -1326,8 +1344,12 @@ repeat:
                        shmem_swp_unmap(entry);
                        spin_unlock(&info->lock);
                        if (error == -ENOMEM) {
-                               /* allow reclaim from this memory cgroup */
-                               error = mem_cgroup_shrink_usage(swappage,
+                               /*
+                                * reclaim from proper memory cgroup and
+                                * call memcg's OOM if needed.
+                                */
+                               error = mem_cgroup_shmem_charge_fallback(
+                                                               swappage,
                                                                current->mm,
                                                                gfp);
                                if (error) {
@@ -1516,8 +1538,8 @@ static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
        return 0;
 }
 
-static struct inode *
-shmem_get_inode(struct super_block *sb, int mode, dev_t dev)
+static struct inode *shmem_get_inode(struct super_block *sb, int mode,
+                                       dev_t dev, unsigned long flags)
 {
        struct inode *inode;
        struct shmem_inode_info *info;
@@ -1538,7 +1560,9 @@ shmem_get_inode(struct super_block *sb, int mode, dev_t dev)
                info = SHMEM_I(inode);
                memset(info, 0, (char *)inode - (char *)info);
                spin_lock_init(&info->lock);
+               info->flags = flags & VM_NORESERVE;
                INIT_LIST_HEAD(&info->swaplist);
+               cache_no_acl(inode);
 
                switch (mode & S_IFMT) {
                default:
@@ -1610,8 +1634,8 @@ shmem_write_end(struct file *file, struct address_space *mapping,
        if (pos + copied > inode->i_size)
                i_size_write(inode, pos + copied);
 
-       unlock_page(page);
        set_page_dirty(page);
+       unlock_page(page);
        page_cache_release(page);
 
        return copied;
@@ -1780,9 +1804,10 @@ static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
 static int
 shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev)
 {
-       struct inode *inode = shmem_get_inode(dir->i_sb, mode, dev);
+       struct inode *inode;
        int error = -ENOSPC;
 
+       inode = shmem_get_inode(dir->i_sb, mode, dev, VM_NORESERVE);
        if (inode) {
                error = security_inode_init_security(inode, dir, NULL, NULL,
                                                     NULL);
@@ -1921,7 +1946,7 @@ static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *s
        if (len > PAGE_CACHE_SIZE)
                return -ENAMETOOLONG;
 
-       inode = shmem_get_inode(dir->i_sb, S_IFLNK|S_IRWXUGO, 0);
+       inode = shmem_get_inode(dir->i_sb, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE);
        if (!inode)
                return -ENOSPC;
 
@@ -1947,13 +1972,13 @@ static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *s
                        iput(inode);
                        return error;
                }
-               unlock_page(page);
                inode->i_mapping->a_ops = &shmem_aops;
                inode->i_op = &shmem_symlink_inode_operations;
                kaddr = kmap_atomic(page, KM_USER0);
                memcpy(kaddr, symname, len);
                kunmap_atomic(kaddr, KM_USER0);
                set_page_dirty(page);
+               unlock_page(page);
                page_cache_release(page);
        }
        if (dir->i_mode & S_ISGID)
@@ -2277,8 +2302,7 @@ static void shmem_put_super(struct super_block *sb)
        sb->s_fs_info = NULL;
 }
 
-static int shmem_fill_super(struct super_block *sb,
-                           void *data, int silent)
+int shmem_fill_super(struct super_block *sb, void *data, int silent)
 {
        struct inode *inode;
        struct dentry *root;
@@ -2286,17 +2310,14 @@ static int shmem_fill_super(struct super_block *sb,
        int err = -ENOMEM;
 
        /* Round up to L1_CACHE_BYTES to resist false sharing */
-       sbinfo = kmalloc(max((int)sizeof(struct shmem_sb_info),
+       sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
                                L1_CACHE_BYTES), GFP_KERNEL);
        if (!sbinfo)
                return -ENOMEM;
 
-       sbinfo->max_blocks = 0;
-       sbinfo->max_inodes = 0;
        sbinfo->mode = S_IRWXUGO | S_ISVTX;
        sbinfo->uid = current_fsuid();
        sbinfo->gid = current_fsgid();
-       sbinfo->mpol = NULL;
        sb->s_fs_info = sbinfo;
 
 #ifdef CONFIG_TMPFS
@@ -2333,7 +2354,7 @@ static int shmem_fill_super(struct super_block *sb,
        sb->s_flags |= MS_POSIXACL;
 #endif
 
-       inode = shmem_get_inode(sb, S_IFDIR | sbinfo->mode, 0);
+       inode = shmem_get_inode(sb, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
        if (!inode)
                goto failed;
        inode->i_uid = sbinfo->uid;
@@ -2368,7 +2389,6 @@ static void shmem_destroy_inode(struct inode *inode)
                /* only struct inode is valid if it's an inline symlink */
                mpol_free_shared_policy(&SHMEM_I(inode)->policy);
        }
-       shmem_acl_destroy_inode(inode);
        kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
 }
 
@@ -2377,10 +2397,6 @@ static void init_once(void *foo)
        struct shmem_inode_info *p = (struct shmem_inode_info *) foo;
 
        inode_init_once(&p->vfs_inode);
-#ifdef CONFIG_TMPFS_POSIX_ACL
-       p->i_acl = NULL;
-       p->i_default_acl = NULL;
-#endif
 }
 
 static int init_inodecache(void)
@@ -2405,6 +2421,7 @@ static const struct address_space_operations shmem_aops = {
        .write_end      = shmem_write_end,
 #endif
        .migratepage    = migrate_page,
+       .error_remove_page = generic_error_remove_page,
 };
 
 static const struct file_operations shmem_file_operations = {
@@ -2430,7 +2447,7 @@ static const struct inode_operations shmem_inode_operations = {
        .getxattr       = generic_getxattr,
        .listxattr      = generic_listxattr,
        .removexattr    = generic_removexattr,
-       .permission     = shmem_permission,
+       .check_acl      = shmem_check_acl,
 #endif
 
 };
@@ -2453,7 +2470,7 @@ static const struct inode_operations shmem_dir_inode_operations = {
        .getxattr       = generic_getxattr,
        .listxattr      = generic_listxattr,
        .removexattr    = generic_removexattr,
-       .permission     = shmem_permission,
+       .check_acl      = shmem_check_acl,
 #endif
 };
 
@@ -2464,7 +2481,7 @@ static const struct inode_operations shmem_special_inode_operations = {
        .getxattr       = generic_getxattr,
        .listxattr      = generic_listxattr,
        .removexattr    = generic_removexattr,
-       .permission     = shmem_permission,
+       .check_acl      = shmem_check_acl,
 #endif
 };
 
@@ -2481,7 +2498,7 @@ static const struct super_operations shmem_ops = {
        .put_super      = shmem_put_super,
 };
 
-static struct vm_operations_struct shmem_vm_ops = {
+static const struct vm_operations_struct shmem_vm_ops = {
        .fault          = shmem_fault,
 #ifdef CONFIG_NUMA
        .set_policy     = shmem_set_policy,
@@ -2503,7 +2520,7 @@ static struct file_system_type tmpfs_fs_type = {
        .kill_sb        = kill_litter_super,
 };
 
-static int __init init_tmpfs(void)
+int __init init_tmpfs(void)
 {
        int error;
 
@@ -2560,7 +2577,7 @@ static struct file_system_type tmpfs_fs_type = {
        .kill_sb        = kill_litter_super,
 };
 
-static int __init init_tmpfs(void)
+int __init init_tmpfs(void)
 {
        BUG_ON(register_filesystem(&tmpfs_fs_type) != 0);
 
@@ -2575,12 +2592,17 @@ int shmem_unuse(swp_entry_t entry, struct page *page)
        return 0;
 }
 
-#define shmem_file_operations ramfs_file_operations
-#define shmem_vm_ops generic_file_vm_ops
-#define shmem_get_inode ramfs_get_inode
-#define shmem_acct_size(a, b) 0
-#define shmem_unacct_size(a, b) do {} while (0)
-#define SHMEM_MAX_BYTES LLONG_MAX
+int shmem_lock(struct file *file, int lock, struct user_struct *user)
+{
+       return 0;
+}
+
+#define shmem_vm_ops                           generic_file_vm_ops
+#define shmem_file_operations                  ramfs_file_operations
+#define shmem_get_inode(sb, mode, dev, flags)  ramfs_get_inode(sb, mode, dev)
+#define shmem_acct_size(flags, size)           0
+#define shmem_unacct_size(flags, size)         do {} while (0)
+#define SHMEM_MAX_BYTES                                MAX_LFS_FILESIZE
 
 #endif /* CONFIG_SHMEM */
 
@@ -2590,9 +2612,9 @@ int shmem_unuse(swp_entry_t entry, struct page *page)
  * shmem_file_setup - get an unlinked file living in tmpfs
  * @name: name for dentry (to be seen in /proc/<pid>/maps
  * @size: size to be set for the file
- * @flags: vm_flags
+ * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
  */
-struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags)
+struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
 {
        int error;
        struct file *file;
@@ -2624,13 +2646,10 @@ struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags)
                goto put_dentry;
 
        error = -ENOSPC;
-       inode = shmem_get_inode(root->d_sb, S_IFREG | S_IRWXUGO, 0);
+       inode = shmem_get_inode(root->d_sb, S_IFREG | S_IRWXUGO, 0, flags);
        if (!inode)
                goto close_file;
 
-#ifdef CONFIG_SHMEM
-       SHMEM_I(inode)->flags = (flags & VM_NORESERVE) ? 0 : VM_ACCOUNT;
-#endif
        d_instantiate(dentry, inode);
        inode->i_size = size;
        inode->i_nlink = 0;     /* It is unlinked */
@@ -2642,6 +2661,7 @@ struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags)
        if (error)
                goto close_file;
 #endif
+       ima_counts_get(file);
        return file;
 
 close_file:
@@ -2667,12 +2687,9 @@ int shmem_zero_setup(struct vm_area_struct *vma)
        if (IS_ERR(file))
                return PTR_ERR(file);
 
-       ima_shm_check(file);
        if (vma->vm_file)
                fput(vma->vm_file);
        vma->vm_file = file;
        vma->vm_ops = &shmem_vm_ops;
        return 0;
 }
-
-module_init(init_tmpfs)