memcg: handle swap caches
[linux-2.6.git] / mm / shmem.c
1 /*
2  * Resizable virtual memory filesystem for Linux.
3  *
4  * Copyright (C) 2000 Linus Torvalds.
5  *               2000 Transmeta Corp.
6  *               2000-2001 Christoph Rohland
7  *               2000-2001 SAP AG
8  *               2002 Red Hat Inc.
9  * Copyright (C) 2002-2005 Hugh Dickins.
10  * Copyright (C) 2002-2005 VERITAS Software Corporation.
11  * Copyright (C) 2004 Andi Kleen, SuSE Labs
12  *
13  * Extended attribute support for tmpfs:
14  * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
15  * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
16  *
17  * tiny-shmem:
18  * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
19  *
20  * This file is released under the GPL.
21  */
22
23 #include <linux/fs.h>
24 #include <linux/init.h>
25 #include <linux/vfs.h>
26 #include <linux/mount.h>
27 #include <linux/file.h>
28 #include <linux/mm.h>
29 #include <linux/module.h>
30 #include <linux/swap.h>
31
32 static struct vfsmount *shm_mnt;
33
34 #ifdef CONFIG_SHMEM
35 /*
36  * This virtual memory filesystem is heavily based on the ramfs. It
37  * extends ramfs by the ability to use swap and honor resource limits
38  * which makes it a completely usable filesystem.
39  */
40
41 #include <linux/xattr.h>
42 #include <linux/exportfs.h>
43 #include <linux/generic_acl.h>
44 #include <linux/mman.h>
45 #include <linux/pagemap.h>
46 #include <linux/string.h>
47 #include <linux/slab.h>
48 #include <linux/backing-dev.h>
49 #include <linux/shmem_fs.h>
50 #include <linux/writeback.h>
51 #include <linux/vfs.h>
52 #include <linux/blkdev.h>
53 #include <linux/security.h>
54 #include <linux/swapops.h>
55 #include <linux/mempolicy.h>
56 #include <linux/namei.h>
57 #include <linux/ctype.h>
58 #include <linux/migrate.h>
59 #include <linux/highmem.h>
60 #include <linux/seq_file.h>
61 #include <linux/magic.h>
62
63 #include <asm/uaccess.h>
64 #include <asm/div64.h>
65 #include <asm/pgtable.h>
66
67 #define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long))
68 #define ENTRIES_PER_PAGEPAGE (ENTRIES_PER_PAGE*ENTRIES_PER_PAGE)
69 #define BLOCKS_PER_PAGE  (PAGE_CACHE_SIZE/512)
70
71 #define SHMEM_MAX_INDEX  (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
72 #define SHMEM_MAX_BYTES  ((unsigned long long)SHMEM_MAX_INDEX << PAGE_CACHE_SHIFT)
73
74 #define VM_ACCT(size)    (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
75
76 /* info->flags needs VM_flags to handle pagein/truncate races efficiently */
77 #define SHMEM_PAGEIN     VM_READ
78 #define SHMEM_TRUNCATE   VM_WRITE
79
80 /* Definition to limit shmem_truncate's steps between cond_rescheds */
81 #define LATENCY_LIMIT    64
82
83 /* Pretend that each entry is of this size in directory's i_size */
84 #define BOGO_DIRENT_SIZE 20
85
86 /* Flag allocation requirements to shmem_getpage and shmem_swp_alloc */
87 enum sgp_type {
88         SGP_READ,       /* don't exceed i_size, don't allocate page */
89         SGP_CACHE,      /* don't exceed i_size, may allocate page */
90         SGP_DIRTY,      /* like SGP_CACHE, but set new page dirty */
91         SGP_WRITE,      /* may exceed i_size, may allocate page */
92 };
93
94 #ifdef CONFIG_TMPFS
95 static unsigned long shmem_default_max_blocks(void)
96 {
97         return totalram_pages / 2;
98 }
99
100 static unsigned long shmem_default_max_inodes(void)
101 {
102         return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
103 }
104 #endif
105
106 static int shmem_getpage(struct inode *inode, unsigned long idx,
107                          struct page **pagep, enum sgp_type sgp, int *type);
108
109 static inline struct page *shmem_dir_alloc(gfp_t gfp_mask)
110 {
111         /*
112          * The above definition of ENTRIES_PER_PAGE, and the use of
113          * BLOCKS_PER_PAGE on indirect pages, assume PAGE_CACHE_SIZE:
114          * might be reconsidered if it ever diverges from PAGE_SIZE.
115          *
116          * Mobility flags are masked out as swap vectors cannot move
117          */
118         return alloc_pages((gfp_mask & ~GFP_MOVABLE_MASK) | __GFP_ZERO,
119                                 PAGE_CACHE_SHIFT-PAGE_SHIFT);
120 }
121
122 static inline void shmem_dir_free(struct page *page)
123 {
124         __free_pages(page, PAGE_CACHE_SHIFT-PAGE_SHIFT);
125 }
126
127 static struct page **shmem_dir_map(struct page *page)
128 {
129         return (struct page **)kmap_atomic(page, KM_USER0);
130 }
131
132 static inline void shmem_dir_unmap(struct page **dir)
133 {
134         kunmap_atomic(dir, KM_USER0);
135 }
136
137 static swp_entry_t *shmem_swp_map(struct page *page)
138 {
139         return (swp_entry_t *)kmap_atomic(page, KM_USER1);
140 }
141
142 static inline void shmem_swp_balance_unmap(void)
143 {
144         /*
145          * When passing a pointer to an i_direct entry, to code which
146          * also handles indirect entries and so will shmem_swp_unmap,
147          * we must arrange for the preempt count to remain in balance.
148          * What kmap_atomic of a lowmem page does depends on config
149          * and architecture, so pretend to kmap_atomic some lowmem page.
150          */
151         (void) kmap_atomic(ZERO_PAGE(0), KM_USER1);
152 }
153
154 static inline void shmem_swp_unmap(swp_entry_t *entry)
155 {
156         kunmap_atomic(entry, KM_USER1);
157 }
158
159 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
160 {
161         return sb->s_fs_info;
162 }
163
164 /*
165  * shmem_file_setup pre-accounts the whole fixed size of a VM object,
166  * for shared memory and for shared anonymous (/dev/zero) mappings
167  * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
168  * consistent with the pre-accounting of private mappings ...
169  */
170 static inline int shmem_acct_size(unsigned long flags, loff_t size)
171 {
172         return (flags & VM_ACCOUNT) ?
173                 security_vm_enough_memory_kern(VM_ACCT(size)) : 0;
174 }
175
176 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
177 {
178         if (flags & VM_ACCOUNT)
179                 vm_unacct_memory(VM_ACCT(size));
180 }
181
182 /*
183  * ... whereas tmpfs objects are accounted incrementally as
184  * pages are allocated, in order to allow huge sparse files.
185  * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
186  * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
187  */
188 static inline int shmem_acct_block(unsigned long flags)
189 {
190         return (flags & VM_ACCOUNT) ?
191                 0 : security_vm_enough_memory_kern(VM_ACCT(PAGE_CACHE_SIZE));
192 }
193
194 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
195 {
196         if (!(flags & VM_ACCOUNT))
197                 vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
198 }
199
200 static const struct super_operations shmem_ops;
201 static const struct address_space_operations shmem_aops;
202 static const struct file_operations shmem_file_operations;
203 static const struct inode_operations shmem_inode_operations;
204 static const struct inode_operations shmem_dir_inode_operations;
205 static const struct inode_operations shmem_special_inode_operations;
206 static struct vm_operations_struct shmem_vm_ops;
207
208 static struct backing_dev_info shmem_backing_dev_info  __read_mostly = {
209         .ra_pages       = 0,    /* No readahead */
210         .capabilities   = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED,
211         .unplug_io_fn   = default_unplug_io_fn,
212 };
213
214 static LIST_HEAD(shmem_swaplist);
215 static DEFINE_MUTEX(shmem_swaplist_mutex);
216
217 static void shmem_free_blocks(struct inode *inode, long pages)
218 {
219         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
220         if (sbinfo->max_blocks) {
221                 spin_lock(&sbinfo->stat_lock);
222                 sbinfo->free_blocks += pages;
223                 inode->i_blocks -= pages*BLOCKS_PER_PAGE;
224                 spin_unlock(&sbinfo->stat_lock);
225         }
226 }
227
228 static int shmem_reserve_inode(struct super_block *sb)
229 {
230         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
231         if (sbinfo->max_inodes) {
232                 spin_lock(&sbinfo->stat_lock);
233                 if (!sbinfo->free_inodes) {
234                         spin_unlock(&sbinfo->stat_lock);
235                         return -ENOSPC;
236                 }
237                 sbinfo->free_inodes--;
238                 spin_unlock(&sbinfo->stat_lock);
239         }
240         return 0;
241 }
242
243 static void shmem_free_inode(struct super_block *sb)
244 {
245         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
246         if (sbinfo->max_inodes) {
247                 spin_lock(&sbinfo->stat_lock);
248                 sbinfo->free_inodes++;
249                 spin_unlock(&sbinfo->stat_lock);
250         }
251 }
252
253 /**
254  * shmem_recalc_inode - recalculate the size of an inode
255  * @inode: inode to recalc
256  *
257  * We have to calculate the free blocks since the mm can drop
258  * undirtied hole pages behind our back.
259  *
260  * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
261  * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
262  *
263  * It has to be called with the spinlock held.
264  */
265 static void shmem_recalc_inode(struct inode *inode)
266 {
267         struct shmem_inode_info *info = SHMEM_I(inode);
268         long freed;
269
270         freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
271         if (freed > 0) {
272                 info->alloced -= freed;
273                 shmem_unacct_blocks(info->flags, freed);
274                 shmem_free_blocks(inode, freed);
275         }
276 }
277
278 /**
279  * shmem_swp_entry - find the swap vector position in the info structure
280  * @info:  info structure for the inode
281  * @index: index of the page to find
282  * @page:  optional page to add to the structure. Has to be preset to
283  *         all zeros
284  *
285  * If there is no space allocated yet it will return NULL when
286  * page is NULL, else it will use the page for the needed block,
287  * setting it to NULL on return to indicate that it has been used.
288  *
289  * The swap vector is organized the following way:
290  *
291  * There are SHMEM_NR_DIRECT entries directly stored in the
292  * shmem_inode_info structure. So small files do not need an addional
293  * allocation.
294  *
295  * For pages with index > SHMEM_NR_DIRECT there is the pointer
296  * i_indirect which points to a page which holds in the first half
297  * doubly indirect blocks, in the second half triple indirect blocks:
298  *
299  * For an artificial ENTRIES_PER_PAGE = 4 this would lead to the
300  * following layout (for SHMEM_NR_DIRECT == 16):
301  *
302  * i_indirect -> dir --> 16-19
303  *            |      +-> 20-23
304  *            |
305  *            +-->dir2 --> 24-27
306  *            |        +-> 28-31
307  *            |        +-> 32-35
308  *            |        +-> 36-39
309  *            |
310  *            +-->dir3 --> 40-43
311  *                     +-> 44-47
312  *                     +-> 48-51
313  *                     +-> 52-55
314  */
315 static swp_entry_t *shmem_swp_entry(struct shmem_inode_info *info, unsigned long index, struct page **page)
316 {
317         unsigned long offset;
318         struct page **dir;
319         struct page *subdir;
320
321         if (index < SHMEM_NR_DIRECT) {
322                 shmem_swp_balance_unmap();
323                 return info->i_direct+index;
324         }
325         if (!info->i_indirect) {
326                 if (page) {
327                         info->i_indirect = *page;
328                         *page = NULL;
329                 }
330                 return NULL;                    /* need another page */
331         }
332
333         index -= SHMEM_NR_DIRECT;
334         offset = index % ENTRIES_PER_PAGE;
335         index /= ENTRIES_PER_PAGE;
336         dir = shmem_dir_map(info->i_indirect);
337
338         if (index >= ENTRIES_PER_PAGE/2) {
339                 index -= ENTRIES_PER_PAGE/2;
340                 dir += ENTRIES_PER_PAGE/2 + index/ENTRIES_PER_PAGE;
341                 index %= ENTRIES_PER_PAGE;
342                 subdir = *dir;
343                 if (!subdir) {
344                         if (page) {
345                                 *dir = *page;
346                                 *page = NULL;
347                         }
348                         shmem_dir_unmap(dir);
349                         return NULL;            /* need another page */
350                 }
351                 shmem_dir_unmap(dir);
352                 dir = shmem_dir_map(subdir);
353         }
354
355         dir += index;
356         subdir = *dir;
357         if (!subdir) {
358                 if (!page || !(subdir = *page)) {
359                         shmem_dir_unmap(dir);
360                         return NULL;            /* need a page */
361                 }
362                 *dir = subdir;
363                 *page = NULL;
364         }
365         shmem_dir_unmap(dir);
366         return shmem_swp_map(subdir) + offset;
367 }
368
369 static void shmem_swp_set(struct shmem_inode_info *info, swp_entry_t *entry, unsigned long value)
370 {
371         long incdec = value? 1: -1;
372
373         entry->val = value;
374         info->swapped += incdec;
375         if ((unsigned long)(entry - info->i_direct) >= SHMEM_NR_DIRECT) {
376                 struct page *page = kmap_atomic_to_page(entry);
377                 set_page_private(page, page_private(page) + incdec);
378         }
379 }
380
381 /**
382  * shmem_swp_alloc - get the position of the swap entry for the page.
383  * @info:       info structure for the inode
384  * @index:      index of the page to find
385  * @sgp:        check and recheck i_size? skip allocation?
386  *
387  * If the entry does not exist, allocate it.
388  */
389 static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long index, enum sgp_type sgp)
390 {
391         struct inode *inode = &info->vfs_inode;
392         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
393         struct page *page = NULL;
394         swp_entry_t *entry;
395
396         if (sgp != SGP_WRITE &&
397             ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode))
398                 return ERR_PTR(-EINVAL);
399
400         while (!(entry = shmem_swp_entry(info, index, &page))) {
401                 if (sgp == SGP_READ)
402                         return shmem_swp_map(ZERO_PAGE(0));
403                 /*
404                  * Test free_blocks against 1 not 0, since we have 1 data
405                  * page (and perhaps indirect index pages) yet to allocate:
406                  * a waste to allocate index if we cannot allocate data.
407                  */
408                 if (sbinfo->max_blocks) {
409                         spin_lock(&sbinfo->stat_lock);
410                         if (sbinfo->free_blocks <= 1) {
411                                 spin_unlock(&sbinfo->stat_lock);
412                                 return ERR_PTR(-ENOSPC);
413                         }
414                         sbinfo->free_blocks--;
415                         inode->i_blocks += BLOCKS_PER_PAGE;
416                         spin_unlock(&sbinfo->stat_lock);
417                 }
418
419                 spin_unlock(&info->lock);
420                 page = shmem_dir_alloc(mapping_gfp_mask(inode->i_mapping));
421                 if (page)
422                         set_page_private(page, 0);
423                 spin_lock(&info->lock);
424
425                 if (!page) {
426                         shmem_free_blocks(inode, 1);
427                         return ERR_PTR(-ENOMEM);
428                 }
429                 if (sgp != SGP_WRITE &&
430                     ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
431                         entry = ERR_PTR(-EINVAL);
432                         break;
433                 }
434                 if (info->next_index <= index)
435                         info->next_index = index + 1;
436         }
437         if (page) {
438                 /* another task gave its page, or truncated the file */
439                 shmem_free_blocks(inode, 1);
440                 shmem_dir_free(page);
441         }
442         if (info->next_index <= index && !IS_ERR(entry))
443                 info->next_index = index + 1;
444         return entry;
445 }
446
447 /**
448  * shmem_free_swp - free some swap entries in a directory
449  * @dir:        pointer to the directory
450  * @edir:       pointer after last entry of the directory
451  * @punch_lock: pointer to spinlock when needed for the holepunch case
452  */
453 static int shmem_free_swp(swp_entry_t *dir, swp_entry_t *edir,
454                                                 spinlock_t *punch_lock)
455 {
456         spinlock_t *punch_unlock = NULL;
457         swp_entry_t *ptr;
458         int freed = 0;
459
460         for (ptr = dir; ptr < edir; ptr++) {
461                 if (ptr->val) {
462                         if (unlikely(punch_lock)) {
463                                 punch_unlock = punch_lock;
464                                 punch_lock = NULL;
465                                 spin_lock(punch_unlock);
466                                 if (!ptr->val)
467                                         continue;
468                         }
469                         free_swap_and_cache(*ptr);
470                         *ptr = (swp_entry_t){0};
471                         freed++;
472                 }
473         }
474         if (punch_unlock)
475                 spin_unlock(punch_unlock);
476         return freed;
477 }
478
479 static int shmem_map_and_free_swp(struct page *subdir, int offset,
480                 int limit, struct page ***dir, spinlock_t *punch_lock)
481 {
482         swp_entry_t *ptr;
483         int freed = 0;
484
485         ptr = shmem_swp_map(subdir);
486         for (; offset < limit; offset += LATENCY_LIMIT) {
487                 int size = limit - offset;
488                 if (size > LATENCY_LIMIT)
489                         size = LATENCY_LIMIT;
490                 freed += shmem_free_swp(ptr+offset, ptr+offset+size,
491                                                         punch_lock);
492                 if (need_resched()) {
493                         shmem_swp_unmap(ptr);
494                         if (*dir) {
495                                 shmem_dir_unmap(*dir);
496                                 *dir = NULL;
497                         }
498                         cond_resched();
499                         ptr = shmem_swp_map(subdir);
500                 }
501         }
502         shmem_swp_unmap(ptr);
503         return freed;
504 }
505
506 static void shmem_free_pages(struct list_head *next)
507 {
508         struct page *page;
509         int freed = 0;
510
511         do {
512                 page = container_of(next, struct page, lru);
513                 next = next->next;
514                 shmem_dir_free(page);
515                 freed++;
516                 if (freed >= LATENCY_LIMIT) {
517                         cond_resched();
518                         freed = 0;
519                 }
520         } while (next);
521 }
522
523 static void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end)
524 {
525         struct shmem_inode_info *info = SHMEM_I(inode);
526         unsigned long idx;
527         unsigned long size;
528         unsigned long limit;
529         unsigned long stage;
530         unsigned long diroff;
531         struct page **dir;
532         struct page *topdir;
533         struct page *middir;
534         struct page *subdir;
535         swp_entry_t *ptr;
536         LIST_HEAD(pages_to_free);
537         long nr_pages_to_free = 0;
538         long nr_swaps_freed = 0;
539         int offset;
540         int freed;
541         int punch_hole;
542         spinlock_t *needs_lock;
543         spinlock_t *punch_lock;
544         unsigned long upper_limit;
545
546         inode->i_ctime = inode->i_mtime = CURRENT_TIME;
547         idx = (start + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
548         if (idx >= info->next_index)
549                 return;
550
551         spin_lock(&info->lock);
552         info->flags |= SHMEM_TRUNCATE;
553         if (likely(end == (loff_t) -1)) {
554                 limit = info->next_index;
555                 upper_limit = SHMEM_MAX_INDEX;
556                 info->next_index = idx;
557                 needs_lock = NULL;
558                 punch_hole = 0;
559         } else {
560                 if (end + 1 >= inode->i_size) { /* we may free a little more */
561                         limit = (inode->i_size + PAGE_CACHE_SIZE - 1) >>
562                                                         PAGE_CACHE_SHIFT;
563                         upper_limit = SHMEM_MAX_INDEX;
564                 } else {
565                         limit = (end + 1) >> PAGE_CACHE_SHIFT;
566                         upper_limit = limit;
567                 }
568                 needs_lock = &info->lock;
569                 punch_hole = 1;
570         }
571
572         topdir = info->i_indirect;
573         if (topdir && idx <= SHMEM_NR_DIRECT && !punch_hole) {
574                 info->i_indirect = NULL;
575                 nr_pages_to_free++;
576                 list_add(&topdir->lru, &pages_to_free);
577         }
578         spin_unlock(&info->lock);
579
580         if (info->swapped && idx < SHMEM_NR_DIRECT) {
581                 ptr = info->i_direct;
582                 size = limit;
583                 if (size > SHMEM_NR_DIRECT)
584                         size = SHMEM_NR_DIRECT;
585                 nr_swaps_freed = shmem_free_swp(ptr+idx, ptr+size, needs_lock);
586         }
587
588         /*
589          * If there are no indirect blocks or we are punching a hole
590          * below indirect blocks, nothing to be done.
591          */
592         if (!topdir || limit <= SHMEM_NR_DIRECT)
593                 goto done2;
594
595         /*
596          * The truncation case has already dropped info->lock, and we're safe
597          * because i_size and next_index have already been lowered, preventing
598          * access beyond.  But in the punch_hole case, we still need to take
599          * the lock when updating the swap directory, because there might be
600          * racing accesses by shmem_getpage(SGP_CACHE), shmem_unuse_inode or
601          * shmem_writepage.  However, whenever we find we can remove a whole
602          * directory page (not at the misaligned start or end of the range),
603          * we first NULLify its pointer in the level above, and then have no
604          * need to take the lock when updating its contents: needs_lock and
605          * punch_lock (either pointing to info->lock or NULL) manage this.
606          */
607
608         upper_limit -= SHMEM_NR_DIRECT;
609         limit -= SHMEM_NR_DIRECT;
610         idx = (idx > SHMEM_NR_DIRECT)? (idx - SHMEM_NR_DIRECT): 0;
611         offset = idx % ENTRIES_PER_PAGE;
612         idx -= offset;
613
614         dir = shmem_dir_map(topdir);
615         stage = ENTRIES_PER_PAGEPAGE/2;
616         if (idx < ENTRIES_PER_PAGEPAGE/2) {
617                 middir = topdir;
618                 diroff = idx/ENTRIES_PER_PAGE;
619         } else {
620                 dir += ENTRIES_PER_PAGE/2;
621                 dir += (idx - ENTRIES_PER_PAGEPAGE/2)/ENTRIES_PER_PAGEPAGE;
622                 while (stage <= idx)
623                         stage += ENTRIES_PER_PAGEPAGE;
624                 middir = *dir;
625                 if (*dir) {
626                         diroff = ((idx - ENTRIES_PER_PAGEPAGE/2) %
627                                 ENTRIES_PER_PAGEPAGE) / ENTRIES_PER_PAGE;
628                         if (!diroff && !offset && upper_limit >= stage) {
629                                 if (needs_lock) {
630                                         spin_lock(needs_lock);
631                                         *dir = NULL;
632                                         spin_unlock(needs_lock);
633                                         needs_lock = NULL;
634                                 } else
635                                         *dir = NULL;
636                                 nr_pages_to_free++;
637                                 list_add(&middir->lru, &pages_to_free);
638                         }
639                         shmem_dir_unmap(dir);
640                         dir = shmem_dir_map(middir);
641                 } else {
642                         diroff = 0;
643                         offset = 0;
644                         idx = stage;
645                 }
646         }
647
648         for (; idx < limit; idx += ENTRIES_PER_PAGE, diroff++) {
649                 if (unlikely(idx == stage)) {
650                         shmem_dir_unmap(dir);
651                         dir = shmem_dir_map(topdir) +
652                             ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
653                         while (!*dir) {
654                                 dir++;
655                                 idx += ENTRIES_PER_PAGEPAGE;
656                                 if (idx >= limit)
657                                         goto done1;
658                         }
659                         stage = idx + ENTRIES_PER_PAGEPAGE;
660                         middir = *dir;
661                         if (punch_hole)
662                                 needs_lock = &info->lock;
663                         if (upper_limit >= stage) {
664                                 if (needs_lock) {
665                                         spin_lock(needs_lock);
666                                         *dir = NULL;
667                                         spin_unlock(needs_lock);
668                                         needs_lock = NULL;
669                                 } else
670                                         *dir = NULL;
671                                 nr_pages_to_free++;
672                                 list_add(&middir->lru, &pages_to_free);
673                         }
674                         shmem_dir_unmap(dir);
675                         cond_resched();
676                         dir = shmem_dir_map(middir);
677                         diroff = 0;
678                 }
679                 punch_lock = needs_lock;
680                 subdir = dir[diroff];
681                 if (subdir && !offset && upper_limit-idx >= ENTRIES_PER_PAGE) {
682                         if (needs_lock) {
683                                 spin_lock(needs_lock);
684                                 dir[diroff] = NULL;
685                                 spin_unlock(needs_lock);
686                                 punch_lock = NULL;
687                         } else
688                                 dir[diroff] = NULL;
689                         nr_pages_to_free++;
690                         list_add(&subdir->lru, &pages_to_free);
691                 }
692                 if (subdir && page_private(subdir) /* has swap entries */) {
693                         size = limit - idx;
694                         if (size > ENTRIES_PER_PAGE)
695                                 size = ENTRIES_PER_PAGE;
696                         freed = shmem_map_and_free_swp(subdir,
697                                         offset, size, &dir, punch_lock);
698                         if (!dir)
699                                 dir = shmem_dir_map(middir);
700                         nr_swaps_freed += freed;
701                         if (offset || punch_lock) {
702                                 spin_lock(&info->lock);
703                                 set_page_private(subdir,
704                                         page_private(subdir) - freed);
705                                 spin_unlock(&info->lock);
706                         } else
707                                 BUG_ON(page_private(subdir) != freed);
708                 }
709                 offset = 0;
710         }
711 done1:
712         shmem_dir_unmap(dir);
713 done2:
714         if (inode->i_mapping->nrpages && (info->flags & SHMEM_PAGEIN)) {
715                 /*
716                  * Call truncate_inode_pages again: racing shmem_unuse_inode
717                  * may have swizzled a page in from swap since vmtruncate or
718                  * generic_delete_inode did it, before we lowered next_index.
719                  * Also, though shmem_getpage checks i_size before adding to
720                  * cache, no recheck after: so fix the narrow window there too.
721                  *
722                  * Recalling truncate_inode_pages_range and unmap_mapping_range
723                  * every time for punch_hole (which never got a chance to clear
724                  * SHMEM_PAGEIN at the start of vmtruncate_range) is expensive,
725                  * yet hardly ever necessary: try to optimize them out later.
726                  */
727                 truncate_inode_pages_range(inode->i_mapping, start, end);
728                 if (punch_hole)
729                         unmap_mapping_range(inode->i_mapping, start,
730                                                         end - start, 1);
731         }
732
733         spin_lock(&info->lock);
734         info->flags &= ~SHMEM_TRUNCATE;
735         info->swapped -= nr_swaps_freed;
736         if (nr_pages_to_free)
737                 shmem_free_blocks(inode, nr_pages_to_free);
738         shmem_recalc_inode(inode);
739         spin_unlock(&info->lock);
740
741         /*
742          * Empty swap vector directory pages to be freed?
743          */
744         if (!list_empty(&pages_to_free)) {
745                 pages_to_free.prev->next = NULL;
746                 shmem_free_pages(pages_to_free.next);
747         }
748 }
749
750 static void shmem_truncate(struct inode *inode)
751 {
752         shmem_truncate_range(inode, inode->i_size, (loff_t)-1);
753 }
754
755 static int shmem_notify_change(struct dentry *dentry, struct iattr *attr)
756 {
757         struct inode *inode = dentry->d_inode;
758         struct page *page = NULL;
759         int error;
760
761         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
762                 if (attr->ia_size < inode->i_size) {
763                         /*
764                          * If truncating down to a partial page, then
765                          * if that page is already allocated, hold it
766                          * in memory until the truncation is over, so
767                          * truncate_partial_page cannnot miss it were
768                          * it assigned to swap.
769                          */
770                         if (attr->ia_size & (PAGE_CACHE_SIZE-1)) {
771                                 (void) shmem_getpage(inode,
772                                         attr->ia_size>>PAGE_CACHE_SHIFT,
773                                                 &page, SGP_READ, NULL);
774                                 if (page)
775                                         unlock_page(page);
776                         }
777                         /*
778                          * Reset SHMEM_PAGEIN flag so that shmem_truncate can
779                          * detect if any pages might have been added to cache
780                          * after truncate_inode_pages.  But we needn't bother
781                          * if it's being fully truncated to zero-length: the
782                          * nrpages check is efficient enough in that case.
783                          */
784                         if (attr->ia_size) {
785                                 struct shmem_inode_info *info = SHMEM_I(inode);
786                                 spin_lock(&info->lock);
787                                 info->flags &= ~SHMEM_PAGEIN;
788                                 spin_unlock(&info->lock);
789                         }
790                 }
791         }
792
793         error = inode_change_ok(inode, attr);
794         if (!error)
795                 error = inode_setattr(inode, attr);
796 #ifdef CONFIG_TMPFS_POSIX_ACL
797         if (!error && (attr->ia_valid & ATTR_MODE))
798                 error = generic_acl_chmod(inode, &shmem_acl_ops);
799 #endif
800         if (page)
801                 page_cache_release(page);
802         return error;
803 }
804
805 static void shmem_delete_inode(struct inode *inode)
806 {
807         struct shmem_inode_info *info = SHMEM_I(inode);
808
809         if (inode->i_op->truncate == shmem_truncate) {
810                 truncate_inode_pages(inode->i_mapping, 0);
811                 shmem_unacct_size(info->flags, inode->i_size);
812                 inode->i_size = 0;
813                 shmem_truncate(inode);
814                 if (!list_empty(&info->swaplist)) {
815                         mutex_lock(&shmem_swaplist_mutex);
816                         list_del_init(&info->swaplist);
817                         mutex_unlock(&shmem_swaplist_mutex);
818                 }
819         }
820         BUG_ON(inode->i_blocks);
821         shmem_free_inode(inode->i_sb);
822         clear_inode(inode);
823 }
824
825 static inline int shmem_find_swp(swp_entry_t entry, swp_entry_t *dir, swp_entry_t *edir)
826 {
827         swp_entry_t *ptr;
828
829         for (ptr = dir; ptr < edir; ptr++) {
830                 if (ptr->val == entry.val)
831                         return ptr - dir;
832         }
833         return -1;
834 }
835
836 static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, struct page *page)
837 {
838         struct inode *inode;
839         unsigned long idx;
840         unsigned long size;
841         unsigned long limit;
842         unsigned long stage;
843         struct page **dir;
844         struct page *subdir;
845         swp_entry_t *ptr;
846         int offset;
847         int error;
848
849         idx = 0;
850         ptr = info->i_direct;
851         spin_lock(&info->lock);
852         if (!info->swapped) {
853                 list_del_init(&info->swaplist);
854                 goto lost2;
855         }
856         limit = info->next_index;
857         size = limit;
858         if (size > SHMEM_NR_DIRECT)
859                 size = SHMEM_NR_DIRECT;
860         offset = shmem_find_swp(entry, ptr, ptr+size);
861         if (offset >= 0)
862                 goto found;
863         if (!info->i_indirect)
864                 goto lost2;
865
866         dir = shmem_dir_map(info->i_indirect);
867         stage = SHMEM_NR_DIRECT + ENTRIES_PER_PAGEPAGE/2;
868
869         for (idx = SHMEM_NR_DIRECT; idx < limit; idx += ENTRIES_PER_PAGE, dir++) {
870                 if (unlikely(idx == stage)) {
871                         shmem_dir_unmap(dir-1);
872                         if (cond_resched_lock(&info->lock)) {
873                                 /* check it has not been truncated */
874                                 if (limit > info->next_index) {
875                                         limit = info->next_index;
876                                         if (idx >= limit)
877                                                 goto lost2;
878                                 }
879                         }
880                         dir = shmem_dir_map(info->i_indirect) +
881                             ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
882                         while (!*dir) {
883                                 dir++;
884                                 idx += ENTRIES_PER_PAGEPAGE;
885                                 if (idx >= limit)
886                                         goto lost1;
887                         }
888                         stage = idx + ENTRIES_PER_PAGEPAGE;
889                         subdir = *dir;
890                         shmem_dir_unmap(dir);
891                         dir = shmem_dir_map(subdir);
892                 }
893                 subdir = *dir;
894                 if (subdir && page_private(subdir)) {
895                         ptr = shmem_swp_map(subdir);
896                         size = limit - idx;
897                         if (size > ENTRIES_PER_PAGE)
898                                 size = ENTRIES_PER_PAGE;
899                         offset = shmem_find_swp(entry, ptr, ptr+size);
900                         shmem_swp_unmap(ptr);
901                         if (offset >= 0) {
902                                 shmem_dir_unmap(dir);
903                                 goto found;
904                         }
905                 }
906         }
907 lost1:
908         shmem_dir_unmap(dir-1);
909 lost2:
910         spin_unlock(&info->lock);
911         return 0;
912 found:
913         idx += offset;
914         inode = igrab(&info->vfs_inode);
915         spin_unlock(&info->lock);
916
917         /*
918          * Move _head_ to start search for next from here.
919          * But be careful: shmem_delete_inode checks list_empty without taking
920          * mutex, and there's an instant in list_move_tail when info->swaplist
921          * would appear empty, if it were the only one on shmem_swaplist.  We
922          * could avoid doing it if inode NULL; or use this minor optimization.
923          */
924         if (shmem_swaplist.next != &info->swaplist)
925                 list_move_tail(&shmem_swaplist, &info->swaplist);
926         mutex_unlock(&shmem_swaplist_mutex);
927
928         error = 1;
929         if (!inode)
930                 goto out;
931         /*
932          * Charge page using GFP_HIGHUSER_MOVABLE while we can wait.
933          * charged back to the user(not to caller) when swap account is used.
934          */
935         error = mem_cgroup_cache_charge_swapin(page,
936                         current->mm, GFP_HIGHUSER_MOVABLE, true);
937         if (error)
938                 goto out;
939         error = radix_tree_preload(GFP_KERNEL);
940         if (error) {
941                 mem_cgroup_uncharge_cache_page(page);
942                 goto out;
943         }
944         error = 1;
945
946         spin_lock(&info->lock);
947         ptr = shmem_swp_entry(info, idx, NULL);
948         if (ptr && ptr->val == entry.val) {
949                 error = add_to_page_cache_locked(page, inode->i_mapping,
950                                                 idx, GFP_NOWAIT);
951                 /* does mem_cgroup_uncharge_cache_page on error */
952         } else  /* we must compensate for our precharge above */
953                 mem_cgroup_uncharge_cache_page(page);
954
955         if (error == -EEXIST) {
956                 struct page *filepage = find_get_page(inode->i_mapping, idx);
957                 error = 1;
958                 if (filepage) {
959                         /*
960                          * There might be a more uptodate page coming down
961                          * from a stacked writepage: forget our swappage if so.
962                          */
963                         if (PageUptodate(filepage))
964                                 error = 0;
965                         page_cache_release(filepage);
966                 }
967         }
968         if (!error) {
969                 delete_from_swap_cache(page);
970                 set_page_dirty(page);
971                 info->flags |= SHMEM_PAGEIN;
972                 shmem_swp_set(info, ptr, 0);
973                 swap_free(entry);
974                 error = 1;      /* not an error, but entry was found */
975         }
976         if (ptr)
977                 shmem_swp_unmap(ptr);
978         spin_unlock(&info->lock);
979         radix_tree_preload_end();
980 out:
981         unlock_page(page);
982         page_cache_release(page);
983         iput(inode);            /* allows for NULL */
984         return error;
985 }
986
987 /*
988  * shmem_unuse() search for an eventually swapped out shmem page.
989  */
990 int shmem_unuse(swp_entry_t entry, struct page *page)
991 {
992         struct list_head *p, *next;
993         struct shmem_inode_info *info;
994         int found = 0;
995
996         mutex_lock(&shmem_swaplist_mutex);
997         list_for_each_safe(p, next, &shmem_swaplist) {
998                 info = list_entry(p, struct shmem_inode_info, swaplist);
999                 found = shmem_unuse_inode(info, entry, page);
1000                 cond_resched();
1001                 if (found)
1002                         goto out;
1003         }
1004         mutex_unlock(&shmem_swaplist_mutex);
1005 out:    return found;   /* 0 or 1 or -ENOMEM */
1006 }
1007
1008 /*
1009  * Move the page from the page cache to the swap cache.
1010  */
1011 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1012 {
1013         struct shmem_inode_info *info;
1014         swp_entry_t *entry, swap;
1015         struct address_space *mapping;
1016         unsigned long index;
1017         struct inode *inode;
1018
1019         BUG_ON(!PageLocked(page));
1020         mapping = page->mapping;
1021         index = page->index;
1022         inode = mapping->host;
1023         info = SHMEM_I(inode);
1024         if (info->flags & VM_LOCKED)
1025                 goto redirty;
1026         if (!total_swap_pages)
1027                 goto redirty;
1028
1029         /*
1030          * shmem_backing_dev_info's capabilities prevent regular writeback or
1031          * sync from ever calling shmem_writepage; but a stacking filesystem
1032          * may use the ->writepage of its underlying filesystem, in which case
1033          * tmpfs should write out to swap only in response to memory pressure,
1034          * and not for pdflush or sync.  However, in those cases, we do still
1035          * want to check if there's a redundant swappage to be discarded.
1036          */
1037         if (wbc->for_reclaim)
1038                 swap = get_swap_page();
1039         else
1040                 swap.val = 0;
1041
1042         spin_lock(&info->lock);
1043         if (index >= info->next_index) {
1044                 BUG_ON(!(info->flags & SHMEM_TRUNCATE));
1045                 goto unlock;
1046         }
1047         entry = shmem_swp_entry(info, index, NULL);
1048         if (entry->val) {
1049                 /*
1050                  * The more uptodate page coming down from a stacked
1051                  * writepage should replace our old swappage.
1052                  */
1053                 free_swap_and_cache(*entry);
1054                 shmem_swp_set(info, entry, 0);
1055         }
1056         shmem_recalc_inode(inode);
1057
1058         if (swap.val && add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
1059                 remove_from_page_cache(page);
1060                 shmem_swp_set(info, entry, swap.val);
1061                 shmem_swp_unmap(entry);
1062                 if (list_empty(&info->swaplist))
1063                         inode = igrab(inode);
1064                 else
1065                         inode = NULL;
1066                 spin_unlock(&info->lock);
1067                 swap_duplicate(swap);
1068                 BUG_ON(page_mapped(page));
1069                 page_cache_release(page);       /* pagecache ref */
1070                 set_page_dirty(page);
1071                 unlock_page(page);
1072                 if (inode) {
1073                         mutex_lock(&shmem_swaplist_mutex);
1074                         /* move instead of add in case we're racing */
1075                         list_move_tail(&info->swaplist, &shmem_swaplist);
1076                         mutex_unlock(&shmem_swaplist_mutex);
1077                         iput(inode);
1078                 }
1079                 return 0;
1080         }
1081
1082         shmem_swp_unmap(entry);
1083 unlock:
1084         spin_unlock(&info->lock);
1085         swap_free(swap);
1086 redirty:
1087         set_page_dirty(page);
1088         if (wbc->for_reclaim)
1089                 return AOP_WRITEPAGE_ACTIVATE;  /* Return with page locked */
1090         unlock_page(page);
1091         return 0;
1092 }
1093
1094 #ifdef CONFIG_NUMA
1095 #ifdef CONFIG_TMPFS
1096 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1097 {
1098         char buffer[64];
1099
1100         if (!mpol || mpol->mode == MPOL_DEFAULT)
1101                 return;         /* show nothing */
1102
1103         mpol_to_str(buffer, sizeof(buffer), mpol, 1);
1104
1105         seq_printf(seq, ",mpol=%s", buffer);
1106 }
1107
1108 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1109 {
1110         struct mempolicy *mpol = NULL;
1111         if (sbinfo->mpol) {
1112                 spin_lock(&sbinfo->stat_lock);  /* prevent replace/use races */
1113                 mpol = sbinfo->mpol;
1114                 mpol_get(mpol);
1115                 spin_unlock(&sbinfo->stat_lock);
1116         }
1117         return mpol;
1118 }
1119 #endif /* CONFIG_TMPFS */
1120
1121 static struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp,
1122                         struct shmem_inode_info *info, unsigned long idx)
1123 {
1124         struct mempolicy mpol, *spol;
1125         struct vm_area_struct pvma;
1126         struct page *page;
1127
1128         spol = mpol_cond_copy(&mpol,
1129                                 mpol_shared_policy_lookup(&info->policy, idx));
1130
1131         /* Create a pseudo vma that just contains the policy */
1132         pvma.vm_start = 0;
1133         pvma.vm_pgoff = idx;
1134         pvma.vm_ops = NULL;
1135         pvma.vm_policy = spol;
1136         page = swapin_readahead(entry, gfp, &pvma, 0);
1137         return page;
1138 }
1139
1140 static struct page *shmem_alloc_page(gfp_t gfp,
1141                         struct shmem_inode_info *info, unsigned long idx)
1142 {
1143         struct vm_area_struct pvma;
1144
1145         /* Create a pseudo vma that just contains the policy */
1146         pvma.vm_start = 0;
1147         pvma.vm_pgoff = idx;
1148         pvma.vm_ops = NULL;
1149         pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx);
1150
1151         /*
1152          * alloc_page_vma() will drop the shared policy reference
1153          */
1154         return alloc_page_vma(gfp, &pvma, 0);
1155 }
1156 #else /* !CONFIG_NUMA */
1157 #ifdef CONFIG_TMPFS
1158 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *p)
1159 {
1160 }
1161 #endif /* CONFIG_TMPFS */
1162
1163 static inline struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp,
1164                         struct shmem_inode_info *info, unsigned long idx)
1165 {
1166         return swapin_readahead(entry, gfp, NULL, 0);
1167 }
1168
1169 static inline struct page *shmem_alloc_page(gfp_t gfp,
1170                         struct shmem_inode_info *info, unsigned long idx)
1171 {
1172         return alloc_page(gfp);
1173 }
1174 #endif /* CONFIG_NUMA */
1175
1176 #if !defined(CONFIG_NUMA) || !defined(CONFIG_TMPFS)
1177 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1178 {
1179         return NULL;
1180 }
1181 #endif
1182
1183 /*
1184  * shmem_getpage - either get the page from swap or allocate a new one
1185  *
1186  * If we allocate a new one we do not mark it dirty. That's up to the
1187  * vm. If we swap it in we mark it dirty since we also free the swap
1188  * entry since a page cannot live in both the swap and page cache
1189  */
1190 static int shmem_getpage(struct inode *inode, unsigned long idx,
1191                         struct page **pagep, enum sgp_type sgp, int *type)
1192 {
1193         struct address_space *mapping = inode->i_mapping;
1194         struct shmem_inode_info *info = SHMEM_I(inode);
1195         struct shmem_sb_info *sbinfo;
1196         struct page *filepage = *pagep;
1197         struct page *swappage;
1198         swp_entry_t *entry;
1199         swp_entry_t swap;
1200         gfp_t gfp;
1201         int error;
1202
1203         if (idx >= SHMEM_MAX_INDEX)
1204                 return -EFBIG;
1205
1206         if (type)
1207                 *type = 0;
1208
1209         /*
1210          * Normally, filepage is NULL on entry, and either found
1211          * uptodate immediately, or allocated and zeroed, or read
1212          * in under swappage, which is then assigned to filepage.
1213          * But shmem_readpage (required for splice) passes in a locked
1214          * filepage, which may be found not uptodate by other callers
1215          * too, and may need to be copied from the swappage read in.
1216          */
1217 repeat:
1218         if (!filepage)
1219                 filepage = find_lock_page(mapping, idx);
1220         if (filepage && PageUptodate(filepage))
1221                 goto done;
1222         error = 0;
1223         gfp = mapping_gfp_mask(mapping);
1224         if (!filepage) {
1225                 /*
1226                  * Try to preload while we can wait, to not make a habit of
1227                  * draining atomic reserves; but don't latch on to this cpu.
1228                  */
1229                 error = radix_tree_preload(gfp & ~__GFP_HIGHMEM);
1230                 if (error)
1231                         goto failed;
1232                 radix_tree_preload_end();
1233         }
1234
1235         spin_lock(&info->lock);
1236         shmem_recalc_inode(inode);
1237         entry = shmem_swp_alloc(info, idx, sgp);
1238         if (IS_ERR(entry)) {
1239                 spin_unlock(&info->lock);
1240                 error = PTR_ERR(entry);
1241                 goto failed;
1242         }
1243         swap = *entry;
1244
1245         if (swap.val) {
1246                 /* Look it up and read it in.. */
1247                 swappage = lookup_swap_cache(swap);
1248                 if (!swappage) {
1249                         shmem_swp_unmap(entry);
1250                         /* here we actually do the io */
1251                         if (type && !(*type & VM_FAULT_MAJOR)) {
1252                                 __count_vm_event(PGMAJFAULT);
1253                                 *type |= VM_FAULT_MAJOR;
1254                         }
1255                         spin_unlock(&info->lock);
1256                         swappage = shmem_swapin(swap, gfp, info, idx);
1257                         if (!swappage) {
1258                                 spin_lock(&info->lock);
1259                                 entry = shmem_swp_alloc(info, idx, sgp);
1260                                 if (IS_ERR(entry))
1261                                         error = PTR_ERR(entry);
1262                                 else {
1263                                         if (entry->val == swap.val)
1264                                                 error = -ENOMEM;
1265                                         shmem_swp_unmap(entry);
1266                                 }
1267                                 spin_unlock(&info->lock);
1268                                 if (error)
1269                                         goto failed;
1270                                 goto repeat;
1271                         }
1272                         wait_on_page_locked(swappage);
1273                         /*
1274                          * We want to avoid charge at add_to_page_cache().
1275                          * charge against this swap cache here.
1276                          */
1277                         if (mem_cgroup_cache_charge_swapin(swappage,
1278                                                 current->mm, gfp, false)) {
1279                                 page_cache_release(swappage);
1280                                 error = -ENOMEM;
1281                                 goto failed;
1282                         }
1283                         page_cache_release(swappage);
1284                         goto repeat;
1285                 }
1286
1287                 /* We have to do this with page locked to prevent races */
1288                 if (!trylock_page(swappage)) {
1289                         shmem_swp_unmap(entry);
1290                         spin_unlock(&info->lock);
1291                         wait_on_page_locked(swappage);
1292                         page_cache_release(swappage);
1293                         goto repeat;
1294                 }
1295                 if (PageWriteback(swappage)) {
1296                         shmem_swp_unmap(entry);
1297                         spin_unlock(&info->lock);
1298                         wait_on_page_writeback(swappage);
1299                         unlock_page(swappage);
1300                         page_cache_release(swappage);
1301                         goto repeat;
1302                 }
1303                 if (!PageUptodate(swappage)) {
1304                         shmem_swp_unmap(entry);
1305                         spin_unlock(&info->lock);
1306                         unlock_page(swappage);
1307                         page_cache_release(swappage);
1308                         error = -EIO;
1309                         goto failed;
1310                 }
1311
1312                 if (filepage) {
1313                         shmem_swp_set(info, entry, 0);
1314                         shmem_swp_unmap(entry);
1315                         delete_from_swap_cache(swappage);
1316                         spin_unlock(&info->lock);
1317                         copy_highpage(filepage, swappage);
1318                         unlock_page(swappage);
1319                         page_cache_release(swappage);
1320                         flush_dcache_page(filepage);
1321                         SetPageUptodate(filepage);
1322                         set_page_dirty(filepage);
1323                         swap_free(swap);
1324                 } else if (!(error = add_to_page_cache_locked(swappage, mapping,
1325                                         idx, GFP_NOWAIT))) {
1326                         info->flags |= SHMEM_PAGEIN;
1327                         shmem_swp_set(info, entry, 0);
1328                         shmem_swp_unmap(entry);
1329                         delete_from_swap_cache(swappage);
1330                         spin_unlock(&info->lock);
1331                         filepage = swappage;
1332                         set_page_dirty(filepage);
1333                         swap_free(swap);
1334                 } else {
1335                         shmem_swp_unmap(entry);
1336                         spin_unlock(&info->lock);
1337                         unlock_page(swappage);
1338                         page_cache_release(swappage);
1339                         if (error == -ENOMEM) {
1340                                 /* allow reclaim from this memory cgroup */
1341                                 error = mem_cgroup_shrink_usage(current->mm,
1342                                                                 gfp);
1343                                 if (error)
1344                                         goto failed;
1345                         }
1346                         goto repeat;
1347                 }
1348         } else if (sgp == SGP_READ && !filepage) {
1349                 shmem_swp_unmap(entry);
1350                 filepage = find_get_page(mapping, idx);
1351                 if (filepage &&
1352                     (!PageUptodate(filepage) || !trylock_page(filepage))) {
1353                         spin_unlock(&info->lock);
1354                         wait_on_page_locked(filepage);
1355                         page_cache_release(filepage);
1356                         filepage = NULL;
1357                         goto repeat;
1358                 }
1359                 spin_unlock(&info->lock);
1360         } else {
1361                 shmem_swp_unmap(entry);
1362                 sbinfo = SHMEM_SB(inode->i_sb);
1363                 if (sbinfo->max_blocks) {
1364                         spin_lock(&sbinfo->stat_lock);
1365                         if (sbinfo->free_blocks == 0 ||
1366                             shmem_acct_block(info->flags)) {
1367                                 spin_unlock(&sbinfo->stat_lock);
1368                                 spin_unlock(&info->lock);
1369                                 error = -ENOSPC;
1370                                 goto failed;
1371                         }
1372                         sbinfo->free_blocks--;
1373                         inode->i_blocks += BLOCKS_PER_PAGE;
1374                         spin_unlock(&sbinfo->stat_lock);
1375                 } else if (shmem_acct_block(info->flags)) {
1376                         spin_unlock(&info->lock);
1377                         error = -ENOSPC;
1378                         goto failed;
1379                 }
1380
1381                 if (!filepage) {
1382                         int ret;
1383
1384                         spin_unlock(&info->lock);
1385                         filepage = shmem_alloc_page(gfp, info, idx);
1386                         if (!filepage) {
1387                                 shmem_unacct_blocks(info->flags, 1);
1388                                 shmem_free_blocks(inode, 1);
1389                                 error = -ENOMEM;
1390                                 goto failed;
1391                         }
1392                         SetPageSwapBacked(filepage);
1393
1394                         /* Precharge page while we can wait, compensate after */
1395                         error = mem_cgroup_cache_charge(filepage, current->mm,
1396                                         GFP_HIGHUSER_MOVABLE);
1397                         if (error) {
1398                                 page_cache_release(filepage);
1399                                 shmem_unacct_blocks(info->flags, 1);
1400                                 shmem_free_blocks(inode, 1);
1401                                 filepage = NULL;
1402                                 goto failed;
1403                         }
1404
1405                         spin_lock(&info->lock);
1406                         entry = shmem_swp_alloc(info, idx, sgp);
1407                         if (IS_ERR(entry))
1408                                 error = PTR_ERR(entry);
1409                         else {
1410                                 swap = *entry;
1411                                 shmem_swp_unmap(entry);
1412                         }
1413                         ret = error || swap.val;
1414                         if (ret)
1415                                 mem_cgroup_uncharge_cache_page(filepage);
1416                         else
1417                                 ret = add_to_page_cache_lru(filepage, mapping,
1418                                                 idx, GFP_NOWAIT);
1419                         /*
1420                          * At add_to_page_cache_lru() failure, uncharge will
1421                          * be done automatically.
1422                          */
1423                         if (ret) {
1424                                 spin_unlock(&info->lock);
1425                                 page_cache_release(filepage);
1426                                 shmem_unacct_blocks(info->flags, 1);
1427                                 shmem_free_blocks(inode, 1);
1428                                 filepage = NULL;
1429                                 if (error)
1430                                         goto failed;
1431                                 goto repeat;
1432                         }
1433                         info->flags |= SHMEM_PAGEIN;
1434                 }
1435
1436                 info->alloced++;
1437                 spin_unlock(&info->lock);
1438                 clear_highpage(filepage);
1439                 flush_dcache_page(filepage);
1440                 SetPageUptodate(filepage);
1441                 if (sgp == SGP_DIRTY)
1442                         set_page_dirty(filepage);
1443         }
1444 done:
1445         *pagep = filepage;
1446         return 0;
1447
1448 failed:
1449         if (*pagep != filepage) {
1450                 unlock_page(filepage);
1451                 page_cache_release(filepage);
1452         }
1453         return error;
1454 }
1455
1456 static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1457 {
1458         struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1459         int error;
1460         int ret;
1461
1462         if (((loff_t)vmf->pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode))
1463                 return VM_FAULT_SIGBUS;
1464
1465         error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
1466         if (error)
1467                 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
1468
1469         return ret | VM_FAULT_LOCKED;
1470 }
1471
1472 #ifdef CONFIG_NUMA
1473 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
1474 {
1475         struct inode *i = vma->vm_file->f_path.dentry->d_inode;
1476         return mpol_set_shared_policy(&SHMEM_I(i)->policy, vma, new);
1477 }
1478
1479 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
1480                                           unsigned long addr)
1481 {
1482         struct inode *i = vma->vm_file->f_path.dentry->d_inode;
1483         unsigned long idx;
1484
1485         idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1486         return mpol_shared_policy_lookup(&SHMEM_I(i)->policy, idx);
1487 }
1488 #endif
1489
1490 int shmem_lock(struct file *file, int lock, struct user_struct *user)
1491 {
1492         struct inode *inode = file->f_path.dentry->d_inode;
1493         struct shmem_inode_info *info = SHMEM_I(inode);
1494         int retval = -ENOMEM;
1495
1496         spin_lock(&info->lock);
1497         if (lock && !(info->flags & VM_LOCKED)) {
1498                 if (!user_shm_lock(inode->i_size, user))
1499                         goto out_nomem;
1500                 info->flags |= VM_LOCKED;
1501                 mapping_set_unevictable(file->f_mapping);
1502         }
1503         if (!lock && (info->flags & VM_LOCKED) && user) {
1504                 user_shm_unlock(inode->i_size, user);
1505                 info->flags &= ~VM_LOCKED;
1506                 mapping_clear_unevictable(file->f_mapping);
1507                 scan_mapping_unevictable_pages(file->f_mapping);
1508         }
1509         retval = 0;
1510
1511 out_nomem:
1512         spin_unlock(&info->lock);
1513         return retval;
1514 }
1515
1516 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
1517 {
1518         file_accessed(file);
1519         vma->vm_ops = &shmem_vm_ops;
1520         vma->vm_flags |= VM_CAN_NONLINEAR;
1521         return 0;
1522 }
1523
1524 static struct inode *
1525 shmem_get_inode(struct super_block *sb, int mode, dev_t dev)
1526 {
1527         struct inode *inode;
1528         struct shmem_inode_info *info;
1529         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
1530
1531         if (shmem_reserve_inode(sb))
1532                 return NULL;
1533
1534         inode = new_inode(sb);
1535         if (inode) {
1536                 inode->i_mode = mode;
1537                 inode->i_uid = current_fsuid();
1538                 inode->i_gid = current_fsgid();
1539                 inode->i_blocks = 0;
1540                 inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
1541                 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
1542                 inode->i_generation = get_seconds();
1543                 info = SHMEM_I(inode);
1544                 memset(info, 0, (char *)inode - (char *)info);
1545                 spin_lock_init(&info->lock);
1546                 INIT_LIST_HEAD(&info->swaplist);
1547
1548                 switch (mode & S_IFMT) {
1549                 default:
1550                         inode->i_op = &shmem_special_inode_operations;
1551                         init_special_inode(inode, mode, dev);
1552                         break;
1553                 case S_IFREG:
1554                         inode->i_mapping->a_ops = &shmem_aops;
1555                         inode->i_op = &shmem_inode_operations;
1556                         inode->i_fop = &shmem_file_operations;
1557                         mpol_shared_policy_init(&info->policy,
1558                                                  shmem_get_sbmpol(sbinfo));
1559                         break;
1560                 case S_IFDIR:
1561                         inc_nlink(inode);
1562                         /* Some things misbehave if size == 0 on a directory */
1563                         inode->i_size = 2 * BOGO_DIRENT_SIZE;
1564                         inode->i_op = &shmem_dir_inode_operations;
1565                         inode->i_fop = &simple_dir_operations;
1566                         break;
1567                 case S_IFLNK:
1568                         /*
1569                          * Must not load anything in the rbtree,
1570                          * mpol_free_shared_policy will not be called.
1571                          */
1572                         mpol_shared_policy_init(&info->policy, NULL);
1573                         break;
1574                 }
1575         } else
1576                 shmem_free_inode(sb);
1577         return inode;
1578 }
1579
1580 #ifdef CONFIG_TMPFS
1581 static const struct inode_operations shmem_symlink_inode_operations;
1582 static const struct inode_operations shmem_symlink_inline_operations;
1583
1584 /*
1585  * Normally tmpfs avoids the use of shmem_readpage and shmem_write_begin;
1586  * but providing them allows a tmpfs file to be used for splice, sendfile, and
1587  * below the loop driver, in the generic fashion that many filesystems support.
1588  */
1589 static int shmem_readpage(struct file *file, struct page *page)
1590 {
1591         struct inode *inode = page->mapping->host;
1592         int error = shmem_getpage(inode, page->index, &page, SGP_CACHE, NULL);
1593         unlock_page(page);
1594         return error;
1595 }
1596
1597 static int
1598 shmem_write_begin(struct file *file, struct address_space *mapping,
1599                         loff_t pos, unsigned len, unsigned flags,
1600                         struct page **pagep, void **fsdata)
1601 {
1602         struct inode *inode = mapping->host;
1603         pgoff_t index = pos >> PAGE_CACHE_SHIFT;
1604         *pagep = NULL;
1605         return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL);
1606 }
1607
1608 static int
1609 shmem_write_end(struct file *file, struct address_space *mapping,
1610                         loff_t pos, unsigned len, unsigned copied,
1611                         struct page *page, void *fsdata)
1612 {
1613         struct inode *inode = mapping->host;
1614
1615         if (pos + copied > inode->i_size)
1616                 i_size_write(inode, pos + copied);
1617
1618         unlock_page(page);
1619         set_page_dirty(page);
1620         page_cache_release(page);
1621
1622         return copied;
1623 }
1624
1625 static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor)
1626 {
1627         struct inode *inode = filp->f_path.dentry->d_inode;
1628         struct address_space *mapping = inode->i_mapping;
1629         unsigned long index, offset;
1630         enum sgp_type sgp = SGP_READ;
1631
1632         /*
1633          * Might this read be for a stacking filesystem?  Then when reading
1634          * holes of a sparse file, we actually need to allocate those pages,
1635          * and even mark them dirty, so it cannot exceed the max_blocks limit.
1636          */
1637         if (segment_eq(get_fs(), KERNEL_DS))
1638                 sgp = SGP_DIRTY;
1639
1640         index = *ppos >> PAGE_CACHE_SHIFT;
1641         offset = *ppos & ~PAGE_CACHE_MASK;
1642
1643         for (;;) {
1644                 struct page *page = NULL;
1645                 unsigned long end_index, nr, ret;
1646                 loff_t i_size = i_size_read(inode);
1647
1648                 end_index = i_size >> PAGE_CACHE_SHIFT;
1649                 if (index > end_index)
1650                         break;
1651                 if (index == end_index) {
1652                         nr = i_size & ~PAGE_CACHE_MASK;
1653                         if (nr <= offset)
1654                                 break;
1655                 }
1656
1657                 desc->error = shmem_getpage(inode, index, &page, sgp, NULL);
1658                 if (desc->error) {
1659                         if (desc->error == -EINVAL)
1660                                 desc->error = 0;
1661                         break;
1662                 }
1663                 if (page)
1664                         unlock_page(page);
1665
1666                 /*
1667                  * We must evaluate after, since reads (unlike writes)
1668                  * are called without i_mutex protection against truncate
1669                  */
1670                 nr = PAGE_CACHE_SIZE;
1671                 i_size = i_size_read(inode);
1672                 end_index = i_size >> PAGE_CACHE_SHIFT;
1673                 if (index == end_index) {
1674                         nr = i_size & ~PAGE_CACHE_MASK;
1675                         if (nr <= offset) {
1676                                 if (page)
1677                                         page_cache_release(page);
1678                                 break;
1679                         }
1680                 }
1681                 nr -= offset;
1682
1683                 if (page) {
1684                         /*
1685                          * If users can be writing to this page using arbitrary
1686                          * virtual addresses, take care about potential aliasing
1687                          * before reading the page on the kernel side.
1688                          */
1689                         if (mapping_writably_mapped(mapping))
1690                                 flush_dcache_page(page);
1691                         /*
1692                          * Mark the page accessed if we read the beginning.
1693                          */
1694                         if (!offset)
1695                                 mark_page_accessed(page);
1696                 } else {
1697                         page = ZERO_PAGE(0);
1698                         page_cache_get(page);
1699                 }
1700
1701                 /*
1702                  * Ok, we have the page, and it's up-to-date, so
1703                  * now we can copy it to user space...
1704                  *
1705                  * The actor routine returns how many bytes were actually used..
1706                  * NOTE! This may not be the same as how much of a user buffer
1707                  * we filled up (we may be padding etc), so we can only update
1708                  * "pos" here (the actor routine has to update the user buffer
1709                  * pointers and the remaining count).
1710                  */
1711                 ret = actor(desc, page, offset, nr);
1712                 offset += ret;
1713                 index += offset >> PAGE_CACHE_SHIFT;
1714                 offset &= ~PAGE_CACHE_MASK;
1715
1716                 page_cache_release(page);
1717                 if (ret != nr || !desc->count)
1718                         break;
1719
1720                 cond_resched();
1721         }
1722
1723         *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
1724         file_accessed(filp);
1725 }
1726
1727 static ssize_t shmem_file_aio_read(struct kiocb *iocb,
1728                 const struct iovec *iov, unsigned long nr_segs, loff_t pos)
1729 {
1730         struct file *filp = iocb->ki_filp;
1731         ssize_t retval;
1732         unsigned long seg;
1733         size_t count;
1734         loff_t *ppos = &iocb->ki_pos;
1735
1736         retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
1737         if (retval)
1738                 return retval;
1739
1740         for (seg = 0; seg < nr_segs; seg++) {
1741                 read_descriptor_t desc;
1742
1743                 desc.written = 0;
1744                 desc.arg.buf = iov[seg].iov_base;
1745                 desc.count = iov[seg].iov_len;
1746                 if (desc.count == 0)
1747                         continue;
1748                 desc.error = 0;
1749                 do_shmem_file_read(filp, ppos, &desc, file_read_actor);
1750                 retval += desc.written;
1751                 if (desc.error) {
1752                         retval = retval ?: desc.error;
1753                         break;
1754                 }
1755                 if (desc.count > 0)
1756                         break;
1757         }
1758         return retval;
1759 }
1760
1761 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
1762 {
1763         struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
1764
1765         buf->f_type = TMPFS_MAGIC;
1766         buf->f_bsize = PAGE_CACHE_SIZE;
1767         buf->f_namelen = NAME_MAX;
1768         spin_lock(&sbinfo->stat_lock);
1769         if (sbinfo->max_blocks) {
1770                 buf->f_blocks = sbinfo->max_blocks;
1771                 buf->f_bavail = buf->f_bfree = sbinfo->free_blocks;
1772         }
1773         if (sbinfo->max_inodes) {
1774                 buf->f_files = sbinfo->max_inodes;
1775                 buf->f_ffree = sbinfo->free_inodes;
1776         }
1777         /* else leave those fields 0 like simple_statfs */
1778         spin_unlock(&sbinfo->stat_lock);
1779         return 0;
1780 }
1781
1782 /*
1783  * File creation. Allocate an inode, and we're done..
1784  */
1785 static int
1786 shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev)
1787 {
1788         struct inode *inode = shmem_get_inode(dir->i_sb, mode, dev);
1789         int error = -ENOSPC;
1790
1791         if (inode) {
1792                 error = security_inode_init_security(inode, dir, NULL, NULL,
1793                                                      NULL);
1794                 if (error) {
1795                         if (error != -EOPNOTSUPP) {
1796                                 iput(inode);
1797                                 return error;
1798                         }
1799                 }
1800                 error = shmem_acl_init(inode, dir);
1801                 if (error) {
1802                         iput(inode);
1803                         return error;
1804                 }
1805                 if (dir->i_mode & S_ISGID) {
1806                         inode->i_gid = dir->i_gid;
1807                         if (S_ISDIR(mode))
1808                                 inode->i_mode |= S_ISGID;
1809                 }
1810                 dir->i_size += BOGO_DIRENT_SIZE;
1811                 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1812                 d_instantiate(dentry, inode);
1813                 dget(dentry); /* Extra count - pin the dentry in core */
1814         }
1815         return error;
1816 }
1817
1818 static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode)
1819 {
1820         int error;
1821
1822         if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
1823                 return error;
1824         inc_nlink(dir);
1825         return 0;
1826 }
1827
1828 static int shmem_create(struct inode *dir, struct dentry *dentry, int mode,
1829                 struct nameidata *nd)
1830 {
1831         return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
1832 }
1833
1834 /*
1835  * Link a file..
1836  */
1837 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
1838 {
1839         struct inode *inode = old_dentry->d_inode;
1840         int ret;
1841
1842         /*
1843          * No ordinary (disk based) filesystem counts links as inodes;
1844          * but each new link needs a new dentry, pinning lowmem, and
1845          * tmpfs dentries cannot be pruned until they are unlinked.
1846          */
1847         ret = shmem_reserve_inode(inode->i_sb);
1848         if (ret)
1849                 goto out;
1850
1851         dir->i_size += BOGO_DIRENT_SIZE;
1852         inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1853         inc_nlink(inode);
1854         atomic_inc(&inode->i_count);    /* New dentry reference */
1855         dget(dentry);           /* Extra pinning count for the created dentry */
1856         d_instantiate(dentry, inode);
1857 out:
1858         return ret;
1859 }
1860
1861 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
1862 {
1863         struct inode *inode = dentry->d_inode;
1864
1865         if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
1866                 shmem_free_inode(inode->i_sb);
1867
1868         dir->i_size -= BOGO_DIRENT_SIZE;
1869         inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1870         drop_nlink(inode);
1871         dput(dentry);   /* Undo the count from "create" - this does all the work */
1872         return 0;
1873 }
1874
1875 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
1876 {
1877         if (!simple_empty(dentry))
1878                 return -ENOTEMPTY;
1879
1880         drop_nlink(dentry->d_inode);
1881         drop_nlink(dir);
1882         return shmem_unlink(dir, dentry);
1883 }
1884
1885 /*
1886  * The VFS layer already does all the dentry stuff for rename,
1887  * we just have to decrement the usage count for the target if
1888  * it exists so that the VFS layer correctly free's it when it
1889  * gets overwritten.
1890  */
1891 static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
1892 {
1893         struct inode *inode = old_dentry->d_inode;
1894         int they_are_dirs = S_ISDIR(inode->i_mode);
1895
1896         if (!simple_empty(new_dentry))
1897                 return -ENOTEMPTY;
1898
1899         if (new_dentry->d_inode) {
1900                 (void) shmem_unlink(new_dir, new_dentry);
1901                 if (they_are_dirs)
1902                         drop_nlink(old_dir);
1903         } else if (they_are_dirs) {
1904                 drop_nlink(old_dir);
1905                 inc_nlink(new_dir);
1906         }
1907
1908         old_dir->i_size -= BOGO_DIRENT_SIZE;
1909         new_dir->i_size += BOGO_DIRENT_SIZE;
1910         old_dir->i_ctime = old_dir->i_mtime =
1911         new_dir->i_ctime = new_dir->i_mtime =
1912         inode->i_ctime = CURRENT_TIME;
1913         return 0;
1914 }
1915
1916 static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
1917 {
1918         int error;
1919         int len;
1920         struct inode *inode;
1921         struct page *page = NULL;
1922         char *kaddr;
1923         struct shmem_inode_info *info;
1924
1925         len = strlen(symname) + 1;
1926         if (len > PAGE_CACHE_SIZE)
1927                 return -ENAMETOOLONG;
1928
1929         inode = shmem_get_inode(dir->i_sb, S_IFLNK|S_IRWXUGO, 0);
1930         if (!inode)
1931                 return -ENOSPC;
1932
1933         error = security_inode_init_security(inode, dir, NULL, NULL,
1934                                              NULL);
1935         if (error) {
1936                 if (error != -EOPNOTSUPP) {
1937                         iput(inode);
1938                         return error;
1939                 }
1940                 error = 0;
1941         }
1942
1943         info = SHMEM_I(inode);
1944         inode->i_size = len-1;
1945         if (len <= (char *)inode - (char *)info) {
1946                 /* do it inline */
1947                 memcpy(info, symname, len);
1948                 inode->i_op = &shmem_symlink_inline_operations;
1949         } else {
1950                 error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
1951                 if (error) {
1952                         iput(inode);
1953                         return error;
1954                 }
1955                 unlock_page(page);
1956                 inode->i_mapping->a_ops = &shmem_aops;
1957                 inode->i_op = &shmem_symlink_inode_operations;
1958                 kaddr = kmap_atomic(page, KM_USER0);
1959                 memcpy(kaddr, symname, len);
1960                 kunmap_atomic(kaddr, KM_USER0);
1961                 set_page_dirty(page);
1962                 page_cache_release(page);
1963         }
1964         if (dir->i_mode & S_ISGID)
1965                 inode->i_gid = dir->i_gid;
1966         dir->i_size += BOGO_DIRENT_SIZE;
1967         dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1968         d_instantiate(dentry, inode);
1969         dget(dentry);
1970         return 0;
1971 }
1972
1973 static void *shmem_follow_link_inline(struct dentry *dentry, struct nameidata *nd)
1974 {
1975         nd_set_link(nd, (char *)SHMEM_I(dentry->d_inode));
1976         return NULL;
1977 }
1978
1979 static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
1980 {
1981         struct page *page = NULL;
1982         int res = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
1983         nd_set_link(nd, res ? ERR_PTR(res) : kmap(page));
1984         if (page)
1985                 unlock_page(page);
1986         return page;
1987 }
1988
1989 static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
1990 {
1991         if (!IS_ERR(nd_get_link(nd))) {
1992                 struct page *page = cookie;
1993                 kunmap(page);
1994                 mark_page_accessed(page);
1995                 page_cache_release(page);
1996         }
1997 }
1998
1999 static const struct inode_operations shmem_symlink_inline_operations = {
2000         .readlink       = generic_readlink,
2001         .follow_link    = shmem_follow_link_inline,
2002 };
2003
2004 static const struct inode_operations shmem_symlink_inode_operations = {
2005         .truncate       = shmem_truncate,
2006         .readlink       = generic_readlink,
2007         .follow_link    = shmem_follow_link,
2008         .put_link       = shmem_put_link,
2009 };
2010
2011 #ifdef CONFIG_TMPFS_POSIX_ACL
2012 /*
2013  * Superblocks without xattr inode operations will get security.* xattr
2014  * support from the VFS "for free". As soon as we have any other xattrs
2015  * like ACLs, we also need to implement the security.* handlers at
2016  * filesystem level, though.
2017  */
2018
2019 static size_t shmem_xattr_security_list(struct inode *inode, char *list,
2020                                         size_t list_len, const char *name,
2021                                         size_t name_len)
2022 {
2023         return security_inode_listsecurity(inode, list, list_len);
2024 }
2025
2026 static int shmem_xattr_security_get(struct inode *inode, const char *name,
2027                                     void *buffer, size_t size)
2028 {
2029         if (strcmp(name, "") == 0)
2030                 return -EINVAL;
2031         return xattr_getsecurity(inode, name, buffer, size);
2032 }
2033
2034 static int shmem_xattr_security_set(struct inode *inode, const char *name,
2035                                     const void *value, size_t size, int flags)
2036 {
2037         if (strcmp(name, "") == 0)
2038                 return -EINVAL;
2039         return security_inode_setsecurity(inode, name, value, size, flags);
2040 }
2041
2042 static struct xattr_handler shmem_xattr_security_handler = {
2043         .prefix = XATTR_SECURITY_PREFIX,
2044         .list   = shmem_xattr_security_list,
2045         .get    = shmem_xattr_security_get,
2046         .set    = shmem_xattr_security_set,
2047 };
2048
2049 static struct xattr_handler *shmem_xattr_handlers[] = {
2050         &shmem_xattr_acl_access_handler,
2051         &shmem_xattr_acl_default_handler,
2052         &shmem_xattr_security_handler,
2053         NULL
2054 };
2055 #endif
2056
2057 static struct dentry *shmem_get_parent(struct dentry *child)
2058 {
2059         return ERR_PTR(-ESTALE);
2060 }
2061
2062 static int shmem_match(struct inode *ino, void *vfh)
2063 {
2064         __u32 *fh = vfh;
2065         __u64 inum = fh[2];
2066         inum = (inum << 32) | fh[1];
2067         return ino->i_ino == inum && fh[0] == ino->i_generation;
2068 }
2069
2070 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
2071                 struct fid *fid, int fh_len, int fh_type)
2072 {
2073         struct inode *inode;
2074         struct dentry *dentry = NULL;
2075         u64 inum = fid->raw[2];
2076         inum = (inum << 32) | fid->raw[1];
2077
2078         if (fh_len < 3)
2079                 return NULL;
2080
2081         inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
2082                         shmem_match, fid->raw);
2083         if (inode) {
2084                 dentry = d_find_alias(inode);
2085                 iput(inode);
2086         }
2087
2088         return dentry;
2089 }
2090
2091 static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len,
2092                                 int connectable)
2093 {
2094         struct inode *inode = dentry->d_inode;
2095
2096         if (*len < 3)
2097                 return 255;
2098
2099         if (hlist_unhashed(&inode->i_hash)) {
2100                 /* Unfortunately insert_inode_hash is not idempotent,
2101                  * so as we hash inodes here rather than at creation
2102                  * time, we need a lock to ensure we only try
2103                  * to do it once
2104                  */
2105                 static DEFINE_SPINLOCK(lock);
2106                 spin_lock(&lock);
2107                 if (hlist_unhashed(&inode->i_hash))
2108                         __insert_inode_hash(inode,
2109                                             inode->i_ino + inode->i_generation);
2110                 spin_unlock(&lock);
2111         }
2112
2113         fh[0] = inode->i_generation;
2114         fh[1] = inode->i_ino;
2115         fh[2] = ((__u64)inode->i_ino) >> 32;
2116
2117         *len = 3;
2118         return 1;
2119 }
2120
2121 static const struct export_operations shmem_export_ops = {
2122         .get_parent     = shmem_get_parent,
2123         .encode_fh      = shmem_encode_fh,
2124         .fh_to_dentry   = shmem_fh_to_dentry,
2125 };
2126
2127 static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
2128                                bool remount)
2129 {
2130         char *this_char, *value, *rest;
2131
2132         while (options != NULL) {
2133                 this_char = options;
2134                 for (;;) {
2135                         /*
2136                          * NUL-terminate this option: unfortunately,
2137                          * mount options form a comma-separated list,
2138                          * but mpol's nodelist may also contain commas.
2139                          */
2140                         options = strchr(options, ',');
2141                         if (options == NULL)
2142                                 break;
2143                         options++;
2144                         if (!isdigit(*options)) {
2145                                 options[-1] = '\0';
2146                                 break;
2147                         }
2148                 }
2149                 if (!*this_char)
2150                         continue;
2151                 if ((value = strchr(this_char,'=')) != NULL) {
2152                         *value++ = 0;
2153                 } else {
2154                         printk(KERN_ERR
2155                             "tmpfs: No value for mount option '%s'\n",
2156                             this_char);
2157                         return 1;
2158                 }
2159
2160                 if (!strcmp(this_char,"size")) {
2161                         unsigned long long size;
2162                         size = memparse(value,&rest);
2163                         if (*rest == '%') {
2164                                 size <<= PAGE_SHIFT;
2165                                 size *= totalram_pages;
2166                                 do_div(size, 100);
2167                                 rest++;
2168                         }
2169                         if (*rest)
2170                                 goto bad_val;
2171                         sbinfo->max_blocks =
2172                                 DIV_ROUND_UP(size, PAGE_CACHE_SIZE);
2173                 } else if (!strcmp(this_char,"nr_blocks")) {
2174                         sbinfo->max_blocks = memparse(value, &rest);
2175                         if (*rest)
2176                                 goto bad_val;
2177                 } else if (!strcmp(this_char,"nr_inodes")) {
2178                         sbinfo->max_inodes = memparse(value, &rest);
2179                         if (*rest)
2180                                 goto bad_val;
2181                 } else if (!strcmp(this_char,"mode")) {
2182                         if (remount)
2183                                 continue;
2184                         sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
2185                         if (*rest)
2186                                 goto bad_val;
2187                 } else if (!strcmp(this_char,"uid")) {
2188                         if (remount)
2189                                 continue;
2190                         sbinfo->uid = simple_strtoul(value, &rest, 0);
2191                         if (*rest)
2192                                 goto bad_val;
2193                 } else if (!strcmp(this_char,"gid")) {
2194                         if (remount)
2195                                 continue;
2196                         sbinfo->gid = simple_strtoul(value, &rest, 0);
2197                         if (*rest)
2198                                 goto bad_val;
2199                 } else if (!strcmp(this_char,"mpol")) {
2200                         if (mpol_parse_str(value, &sbinfo->mpol, 1))
2201                                 goto bad_val;
2202                 } else {
2203                         printk(KERN_ERR "tmpfs: Bad mount option %s\n",
2204                                this_char);
2205                         return 1;
2206                 }
2207         }
2208         return 0;
2209
2210 bad_val:
2211         printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
2212                value, this_char);
2213         return 1;
2214
2215 }
2216
2217 static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
2218 {
2219         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2220         struct shmem_sb_info config = *sbinfo;
2221         unsigned long blocks;
2222         unsigned long inodes;
2223         int error = -EINVAL;
2224
2225         if (shmem_parse_options(data, &config, true))
2226                 return error;
2227
2228         spin_lock(&sbinfo->stat_lock);
2229         blocks = sbinfo->max_blocks - sbinfo->free_blocks;
2230         inodes = sbinfo->max_inodes - sbinfo->free_inodes;
2231         if (config.max_blocks < blocks)
2232                 goto out;
2233         if (config.max_inodes < inodes)
2234                 goto out;
2235         /*
2236          * Those tests also disallow limited->unlimited while any are in
2237          * use, so i_blocks will always be zero when max_blocks is zero;
2238          * but we must separately disallow unlimited->limited, because
2239          * in that case we have no record of how much is already in use.
2240          */
2241         if (config.max_blocks && !sbinfo->max_blocks)
2242                 goto out;
2243         if (config.max_inodes && !sbinfo->max_inodes)
2244                 goto out;
2245
2246         error = 0;
2247         sbinfo->max_blocks  = config.max_blocks;
2248         sbinfo->free_blocks = config.max_blocks - blocks;
2249         sbinfo->max_inodes  = config.max_inodes;
2250         sbinfo->free_inodes = config.max_inodes - inodes;
2251
2252         mpol_put(sbinfo->mpol);
2253         sbinfo->mpol        = config.mpol;      /* transfers initial ref */
2254 out:
2255         spin_unlock(&sbinfo->stat_lock);
2256         return error;
2257 }
2258
2259 static int shmem_show_options(struct seq_file *seq, struct vfsmount *vfs)
2260 {
2261         struct shmem_sb_info *sbinfo = SHMEM_SB(vfs->mnt_sb);
2262
2263         if (sbinfo->max_blocks != shmem_default_max_blocks())
2264                 seq_printf(seq, ",size=%luk",
2265                         sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10));
2266         if (sbinfo->max_inodes != shmem_default_max_inodes())
2267                 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
2268         if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
2269                 seq_printf(seq, ",mode=%03o", sbinfo->mode);
2270         if (sbinfo->uid != 0)
2271                 seq_printf(seq, ",uid=%u", sbinfo->uid);
2272         if (sbinfo->gid != 0)
2273                 seq_printf(seq, ",gid=%u", sbinfo->gid);
2274         shmem_show_mpol(seq, sbinfo->mpol);
2275         return 0;
2276 }
2277 #endif /* CONFIG_TMPFS */
2278
2279 static void shmem_put_super(struct super_block *sb)
2280 {
2281         kfree(sb->s_fs_info);
2282         sb->s_fs_info = NULL;
2283 }
2284
2285 static int shmem_fill_super(struct super_block *sb,
2286                             void *data, int silent)
2287 {
2288         struct inode *inode;
2289         struct dentry *root;
2290         struct shmem_sb_info *sbinfo;
2291         int err = -ENOMEM;
2292
2293         /* Round up to L1_CACHE_BYTES to resist false sharing */
2294         sbinfo = kmalloc(max((int)sizeof(struct shmem_sb_info),
2295                                 L1_CACHE_BYTES), GFP_KERNEL);
2296         if (!sbinfo)
2297                 return -ENOMEM;
2298
2299         sbinfo->max_blocks = 0;
2300         sbinfo->max_inodes = 0;
2301         sbinfo->mode = S_IRWXUGO | S_ISVTX;
2302         sbinfo->uid = current_fsuid();
2303         sbinfo->gid = current_fsgid();
2304         sbinfo->mpol = NULL;
2305         sb->s_fs_info = sbinfo;
2306
2307 #ifdef CONFIG_TMPFS
2308         /*
2309          * Per default we only allow half of the physical ram per
2310          * tmpfs instance, limiting inodes to one per page of lowmem;
2311          * but the internal instance is left unlimited.
2312          */
2313         if (!(sb->s_flags & MS_NOUSER)) {
2314                 sbinfo->max_blocks = shmem_default_max_blocks();
2315                 sbinfo->max_inodes = shmem_default_max_inodes();
2316                 if (shmem_parse_options(data, sbinfo, false)) {
2317                         err = -EINVAL;
2318                         goto failed;
2319                 }
2320         }
2321         sb->s_export_op = &shmem_export_ops;
2322 #else
2323         sb->s_flags |= MS_NOUSER;
2324 #endif
2325
2326         spin_lock_init(&sbinfo->stat_lock);
2327         sbinfo->free_blocks = sbinfo->max_blocks;
2328         sbinfo->free_inodes = sbinfo->max_inodes;
2329
2330         sb->s_maxbytes = SHMEM_MAX_BYTES;
2331         sb->s_blocksize = PAGE_CACHE_SIZE;
2332         sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
2333         sb->s_magic = TMPFS_MAGIC;
2334         sb->s_op = &shmem_ops;
2335         sb->s_time_gran = 1;
2336 #ifdef CONFIG_TMPFS_POSIX_ACL
2337         sb->s_xattr = shmem_xattr_handlers;
2338         sb->s_flags |= MS_POSIXACL;
2339 #endif
2340
2341         inode = shmem_get_inode(sb, S_IFDIR | sbinfo->mode, 0);
2342         if (!inode)
2343                 goto failed;
2344         inode->i_uid = sbinfo->uid;
2345         inode->i_gid = sbinfo->gid;
2346         root = d_alloc_root(inode);
2347         if (!root)
2348                 goto failed_iput;
2349         sb->s_root = root;
2350         return 0;
2351
2352 failed_iput:
2353         iput(inode);
2354 failed:
2355         shmem_put_super(sb);
2356         return err;
2357 }
2358
2359 static struct kmem_cache *shmem_inode_cachep;
2360
2361 static struct inode *shmem_alloc_inode(struct super_block *sb)
2362 {
2363         struct shmem_inode_info *p;
2364         p = (struct shmem_inode_info *)kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
2365         if (!p)
2366                 return NULL;
2367         return &p->vfs_inode;
2368 }
2369
2370 static void shmem_destroy_inode(struct inode *inode)
2371 {
2372         if ((inode->i_mode & S_IFMT) == S_IFREG) {
2373                 /* only struct inode is valid if it's an inline symlink */
2374                 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
2375         }
2376         shmem_acl_destroy_inode(inode);
2377         kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
2378 }
2379
2380 static void init_once(void *foo)
2381 {
2382         struct shmem_inode_info *p = (struct shmem_inode_info *) foo;
2383
2384         inode_init_once(&p->vfs_inode);
2385 #ifdef CONFIG_TMPFS_POSIX_ACL
2386         p->i_acl = NULL;
2387         p->i_default_acl = NULL;
2388 #endif
2389 }
2390
2391 static int init_inodecache(void)
2392 {
2393         shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
2394                                 sizeof(struct shmem_inode_info),
2395                                 0, SLAB_PANIC, init_once);
2396         return 0;
2397 }
2398
2399 static void destroy_inodecache(void)
2400 {
2401         kmem_cache_destroy(shmem_inode_cachep);
2402 }
2403
2404 static const struct address_space_operations shmem_aops = {
2405         .writepage      = shmem_writepage,
2406         .set_page_dirty = __set_page_dirty_no_writeback,
2407 #ifdef CONFIG_TMPFS
2408         .readpage       = shmem_readpage,
2409         .write_begin    = shmem_write_begin,
2410         .write_end      = shmem_write_end,
2411 #endif
2412         .migratepage    = migrate_page,
2413 };
2414
2415 static const struct file_operations shmem_file_operations = {
2416         .mmap           = shmem_mmap,
2417 #ifdef CONFIG_TMPFS
2418         .llseek         = generic_file_llseek,
2419         .read           = do_sync_read,
2420         .write          = do_sync_write,
2421         .aio_read       = shmem_file_aio_read,
2422         .aio_write      = generic_file_aio_write,
2423         .fsync          = simple_sync_file,
2424         .splice_read    = generic_file_splice_read,
2425         .splice_write   = generic_file_splice_write,
2426 #endif
2427 };
2428
2429 static const struct inode_operations shmem_inode_operations = {
2430         .truncate       = shmem_truncate,
2431         .setattr        = shmem_notify_change,
2432         .truncate_range = shmem_truncate_range,
2433 #ifdef CONFIG_TMPFS_POSIX_ACL
2434         .setxattr       = generic_setxattr,
2435         .getxattr       = generic_getxattr,
2436         .listxattr      = generic_listxattr,
2437         .removexattr    = generic_removexattr,
2438         .permission     = shmem_permission,
2439 #endif
2440
2441 };
2442
2443 static const struct inode_operations shmem_dir_inode_operations = {
2444 #ifdef CONFIG_TMPFS
2445         .create         = shmem_create,
2446         .lookup         = simple_lookup,
2447         .link           = shmem_link,
2448         .unlink         = shmem_unlink,
2449         .symlink        = shmem_symlink,
2450         .mkdir          = shmem_mkdir,
2451         .rmdir          = shmem_rmdir,
2452         .mknod          = shmem_mknod,
2453         .rename         = shmem_rename,
2454 #endif
2455 #ifdef CONFIG_TMPFS_POSIX_ACL
2456         .setattr        = shmem_notify_change,
2457         .setxattr       = generic_setxattr,
2458         .getxattr       = generic_getxattr,
2459         .listxattr      = generic_listxattr,
2460         .removexattr    = generic_removexattr,
2461         .permission     = shmem_permission,
2462 #endif
2463 };
2464
2465 static const struct inode_operations shmem_special_inode_operations = {
2466 #ifdef CONFIG_TMPFS_POSIX_ACL
2467         .setattr        = shmem_notify_change,
2468         .setxattr       = generic_setxattr,
2469         .getxattr       = generic_getxattr,
2470         .listxattr      = generic_listxattr,
2471         .removexattr    = generic_removexattr,
2472         .permission     = shmem_permission,
2473 #endif
2474 };
2475
2476 static const struct super_operations shmem_ops = {
2477         .alloc_inode    = shmem_alloc_inode,
2478         .destroy_inode  = shmem_destroy_inode,
2479 #ifdef CONFIG_TMPFS
2480         .statfs         = shmem_statfs,
2481         .remount_fs     = shmem_remount_fs,
2482         .show_options   = shmem_show_options,
2483 #endif
2484         .delete_inode   = shmem_delete_inode,
2485         .drop_inode     = generic_delete_inode,
2486         .put_super      = shmem_put_super,
2487 };
2488
2489 static struct vm_operations_struct shmem_vm_ops = {
2490         .fault          = shmem_fault,
2491 #ifdef CONFIG_NUMA
2492         .set_policy     = shmem_set_policy,
2493         .get_policy     = shmem_get_policy,
2494 #endif
2495 };
2496
2497
2498 static int shmem_get_sb(struct file_system_type *fs_type,
2499         int flags, const char *dev_name, void *data, struct vfsmount *mnt)
2500 {
2501         return get_sb_nodev(fs_type, flags, data, shmem_fill_super, mnt);
2502 }
2503
2504 static struct file_system_type tmpfs_fs_type = {
2505         .owner          = THIS_MODULE,
2506         .name           = "tmpfs",
2507         .get_sb         = shmem_get_sb,
2508         .kill_sb        = kill_litter_super,
2509 };
2510
2511 static int __init init_tmpfs(void)
2512 {
2513         int error;
2514
2515         error = bdi_init(&shmem_backing_dev_info);
2516         if (error)
2517                 goto out4;
2518
2519         error = init_inodecache();
2520         if (error)
2521                 goto out3;
2522
2523         error = register_filesystem(&tmpfs_fs_type);
2524         if (error) {
2525                 printk(KERN_ERR "Could not register tmpfs\n");
2526                 goto out2;
2527         }
2528
2529         shm_mnt = vfs_kern_mount(&tmpfs_fs_type, MS_NOUSER,
2530                                 tmpfs_fs_type.name, NULL);
2531         if (IS_ERR(shm_mnt)) {
2532                 error = PTR_ERR(shm_mnt);
2533                 printk(KERN_ERR "Could not kern_mount tmpfs\n");
2534                 goto out1;
2535         }
2536         return 0;
2537
2538 out1:
2539         unregister_filesystem(&tmpfs_fs_type);
2540 out2:
2541         destroy_inodecache();
2542 out3:
2543         bdi_destroy(&shmem_backing_dev_info);
2544 out4:
2545         shm_mnt = ERR_PTR(error);
2546         return error;
2547 }
2548
2549 #else /* !CONFIG_SHMEM */
2550
2551 /*
2552  * tiny-shmem: simple shmemfs and tmpfs using ramfs code
2553  *
2554  * This is intended for small system where the benefits of the full
2555  * shmem code (swap-backed and resource-limited) are outweighed by
2556  * their complexity. On systems without swap this code should be
2557  * effectively equivalent, but much lighter weight.
2558  */
2559
2560 #include <linux/ramfs.h>
2561
2562 static struct file_system_type tmpfs_fs_type = {
2563         .name           = "tmpfs",
2564         .get_sb         = ramfs_get_sb,
2565         .kill_sb        = kill_litter_super,
2566 };
2567
2568 static int __init init_tmpfs(void)
2569 {
2570         BUG_ON(register_filesystem(&tmpfs_fs_type) != 0);
2571
2572         shm_mnt = kern_mount(&tmpfs_fs_type);
2573         BUG_ON(IS_ERR(shm_mnt));
2574
2575         return 0;
2576 }
2577
2578 int shmem_unuse(swp_entry_t entry, struct page *page)
2579 {
2580         return 0;
2581 }
2582
2583 #define shmem_file_operations ramfs_file_operations
2584 #define shmem_vm_ops generic_file_vm_ops
2585 #define shmem_get_inode ramfs_get_inode
2586 #define shmem_acct_size(a, b) 0
2587 #define shmem_unacct_size(a, b) do {} while (0)
2588 #define SHMEM_MAX_BYTES LLONG_MAX
2589
2590 #endif /* CONFIG_SHMEM */
2591
2592 /* common code */
2593
2594 /**
2595  * shmem_file_setup - get an unlinked file living in tmpfs
2596  * @name: name for dentry (to be seen in /proc/<pid>/maps
2597  * @size: size to be set for the file
2598  * @flags: vm_flags
2599  */
2600 struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags)
2601 {
2602         int error;
2603         struct file *file;
2604         struct inode *inode;
2605         struct dentry *dentry, *root;
2606         struct qstr this;
2607
2608         if (IS_ERR(shm_mnt))
2609                 return (void *)shm_mnt;
2610
2611         if (size < 0 || size > SHMEM_MAX_BYTES)
2612                 return ERR_PTR(-EINVAL);
2613
2614         if (shmem_acct_size(flags, size))
2615                 return ERR_PTR(-ENOMEM);
2616
2617         error = -ENOMEM;
2618         this.name = name;
2619         this.len = strlen(name);
2620         this.hash = 0; /* will go */
2621         root = shm_mnt->mnt_root;
2622         dentry = d_alloc(root, &this);
2623         if (!dentry)
2624                 goto put_memory;
2625
2626         error = -ENFILE;
2627         file = get_empty_filp();
2628         if (!file)
2629                 goto put_dentry;
2630
2631         error = -ENOSPC;
2632         inode = shmem_get_inode(root->d_sb, S_IFREG | S_IRWXUGO, 0);
2633         if (!inode)
2634                 goto close_file;
2635
2636 #ifdef CONFIG_SHMEM
2637         SHMEM_I(inode)->flags = flags & VM_ACCOUNT;
2638 #endif
2639         d_instantiate(dentry, inode);
2640         inode->i_size = size;
2641         inode->i_nlink = 0;     /* It is unlinked */
2642         init_file(file, shm_mnt, dentry, FMODE_WRITE | FMODE_READ,
2643                   &shmem_file_operations);
2644
2645 #ifndef CONFIG_MMU
2646         error = ramfs_nommu_expand_for_mapping(inode, size);
2647         if (error)
2648                 goto close_file;
2649 #endif
2650         return file;
2651
2652 close_file:
2653         put_filp(file);
2654 put_dentry:
2655         dput(dentry);
2656 put_memory:
2657         shmem_unacct_size(flags, size);
2658         return ERR_PTR(error);
2659 }
2660 EXPORT_SYMBOL_GPL(shmem_file_setup);
2661
2662 /**
2663  * shmem_zero_setup - setup a shared anonymous mapping
2664  * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
2665  */
2666 int shmem_zero_setup(struct vm_area_struct *vma)
2667 {
2668         struct file *file;
2669         loff_t size = vma->vm_end - vma->vm_start;
2670
2671         file = shmem_file_setup("dev/zero", size, vma->vm_flags);
2672         if (IS_ERR(file))
2673                 return PTR_ERR(file);
2674
2675         if (vma->vm_file)
2676                 fput(vma->vm_file);
2677         vma->vm_file = file;
2678         vma->vm_ops = &shmem_vm_ops;
2679         return 0;
2680 }
2681
2682 module_init(init_tmpfs)