tmpfs: convert mem_cgroup shmem to radix-swap
[linux-2.6.git] / mm / shmem.c
1 /*
2  * Resizable virtual memory filesystem for Linux.
3  *
4  * Copyright (C) 2000 Linus Torvalds.
5  *               2000 Transmeta Corp.
6  *               2000-2001 Christoph Rohland
7  *               2000-2001 SAP AG
8  *               2002 Red Hat Inc.
9  * Copyright (C) 2002-2005 Hugh Dickins.
10  * Copyright (C) 2002-2005 VERITAS Software Corporation.
11  * Copyright (C) 2004 Andi Kleen, SuSE Labs
12  *
13  * Extended attribute support for tmpfs:
14  * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
15  * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
16  *
17  * tiny-shmem:
18  * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
19  *
20  * This file is released under the GPL.
21  */
22
23 #include <linux/fs.h>
24 #include <linux/init.h>
25 #include <linux/vfs.h>
26 #include <linux/mount.h>
27 #include <linux/pagemap.h>
28 #include <linux/file.h>
29 #include <linux/mm.h>
30 #include <linux/module.h>
31 #include <linux/swap.h>
32
33 static struct vfsmount *shm_mnt;
34
35 #ifdef CONFIG_SHMEM
36 /*
37  * This virtual memory filesystem is heavily based on the ramfs. It
38  * extends ramfs by the ability to use swap and honor resource limits
39  * which makes it a completely usable filesystem.
40  */
41
42 #include <linux/xattr.h>
43 #include <linux/exportfs.h>
44 #include <linux/posix_acl.h>
45 #include <linux/generic_acl.h>
46 #include <linux/mman.h>
47 #include <linux/string.h>
48 #include <linux/slab.h>
49 #include <linux/backing-dev.h>
50 #include <linux/shmem_fs.h>
51 #include <linux/writeback.h>
52 #include <linux/blkdev.h>
53 #include <linux/pagevec.h>
54 #include <linux/percpu_counter.h>
55 #include <linux/splice.h>
56 #include <linux/security.h>
57 #include <linux/swapops.h>
58 #include <linux/mempolicy.h>
59 #include <linux/namei.h>
60 #include <linux/ctype.h>
61 #include <linux/migrate.h>
62 #include <linux/highmem.h>
63 #include <linux/seq_file.h>
64 #include <linux/magic.h>
65
66 #include <asm/uaccess.h>
67 #include <asm/pgtable.h>
68
69 #define BLOCKS_PER_PAGE  (PAGE_CACHE_SIZE/512)
70 #define VM_ACCT(size)    (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
71
72 /* Pretend that each entry is of this size in directory's i_size */
73 #define BOGO_DIRENT_SIZE 20
74
75 struct shmem_xattr {
76         struct list_head list;  /* anchored by shmem_inode_info->xattr_list */
77         char *name;             /* xattr name */
78         size_t size;
79         char value[0];
80 };
81
82 /* Flag allocation requirements to shmem_getpage */
83 enum sgp_type {
84         SGP_READ,       /* don't exceed i_size, don't allocate page */
85         SGP_CACHE,      /* don't exceed i_size, may allocate page */
86         SGP_DIRTY,      /* like SGP_CACHE, but set new page dirty */
87         SGP_WRITE,      /* may exceed i_size, may allocate page */
88 };
89
90 #ifdef CONFIG_TMPFS
91 static unsigned long shmem_default_max_blocks(void)
92 {
93         return totalram_pages / 2;
94 }
95
96 static unsigned long shmem_default_max_inodes(void)
97 {
98         return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
99 }
100 #endif
101
102 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
103         struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type);
104
105 static inline int shmem_getpage(struct inode *inode, pgoff_t index,
106         struct page **pagep, enum sgp_type sgp, int *fault_type)
107 {
108         return shmem_getpage_gfp(inode, index, pagep, sgp,
109                         mapping_gfp_mask(inode->i_mapping), fault_type);
110 }
111
112 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
113 {
114         return sb->s_fs_info;
115 }
116
117 /*
118  * shmem_file_setup pre-accounts the whole fixed size of a VM object,
119  * for shared memory and for shared anonymous (/dev/zero) mappings
120  * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
121  * consistent with the pre-accounting of private mappings ...
122  */
123 static inline int shmem_acct_size(unsigned long flags, loff_t size)
124 {
125         return (flags & VM_NORESERVE) ?
126                 0 : security_vm_enough_memory_kern(VM_ACCT(size));
127 }
128
129 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
130 {
131         if (!(flags & VM_NORESERVE))
132                 vm_unacct_memory(VM_ACCT(size));
133 }
134
135 /*
136  * ... whereas tmpfs objects are accounted incrementally as
137  * pages are allocated, in order to allow huge sparse files.
138  * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
139  * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
140  */
141 static inline int shmem_acct_block(unsigned long flags)
142 {
143         return (flags & VM_NORESERVE) ?
144                 security_vm_enough_memory_kern(VM_ACCT(PAGE_CACHE_SIZE)) : 0;
145 }
146
147 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
148 {
149         if (flags & VM_NORESERVE)
150                 vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
151 }
152
153 static const struct super_operations shmem_ops;
154 static const struct address_space_operations shmem_aops;
155 static const struct file_operations shmem_file_operations;
156 static const struct inode_operations shmem_inode_operations;
157 static const struct inode_operations shmem_dir_inode_operations;
158 static const struct inode_operations shmem_special_inode_operations;
159 static const struct vm_operations_struct shmem_vm_ops;
160
161 static struct backing_dev_info shmem_backing_dev_info  __read_mostly = {
162         .ra_pages       = 0,    /* No readahead */
163         .capabilities   = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED,
164 };
165
166 static LIST_HEAD(shmem_swaplist);
167 static DEFINE_MUTEX(shmem_swaplist_mutex);
168
169 static int shmem_reserve_inode(struct super_block *sb)
170 {
171         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
172         if (sbinfo->max_inodes) {
173                 spin_lock(&sbinfo->stat_lock);
174                 if (!sbinfo->free_inodes) {
175                         spin_unlock(&sbinfo->stat_lock);
176                         return -ENOSPC;
177                 }
178                 sbinfo->free_inodes--;
179                 spin_unlock(&sbinfo->stat_lock);
180         }
181         return 0;
182 }
183
184 static void shmem_free_inode(struct super_block *sb)
185 {
186         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
187         if (sbinfo->max_inodes) {
188                 spin_lock(&sbinfo->stat_lock);
189                 sbinfo->free_inodes++;
190                 spin_unlock(&sbinfo->stat_lock);
191         }
192 }
193
194 /**
195  * shmem_recalc_inode - recalculate the block usage of an inode
196  * @inode: inode to recalc
197  *
198  * We have to calculate the free blocks since the mm can drop
199  * undirtied hole pages behind our back.
200  *
201  * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
202  * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
203  *
204  * It has to be called with the spinlock held.
205  */
206 static void shmem_recalc_inode(struct inode *inode)
207 {
208         struct shmem_inode_info *info = SHMEM_I(inode);
209         long freed;
210
211         freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
212         if (freed > 0) {
213                 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
214                 if (sbinfo->max_blocks)
215                         percpu_counter_add(&sbinfo->used_blocks, -freed);
216                 info->alloced -= freed;
217                 inode->i_blocks -= freed * BLOCKS_PER_PAGE;
218                 shmem_unacct_blocks(info->flags, freed);
219         }
220 }
221
222 static void shmem_put_swap(struct shmem_inode_info *info, pgoff_t index,
223                            swp_entry_t swap)
224 {
225         if (index < SHMEM_NR_DIRECT)
226                 info->i_direct[index] = swap;
227 }
228
229 static swp_entry_t shmem_get_swap(struct shmem_inode_info *info, pgoff_t index)
230 {
231         return (index < SHMEM_NR_DIRECT) ?
232                 info->i_direct[index] : (swp_entry_t){0};
233 }
234
235 /*
236  * Replace item expected in radix tree by a new item, while holding tree lock.
237  */
238 static int shmem_radix_tree_replace(struct address_space *mapping,
239                         pgoff_t index, void *expected, void *replacement)
240 {
241         void **pslot;
242         void *item = NULL;
243
244         VM_BUG_ON(!expected);
245         pslot = radix_tree_lookup_slot(&mapping->page_tree, index);
246         if (pslot)
247                 item = radix_tree_deref_slot_protected(pslot,
248                                                         &mapping->tree_lock);
249         if (item != expected)
250                 return -ENOENT;
251         if (replacement)
252                 radix_tree_replace_slot(pslot, replacement);
253         else
254                 radix_tree_delete(&mapping->page_tree, index);
255         return 0;
256 }
257
258 /*
259  * Like add_to_page_cache_locked, but error if expected item has gone.
260  */
261 static int shmem_add_to_page_cache(struct page *page,
262                                    struct address_space *mapping,
263                                    pgoff_t index, gfp_t gfp, void *expected)
264 {
265         int error = 0;
266
267         VM_BUG_ON(!PageLocked(page));
268         VM_BUG_ON(!PageSwapBacked(page));
269
270         if (!expected)
271                 error = radix_tree_preload(gfp & GFP_RECLAIM_MASK);
272         if (!error) {
273                 page_cache_get(page);
274                 page->mapping = mapping;
275                 page->index = index;
276
277                 spin_lock_irq(&mapping->tree_lock);
278                 if (!expected)
279                         error = radix_tree_insert(&mapping->page_tree,
280                                                         index, page);
281                 else
282                         error = shmem_radix_tree_replace(mapping, index,
283                                                         expected, page);
284                 if (!error) {
285                         mapping->nrpages++;
286                         __inc_zone_page_state(page, NR_FILE_PAGES);
287                         __inc_zone_page_state(page, NR_SHMEM);
288                         spin_unlock_irq(&mapping->tree_lock);
289                 } else {
290                         page->mapping = NULL;
291                         spin_unlock_irq(&mapping->tree_lock);
292                         page_cache_release(page);
293                 }
294                 if (!expected)
295                         radix_tree_preload_end();
296         }
297         if (error)
298                 mem_cgroup_uncharge_cache_page(page);
299         return error;
300 }
301
302 /*
303  * Like find_get_pages, but collecting swap entries as well as pages.
304  */
305 static unsigned shmem_find_get_pages_and_swap(struct address_space *mapping,
306                                         pgoff_t start, unsigned int nr_pages,
307                                         struct page **pages, pgoff_t *indices)
308 {
309         unsigned int i;
310         unsigned int ret;
311         unsigned int nr_found;
312
313         rcu_read_lock();
314 restart:
315         nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,
316                                 (void ***)pages, indices, start, nr_pages);
317         ret = 0;
318         for (i = 0; i < nr_found; i++) {
319                 struct page *page;
320 repeat:
321                 page = radix_tree_deref_slot((void **)pages[i]);
322                 if (unlikely(!page))
323                         continue;
324                 if (radix_tree_exception(page)) {
325                         if (radix_tree_exceptional_entry(page))
326                                 goto export;
327                         /* radix_tree_deref_retry(page) */
328                         goto restart;
329                 }
330                 if (!page_cache_get_speculative(page))
331                         goto repeat;
332
333                 /* Has the page moved? */
334                 if (unlikely(page != *((void **)pages[i]))) {
335                         page_cache_release(page);
336                         goto repeat;
337                 }
338 export:
339                 indices[ret] = indices[i];
340                 pages[ret] = page;
341                 ret++;
342         }
343         if (unlikely(!ret && nr_found))
344                 goto restart;
345         rcu_read_unlock();
346         return ret;
347 }
348
349 /*
350  * Lockless lookup of swap entry in radix tree, avoiding refcount on pages.
351  */
352 static pgoff_t shmem_find_swap(struct address_space *mapping, void *radswap)
353 {
354         void  **slots[PAGEVEC_SIZE];
355         pgoff_t indices[PAGEVEC_SIZE];
356         unsigned int nr_found;
357
358 restart:
359         nr_found = 1;
360         indices[0] = -1;
361         while (nr_found) {
362                 pgoff_t index = indices[nr_found - 1] + 1;
363                 unsigned int i;
364
365                 rcu_read_lock();
366                 nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,
367                                         slots, indices, index, PAGEVEC_SIZE);
368                 for (i = 0; i < nr_found; i++) {
369                         void *item = radix_tree_deref_slot(slots[i]);
370                         if (radix_tree_deref_retry(item)) {
371                                 rcu_read_unlock();
372                                 goto restart;
373                         }
374                         if (item == radswap) {
375                                 rcu_read_unlock();
376                                 return indices[i];
377                         }
378                 }
379                 rcu_read_unlock();
380                 cond_resched();
381         }
382         return -1;
383 }
384
385 /*
386  * Remove swap entry from radix tree, free the swap and its page cache.
387  */
388 static int shmem_free_swap(struct address_space *mapping,
389                            pgoff_t index, void *radswap)
390 {
391         int error;
392
393         spin_lock_irq(&mapping->tree_lock);
394         error = shmem_radix_tree_replace(mapping, index, radswap, NULL);
395         spin_unlock_irq(&mapping->tree_lock);
396         if (!error)
397                 free_swap_and_cache(radix_to_swp_entry(radswap));
398         return error;
399 }
400
401 /*
402  * Pagevec may contain swap entries, so shuffle up pages before releasing.
403  */
404 static void shmem_pagevec_release(struct pagevec *pvec)
405 {
406         int i, j;
407
408         for (i = 0, j = 0; i < pagevec_count(pvec); i++) {
409                 struct page *page = pvec->pages[i];
410                 if (!radix_tree_exceptional_entry(page))
411                         pvec->pages[j++] = page;
412         }
413         pvec->nr = j;
414         pagevec_release(pvec);
415 }
416
417 /*
418  * Remove range of pages and swap entries from radix tree, and free them.
419  */
420 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
421 {
422         struct address_space *mapping = inode->i_mapping;
423         struct shmem_inode_info *info = SHMEM_I(inode);
424         pgoff_t start = (lstart + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
425         unsigned partial = lstart & (PAGE_CACHE_SIZE - 1);
426         pgoff_t end = (lend >> PAGE_CACHE_SHIFT);
427         struct pagevec pvec;
428         pgoff_t indices[PAGEVEC_SIZE];
429         long nr_swaps_freed = 0;
430         pgoff_t index;
431         int i;
432
433         BUG_ON((lend & (PAGE_CACHE_SIZE - 1)) != (PAGE_CACHE_SIZE - 1));
434
435         pagevec_init(&pvec, 0);
436         index = start;
437         while (index <= end) {
438                 pvec.nr = shmem_find_get_pages_and_swap(mapping, index,
439                         min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
440                                                         pvec.pages, indices);
441                 if (!pvec.nr)
442                         break;
443                 mem_cgroup_uncharge_start();
444                 for (i = 0; i < pagevec_count(&pvec); i++) {
445                         struct page *page = pvec.pages[i];
446
447                         index = indices[i];
448                         if (index > end)
449                                 break;
450
451                         if (radix_tree_exceptional_entry(page)) {
452                                 nr_swaps_freed += !shmem_free_swap(mapping,
453                                                                 index, page);
454                                 continue;
455                         }
456
457                         if (!trylock_page(page))
458                                 continue;
459                         if (page->mapping == mapping) {
460                                 VM_BUG_ON(PageWriteback(page));
461                                 truncate_inode_page(mapping, page);
462                         }
463                         unlock_page(page);
464                 }
465                 shmem_pagevec_release(&pvec);
466                 mem_cgroup_uncharge_end();
467                 cond_resched();
468                 index++;
469         }
470
471         if (partial) {
472                 struct page *page = NULL;
473                 shmem_getpage(inode, start - 1, &page, SGP_READ, NULL);
474                 if (page) {
475                         zero_user_segment(page, partial, PAGE_CACHE_SIZE);
476                         set_page_dirty(page);
477                         unlock_page(page);
478                         page_cache_release(page);
479                 }
480         }
481
482         index = start;
483         for ( ; ; ) {
484                 cond_resched();
485                 pvec.nr = shmem_find_get_pages_and_swap(mapping, index,
486                         min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
487                                                         pvec.pages, indices);
488                 if (!pvec.nr) {
489                         if (index == start)
490                                 break;
491                         index = start;
492                         continue;
493                 }
494                 if (index == start && indices[0] > end) {
495                         shmem_pagevec_release(&pvec);
496                         break;
497                 }
498                 mem_cgroup_uncharge_start();
499                 for (i = 0; i < pagevec_count(&pvec); i++) {
500                         struct page *page = pvec.pages[i];
501
502                         index = indices[i];
503                         if (index > end)
504                                 break;
505
506                         if (radix_tree_exceptional_entry(page)) {
507                                 nr_swaps_freed += !shmem_free_swap(mapping,
508                                                                 index, page);
509                                 continue;
510                         }
511
512                         lock_page(page);
513                         if (page->mapping == mapping) {
514                                 VM_BUG_ON(PageWriteback(page));
515                                 truncate_inode_page(mapping, page);
516                         }
517                         unlock_page(page);
518                 }
519                 shmem_pagevec_release(&pvec);
520                 mem_cgroup_uncharge_end();
521                 index++;
522         }
523
524         spin_lock(&info->lock);
525         info->swapped -= nr_swaps_freed;
526         shmem_recalc_inode(inode);
527         spin_unlock(&info->lock);
528
529         inode->i_ctime = inode->i_mtime = CURRENT_TIME;
530 }
531 EXPORT_SYMBOL_GPL(shmem_truncate_range);
532
533 static int shmem_setattr(struct dentry *dentry, struct iattr *attr)
534 {
535         struct inode *inode = dentry->d_inode;
536         int error;
537
538         error = inode_change_ok(inode, attr);
539         if (error)
540                 return error;
541
542         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
543                 loff_t oldsize = inode->i_size;
544                 loff_t newsize = attr->ia_size;
545
546                 if (newsize != oldsize) {
547                         i_size_write(inode, newsize);
548                         inode->i_ctime = inode->i_mtime = CURRENT_TIME;
549                 }
550                 if (newsize < oldsize) {
551                         loff_t holebegin = round_up(newsize, PAGE_SIZE);
552                         unmap_mapping_range(inode->i_mapping, holebegin, 0, 1);
553                         shmem_truncate_range(inode, newsize, (loff_t)-1);
554                         /* unmap again to remove racily COWed private pages */
555                         unmap_mapping_range(inode->i_mapping, holebegin, 0, 1);
556                 }
557         }
558
559         setattr_copy(inode, attr);
560 #ifdef CONFIG_TMPFS_POSIX_ACL
561         if (attr->ia_valid & ATTR_MODE)
562                 error = generic_acl_chmod(inode);
563 #endif
564         return error;
565 }
566
567 static void shmem_evict_inode(struct inode *inode)
568 {
569         struct shmem_inode_info *info = SHMEM_I(inode);
570         struct shmem_xattr *xattr, *nxattr;
571
572         if (inode->i_mapping->a_ops == &shmem_aops) {
573                 shmem_unacct_size(info->flags, inode->i_size);
574                 inode->i_size = 0;
575                 shmem_truncate_range(inode, 0, (loff_t)-1);
576                 if (!list_empty(&info->swaplist)) {
577                         mutex_lock(&shmem_swaplist_mutex);
578                         list_del_init(&info->swaplist);
579                         mutex_unlock(&shmem_swaplist_mutex);
580                 }
581         }
582
583         list_for_each_entry_safe(xattr, nxattr, &info->xattr_list, list) {
584                 kfree(xattr->name);
585                 kfree(xattr);
586         }
587         BUG_ON(inode->i_blocks);
588         shmem_free_inode(inode->i_sb);
589         end_writeback(inode);
590 }
591
592 /*
593  * If swap found in inode, free it and move page from swapcache to filecache.
594  */
595 static int shmem_unuse_inode(struct shmem_inode_info *info,
596                              swp_entry_t swap, struct page *page)
597 {
598         struct address_space *mapping = info->vfs_inode.i_mapping;
599         void *radswap;
600         pgoff_t index;
601         int error;
602
603         radswap = swp_to_radix_entry(swap);
604         index = shmem_find_swap(mapping, radswap);
605         if (index == -1)
606                 return 0;
607
608         /*
609          * Move _head_ to start search for next from here.
610          * But be careful: shmem_evict_inode checks list_empty without taking
611          * mutex, and there's an instant in list_move_tail when info->swaplist
612          * would appear empty, if it were the only one on shmem_swaplist.
613          */
614         if (shmem_swaplist.next != &info->swaplist)
615                 list_move_tail(&shmem_swaplist, &info->swaplist);
616
617         /*
618          * We rely on shmem_swaplist_mutex, not only to protect the swaplist,
619          * but also to hold up shmem_evict_inode(): so inode cannot be freed
620          * beneath us (pagelock doesn't help until the page is in pagecache).
621          */
622         error = shmem_add_to_page_cache(page, mapping, index,
623                                                 GFP_NOWAIT, radswap);
624         /* which does mem_cgroup_uncharge_cache_page on error */
625
626         if (error != -ENOMEM) {
627                 /*
628                  * Truncation and eviction use free_swap_and_cache(), which
629                  * only does trylock page: if we raced, best clean up here.
630                  */
631                 delete_from_swap_cache(page);
632                 set_page_dirty(page);
633                 if (!error) {
634                         spin_lock(&info->lock);
635                         info->swapped--;
636                         spin_unlock(&info->lock);
637                         swap_free(swap);
638                 }
639                 error = 1;      /* not an error, but entry was found */
640         }
641         return error;
642 }
643
644 /*
645  * Search through swapped inodes to find and replace swap by page.
646  */
647 int shmem_unuse(swp_entry_t swap, struct page *page)
648 {
649         struct list_head *this, *next;
650         struct shmem_inode_info *info;
651         int found = 0;
652         int error;
653
654         /*
655          * Charge page using GFP_KERNEL while we can wait, before taking
656          * the shmem_swaplist_mutex which might hold up shmem_writepage().
657          * Charged back to the user (not to caller) when swap account is used.
658          */
659         error = mem_cgroup_cache_charge(page, current->mm, GFP_KERNEL);
660         if (error)
661                 goto out;
662         /* No radix_tree_preload: swap entry keeps a place for page in tree */
663
664         mutex_lock(&shmem_swaplist_mutex);
665         list_for_each_safe(this, next, &shmem_swaplist) {
666                 info = list_entry(this, struct shmem_inode_info, swaplist);
667                 if (!info->swapped) {
668                         spin_lock(&info->lock);
669                         if (!info->swapped)
670                                 list_del_init(&info->swaplist);
671                         spin_unlock(&info->lock);
672                 }
673                 if (info->swapped)
674                         found = shmem_unuse_inode(info, swap, page);
675                 cond_resched();
676                 if (found)
677                         break;
678         }
679         mutex_unlock(&shmem_swaplist_mutex);
680
681         if (!found)
682                 mem_cgroup_uncharge_cache_page(page);
683         if (found < 0)
684                 error = found;
685 out:
686         unlock_page(page);
687         page_cache_release(page);
688         return error;
689 }
690
691 /*
692  * Move the page from the page cache to the swap cache.
693  */
694 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
695 {
696         struct shmem_inode_info *info;
697         swp_entry_t swap, oswap;
698         struct address_space *mapping;
699         pgoff_t index;
700         struct inode *inode;
701
702         BUG_ON(!PageLocked(page));
703         mapping = page->mapping;
704         index = page->index;
705         inode = mapping->host;
706         info = SHMEM_I(inode);
707         if (info->flags & VM_LOCKED)
708                 goto redirty;
709         if (!total_swap_pages)
710                 goto redirty;
711
712         /*
713          * shmem_backing_dev_info's capabilities prevent regular writeback or
714          * sync from ever calling shmem_writepage; but a stacking filesystem
715          * might use ->writepage of its underlying filesystem, in which case
716          * tmpfs should write out to swap only in response to memory pressure,
717          * and not for the writeback threads or sync.
718          */
719         if (!wbc->for_reclaim) {
720                 WARN_ON_ONCE(1);        /* Still happens? Tell us about it! */
721                 goto redirty;
722         }
723
724         /*
725          * Disable even the toy swapping implementation, while we convert
726          * functions one by one to having swap entries in the radix tree.
727          */
728         if (index < ULONG_MAX)
729                 goto redirty;
730
731         swap = get_swap_page();
732         if (!swap.val)
733                 goto redirty;
734
735         /*
736          * Add inode to shmem_unuse()'s list of swapped-out inodes,
737          * if it's not already there.  Do it now because we cannot take
738          * mutex while holding spinlock, and must do so before the page
739          * is moved to swap cache, when its pagelock no longer protects
740          * the inode from eviction.  But don't unlock the mutex until
741          * we've taken the spinlock, because shmem_unuse_inode() will
742          * prune a !swapped inode from the swaplist under both locks.
743          */
744         mutex_lock(&shmem_swaplist_mutex);
745         if (list_empty(&info->swaplist))
746                 list_add_tail(&info->swaplist, &shmem_swaplist);
747
748         spin_lock(&info->lock);
749         mutex_unlock(&shmem_swaplist_mutex);
750
751         oswap = shmem_get_swap(info, index);
752         if (oswap.val) {
753                 WARN_ON_ONCE(1);        /* Still happens? Tell us about it! */
754                 free_swap_and_cache(oswap);
755                 shmem_put_swap(info, index, (swp_entry_t){0});
756                 info->swapped--;
757         }
758         shmem_recalc_inode(inode);
759
760         if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
761                 delete_from_page_cache(page);
762                 shmem_put_swap(info, index, swap);
763                 info->swapped++;
764                 swap_shmem_alloc(swap);
765                 spin_unlock(&info->lock);
766                 BUG_ON(page_mapped(page));
767                 swap_writepage(page, wbc);
768                 return 0;
769         }
770
771         spin_unlock(&info->lock);
772         swapcache_free(swap, NULL);
773 redirty:
774         set_page_dirty(page);
775         if (wbc->for_reclaim)
776                 return AOP_WRITEPAGE_ACTIVATE;  /* Return with page locked */
777         unlock_page(page);
778         return 0;
779 }
780
781 #ifdef CONFIG_NUMA
782 #ifdef CONFIG_TMPFS
783 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
784 {
785         char buffer[64];
786
787         if (!mpol || mpol->mode == MPOL_DEFAULT)
788                 return;         /* show nothing */
789
790         mpol_to_str(buffer, sizeof(buffer), mpol, 1);
791
792         seq_printf(seq, ",mpol=%s", buffer);
793 }
794
795 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
796 {
797         struct mempolicy *mpol = NULL;
798         if (sbinfo->mpol) {
799                 spin_lock(&sbinfo->stat_lock);  /* prevent replace/use races */
800                 mpol = sbinfo->mpol;
801                 mpol_get(mpol);
802                 spin_unlock(&sbinfo->stat_lock);
803         }
804         return mpol;
805 }
806 #endif /* CONFIG_TMPFS */
807
808 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
809                         struct shmem_inode_info *info, pgoff_t index)
810 {
811         struct mempolicy mpol, *spol;
812         struct vm_area_struct pvma;
813
814         spol = mpol_cond_copy(&mpol,
815                         mpol_shared_policy_lookup(&info->policy, index));
816
817         /* Create a pseudo vma that just contains the policy */
818         pvma.vm_start = 0;
819         pvma.vm_pgoff = index;
820         pvma.vm_ops = NULL;
821         pvma.vm_policy = spol;
822         return swapin_readahead(swap, gfp, &pvma, 0);
823 }
824
825 static struct page *shmem_alloc_page(gfp_t gfp,
826                         struct shmem_inode_info *info, pgoff_t index)
827 {
828         struct vm_area_struct pvma;
829
830         /* Create a pseudo vma that just contains the policy */
831         pvma.vm_start = 0;
832         pvma.vm_pgoff = index;
833         pvma.vm_ops = NULL;
834         pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, index);
835
836         /*
837          * alloc_page_vma() will drop the shared policy reference
838          */
839         return alloc_page_vma(gfp, &pvma, 0);
840 }
841 #else /* !CONFIG_NUMA */
842 #ifdef CONFIG_TMPFS
843 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
844 {
845 }
846 #endif /* CONFIG_TMPFS */
847
848 static inline struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
849                         struct shmem_inode_info *info, pgoff_t index)
850 {
851         return swapin_readahead(swap, gfp, NULL, 0);
852 }
853
854 static inline struct page *shmem_alloc_page(gfp_t gfp,
855                         struct shmem_inode_info *info, pgoff_t index)
856 {
857         return alloc_page(gfp);
858 }
859 #endif /* CONFIG_NUMA */
860
861 #if !defined(CONFIG_NUMA) || !defined(CONFIG_TMPFS)
862 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
863 {
864         return NULL;
865 }
866 #endif
867
868 /*
869  * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
870  *
871  * If we allocate a new one we do not mark it dirty. That's up to the
872  * vm. If we swap it in we mark it dirty since we also free the swap
873  * entry since a page cannot live in both the swap and page cache
874  */
875 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
876         struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type)
877 {
878         struct address_space *mapping = inode->i_mapping;
879         struct shmem_inode_info *info;
880         struct shmem_sb_info *sbinfo;
881         struct page *page;
882         swp_entry_t swap;
883         int error;
884         int once = 0;
885
886         if (index > (MAX_LFS_FILESIZE >> PAGE_CACHE_SHIFT))
887                 return -EFBIG;
888 repeat:
889         swap.val = 0;
890         page = find_lock_page(mapping, index);
891         if (radix_tree_exceptional_entry(page)) {
892                 swap = radix_to_swp_entry(page);
893                 page = NULL;
894         }
895
896         if (sgp != SGP_WRITE &&
897             ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
898                 error = -EINVAL;
899                 goto failed;
900         }
901
902         if (page || (sgp == SGP_READ && !swap.val)) {
903                 /*
904                  * Once we can get the page lock, it must be uptodate:
905                  * if there were an error in reading back from swap,
906                  * the page would not be inserted into the filecache.
907                  */
908                 BUG_ON(page && !PageUptodate(page));
909                 *pagep = page;
910                 return 0;
911         }
912
913         /*
914          * Fast cache lookup did not find it:
915          * bring it back from swap or allocate.
916          */
917         info = SHMEM_I(inode);
918         sbinfo = SHMEM_SB(inode->i_sb);
919
920         if (swap.val) {
921                 /* Look it up and read it in.. */
922                 page = lookup_swap_cache(swap);
923                 if (!page) {
924                         /* here we actually do the io */
925                         if (fault_type)
926                                 *fault_type |= VM_FAULT_MAJOR;
927                         page = shmem_swapin(swap, gfp, info, index);
928                         if (!page) {
929                                 error = -ENOMEM;
930                                 goto failed;
931                         }
932                 }
933
934                 /* We have to do this with page locked to prevent races */
935                 lock_page(page);
936                 if (!PageUptodate(page)) {
937                         error = -EIO;
938                         goto failed;
939                 }
940                 wait_on_page_writeback(page);
941
942                 /* Someone may have already done it for us */
943                 if (page->mapping) {
944                         if (page->mapping == mapping &&
945                             page->index == index)
946                                 goto done;
947                         error = -EEXIST;
948                         goto failed;
949                 }
950
951                 error = mem_cgroup_cache_charge(page, current->mm,
952                                                 gfp & GFP_RECLAIM_MASK);
953                 if (!error)
954                         error = shmem_add_to_page_cache(page, mapping, index,
955                                                 gfp, swp_to_radix_entry(swap));
956                 if (error)
957                         goto failed;
958
959                 spin_lock(&info->lock);
960                 info->swapped--;
961                 shmem_recalc_inode(inode);
962                 spin_unlock(&info->lock);
963
964                 delete_from_swap_cache(page);
965                 set_page_dirty(page);
966                 swap_free(swap);
967
968         } else {
969                 if (shmem_acct_block(info->flags)) {
970                         error = -ENOSPC;
971                         goto failed;
972                 }
973                 if (sbinfo->max_blocks) {
974                         if (percpu_counter_compare(&sbinfo->used_blocks,
975                                                 sbinfo->max_blocks) >= 0) {
976                                 error = -ENOSPC;
977                                 goto unacct;
978                         }
979                         percpu_counter_inc(&sbinfo->used_blocks);
980                 }
981
982                 page = shmem_alloc_page(gfp, info, index);
983                 if (!page) {
984                         error = -ENOMEM;
985                         goto decused;
986                 }
987
988                 SetPageSwapBacked(page);
989                 __set_page_locked(page);
990                 error = mem_cgroup_cache_charge(page, current->mm,
991                                                 gfp & GFP_RECLAIM_MASK);
992                 if (!error)
993                         error = shmem_add_to_page_cache(page, mapping, index,
994                                                 gfp, NULL);
995                 if (error)
996                         goto decused;
997                 lru_cache_add_anon(page);
998
999                 spin_lock(&info->lock);
1000                 info->alloced++;
1001                 inode->i_blocks += BLOCKS_PER_PAGE;
1002                 shmem_recalc_inode(inode);
1003                 spin_unlock(&info->lock);
1004
1005                 clear_highpage(page);
1006                 flush_dcache_page(page);
1007                 SetPageUptodate(page);
1008                 if (sgp == SGP_DIRTY)
1009                         set_page_dirty(page);
1010         }
1011 done:
1012         /* Perhaps the file has been truncated since we checked */
1013         if (sgp != SGP_WRITE &&
1014             ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
1015                 error = -EINVAL;
1016                 goto trunc;
1017         }
1018         *pagep = page;
1019         return 0;
1020
1021         /*
1022          * Error recovery.
1023          */
1024 trunc:
1025         ClearPageDirty(page);
1026         delete_from_page_cache(page);
1027         spin_lock(&info->lock);
1028         info->alloced--;
1029         inode->i_blocks -= BLOCKS_PER_PAGE;
1030         spin_unlock(&info->lock);
1031 decused:
1032         if (sbinfo->max_blocks)
1033                 percpu_counter_add(&sbinfo->used_blocks, -1);
1034 unacct:
1035         shmem_unacct_blocks(info->flags, 1);
1036 failed:
1037         if (swap.val && error != -EINVAL) {
1038                 struct page *test = find_get_page(mapping, index);
1039                 if (test && !radix_tree_exceptional_entry(test))
1040                         page_cache_release(test);
1041                 /* Have another try if the entry has changed */
1042                 if (test != swp_to_radix_entry(swap))
1043                         error = -EEXIST;
1044         }
1045         if (page) {
1046                 unlock_page(page);
1047                 page_cache_release(page);
1048         }
1049         if (error == -ENOSPC && !once++) {
1050                 info = SHMEM_I(inode);
1051                 spin_lock(&info->lock);
1052                 shmem_recalc_inode(inode);
1053                 spin_unlock(&info->lock);
1054                 goto repeat;
1055         }
1056         if (error == -EEXIST)
1057                 goto repeat;
1058         return error;
1059 }
1060
1061 static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1062 {
1063         struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1064         int error;
1065         int ret = VM_FAULT_LOCKED;
1066
1067         error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
1068         if (error)
1069                 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
1070
1071         if (ret & VM_FAULT_MAJOR) {
1072                 count_vm_event(PGMAJFAULT);
1073                 mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
1074         }
1075         return ret;
1076 }
1077
1078 #ifdef CONFIG_NUMA
1079 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
1080 {
1081         struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1082         return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
1083 }
1084
1085 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
1086                                           unsigned long addr)
1087 {
1088         struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1089         pgoff_t index;
1090
1091         index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1092         return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
1093 }
1094 #endif
1095
1096 int shmem_lock(struct file *file, int lock, struct user_struct *user)
1097 {
1098         struct inode *inode = file->f_path.dentry->d_inode;
1099         struct shmem_inode_info *info = SHMEM_I(inode);
1100         int retval = -ENOMEM;
1101
1102         spin_lock(&info->lock);
1103         if (lock && !(info->flags & VM_LOCKED)) {
1104                 if (!user_shm_lock(inode->i_size, user))
1105                         goto out_nomem;
1106                 info->flags |= VM_LOCKED;
1107                 mapping_set_unevictable(file->f_mapping);
1108         }
1109         if (!lock && (info->flags & VM_LOCKED) && user) {
1110                 user_shm_unlock(inode->i_size, user);
1111                 info->flags &= ~VM_LOCKED;
1112                 mapping_clear_unevictable(file->f_mapping);
1113                 scan_mapping_unevictable_pages(file->f_mapping);
1114         }
1115         retval = 0;
1116
1117 out_nomem:
1118         spin_unlock(&info->lock);
1119         return retval;
1120 }
1121
1122 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
1123 {
1124         file_accessed(file);
1125         vma->vm_ops = &shmem_vm_ops;
1126         vma->vm_flags |= VM_CAN_NONLINEAR;
1127         return 0;
1128 }
1129
1130 static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
1131                                      int mode, dev_t dev, unsigned long flags)
1132 {
1133         struct inode *inode;
1134         struct shmem_inode_info *info;
1135         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
1136
1137         if (shmem_reserve_inode(sb))
1138                 return NULL;
1139
1140         inode = new_inode(sb);
1141         if (inode) {
1142                 inode->i_ino = get_next_ino();
1143                 inode_init_owner(inode, dir, mode);
1144                 inode->i_blocks = 0;
1145                 inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
1146                 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
1147                 inode->i_generation = get_seconds();
1148                 info = SHMEM_I(inode);
1149                 memset(info, 0, (char *)inode - (char *)info);
1150                 spin_lock_init(&info->lock);
1151                 info->flags = flags & VM_NORESERVE;
1152                 INIT_LIST_HEAD(&info->swaplist);
1153                 INIT_LIST_HEAD(&info->xattr_list);
1154                 cache_no_acl(inode);
1155
1156                 switch (mode & S_IFMT) {
1157                 default:
1158                         inode->i_op = &shmem_special_inode_operations;
1159                         init_special_inode(inode, mode, dev);
1160                         break;
1161                 case S_IFREG:
1162                         inode->i_mapping->a_ops = &shmem_aops;
1163                         inode->i_op = &shmem_inode_operations;
1164                         inode->i_fop = &shmem_file_operations;
1165                         mpol_shared_policy_init(&info->policy,
1166                                                  shmem_get_sbmpol(sbinfo));
1167                         break;
1168                 case S_IFDIR:
1169                         inc_nlink(inode);
1170                         /* Some things misbehave if size == 0 on a directory */
1171                         inode->i_size = 2 * BOGO_DIRENT_SIZE;
1172                         inode->i_op = &shmem_dir_inode_operations;
1173                         inode->i_fop = &simple_dir_operations;
1174                         break;
1175                 case S_IFLNK:
1176                         /*
1177                          * Must not load anything in the rbtree,
1178                          * mpol_free_shared_policy will not be called.
1179                          */
1180                         mpol_shared_policy_init(&info->policy, NULL);
1181                         break;
1182                 }
1183         } else
1184                 shmem_free_inode(sb);
1185         return inode;
1186 }
1187
1188 #ifdef CONFIG_TMPFS
1189 static const struct inode_operations shmem_symlink_inode_operations;
1190 static const struct inode_operations shmem_symlink_inline_operations;
1191
1192 static int
1193 shmem_write_begin(struct file *file, struct address_space *mapping,
1194                         loff_t pos, unsigned len, unsigned flags,
1195                         struct page **pagep, void **fsdata)
1196 {
1197         struct inode *inode = mapping->host;
1198         pgoff_t index = pos >> PAGE_CACHE_SHIFT;
1199         return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL);
1200 }
1201
1202 static int
1203 shmem_write_end(struct file *file, struct address_space *mapping,
1204                         loff_t pos, unsigned len, unsigned copied,
1205                         struct page *page, void *fsdata)
1206 {
1207         struct inode *inode = mapping->host;
1208
1209         if (pos + copied > inode->i_size)
1210                 i_size_write(inode, pos + copied);
1211
1212         set_page_dirty(page);
1213         unlock_page(page);
1214         page_cache_release(page);
1215
1216         return copied;
1217 }
1218
1219 static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor)
1220 {
1221         struct inode *inode = filp->f_path.dentry->d_inode;
1222         struct address_space *mapping = inode->i_mapping;
1223         pgoff_t index;
1224         unsigned long offset;
1225         enum sgp_type sgp = SGP_READ;
1226
1227         /*
1228          * Might this read be for a stacking filesystem?  Then when reading
1229          * holes of a sparse file, we actually need to allocate those pages,
1230          * and even mark them dirty, so it cannot exceed the max_blocks limit.
1231          */
1232         if (segment_eq(get_fs(), KERNEL_DS))
1233                 sgp = SGP_DIRTY;
1234
1235         index = *ppos >> PAGE_CACHE_SHIFT;
1236         offset = *ppos & ~PAGE_CACHE_MASK;
1237
1238         for (;;) {
1239                 struct page *page = NULL;
1240                 pgoff_t end_index;
1241                 unsigned long nr, ret;
1242                 loff_t i_size = i_size_read(inode);
1243
1244                 end_index = i_size >> PAGE_CACHE_SHIFT;
1245                 if (index > end_index)
1246                         break;
1247                 if (index == end_index) {
1248                         nr = i_size & ~PAGE_CACHE_MASK;
1249                         if (nr <= offset)
1250                                 break;
1251                 }
1252
1253                 desc->error = shmem_getpage(inode, index, &page, sgp, NULL);
1254                 if (desc->error) {
1255                         if (desc->error == -EINVAL)
1256                                 desc->error = 0;
1257                         break;
1258                 }
1259                 if (page)
1260                         unlock_page(page);
1261
1262                 /*
1263                  * We must evaluate after, since reads (unlike writes)
1264                  * are called without i_mutex protection against truncate
1265                  */
1266                 nr = PAGE_CACHE_SIZE;
1267                 i_size = i_size_read(inode);
1268                 end_index = i_size >> PAGE_CACHE_SHIFT;
1269                 if (index == end_index) {
1270                         nr = i_size & ~PAGE_CACHE_MASK;
1271                         if (nr <= offset) {
1272                                 if (page)
1273                                         page_cache_release(page);
1274                                 break;
1275                         }
1276                 }
1277                 nr -= offset;
1278
1279                 if (page) {
1280                         /*
1281                          * If users can be writing to this page using arbitrary
1282                          * virtual addresses, take care about potential aliasing
1283                          * before reading the page on the kernel side.
1284                          */
1285                         if (mapping_writably_mapped(mapping))
1286                                 flush_dcache_page(page);
1287                         /*
1288                          * Mark the page accessed if we read the beginning.
1289                          */
1290                         if (!offset)
1291                                 mark_page_accessed(page);
1292                 } else {
1293                         page = ZERO_PAGE(0);
1294                         page_cache_get(page);
1295                 }
1296
1297                 /*
1298                  * Ok, we have the page, and it's up-to-date, so
1299                  * now we can copy it to user space...
1300                  *
1301                  * The actor routine returns how many bytes were actually used..
1302                  * NOTE! This may not be the same as how much of a user buffer
1303                  * we filled up (we may be padding etc), so we can only update
1304                  * "pos" here (the actor routine has to update the user buffer
1305                  * pointers and the remaining count).
1306                  */
1307                 ret = actor(desc, page, offset, nr);
1308                 offset += ret;
1309                 index += offset >> PAGE_CACHE_SHIFT;
1310                 offset &= ~PAGE_CACHE_MASK;
1311
1312                 page_cache_release(page);
1313                 if (ret != nr || !desc->count)
1314                         break;
1315
1316                 cond_resched();
1317         }
1318
1319         *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
1320         file_accessed(filp);
1321 }
1322
1323 static ssize_t shmem_file_aio_read(struct kiocb *iocb,
1324                 const struct iovec *iov, unsigned long nr_segs, loff_t pos)
1325 {
1326         struct file *filp = iocb->ki_filp;
1327         ssize_t retval;
1328         unsigned long seg;
1329         size_t count;
1330         loff_t *ppos = &iocb->ki_pos;
1331
1332         retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
1333         if (retval)
1334                 return retval;
1335
1336         for (seg = 0; seg < nr_segs; seg++) {
1337                 read_descriptor_t desc;
1338
1339                 desc.written = 0;
1340                 desc.arg.buf = iov[seg].iov_base;
1341                 desc.count = iov[seg].iov_len;
1342                 if (desc.count == 0)
1343                         continue;
1344                 desc.error = 0;
1345                 do_shmem_file_read(filp, ppos, &desc, file_read_actor);
1346                 retval += desc.written;
1347                 if (desc.error) {
1348                         retval = retval ?: desc.error;
1349                         break;
1350                 }
1351                 if (desc.count > 0)
1352                         break;
1353         }
1354         return retval;
1355 }
1356
1357 static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos,
1358                                 struct pipe_inode_info *pipe, size_t len,
1359                                 unsigned int flags)
1360 {
1361         struct address_space *mapping = in->f_mapping;
1362         struct inode *inode = mapping->host;
1363         unsigned int loff, nr_pages, req_pages;
1364         struct page *pages[PIPE_DEF_BUFFERS];
1365         struct partial_page partial[PIPE_DEF_BUFFERS];
1366         struct page *page;
1367         pgoff_t index, end_index;
1368         loff_t isize, left;
1369         int error, page_nr;
1370         struct splice_pipe_desc spd = {
1371                 .pages = pages,
1372                 .partial = partial,
1373                 .flags = flags,
1374                 .ops = &page_cache_pipe_buf_ops,
1375                 .spd_release = spd_release_page,
1376         };
1377
1378         isize = i_size_read(inode);
1379         if (unlikely(*ppos >= isize))
1380                 return 0;
1381
1382         left = isize - *ppos;
1383         if (unlikely(left < len))
1384                 len = left;
1385
1386         if (splice_grow_spd(pipe, &spd))
1387                 return -ENOMEM;
1388
1389         index = *ppos >> PAGE_CACHE_SHIFT;
1390         loff = *ppos & ~PAGE_CACHE_MASK;
1391         req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1392         nr_pages = min(req_pages, pipe->buffers);
1393
1394         spd.nr_pages = find_get_pages_contig(mapping, index,
1395                                                 nr_pages, spd.pages);
1396         index += spd.nr_pages;
1397         error = 0;
1398
1399         while (spd.nr_pages < nr_pages) {
1400                 error = shmem_getpage(inode, index, &page, SGP_CACHE, NULL);
1401                 if (error)
1402                         break;
1403                 unlock_page(page);
1404                 spd.pages[spd.nr_pages++] = page;
1405                 index++;
1406         }
1407
1408         index = *ppos >> PAGE_CACHE_SHIFT;
1409         nr_pages = spd.nr_pages;
1410         spd.nr_pages = 0;
1411
1412         for (page_nr = 0; page_nr < nr_pages; page_nr++) {
1413                 unsigned int this_len;
1414
1415                 if (!len)
1416                         break;
1417
1418                 this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
1419                 page = spd.pages[page_nr];
1420
1421                 if (!PageUptodate(page) || page->mapping != mapping) {
1422                         error = shmem_getpage(inode, index, &page,
1423                                                         SGP_CACHE, NULL);
1424                         if (error)
1425                                 break;
1426                         unlock_page(page);
1427                         page_cache_release(spd.pages[page_nr]);
1428                         spd.pages[page_nr] = page;
1429                 }
1430
1431                 isize = i_size_read(inode);
1432                 end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
1433                 if (unlikely(!isize || index > end_index))
1434                         break;
1435
1436                 if (end_index == index) {
1437                         unsigned int plen;
1438
1439                         plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
1440                         if (plen <= loff)
1441                                 break;
1442
1443                         this_len = min(this_len, plen - loff);
1444                         len = this_len;
1445                 }
1446
1447                 spd.partial[page_nr].offset = loff;
1448                 spd.partial[page_nr].len = this_len;
1449                 len -= this_len;
1450                 loff = 0;
1451                 spd.nr_pages++;
1452                 index++;
1453         }
1454
1455         while (page_nr < nr_pages)
1456                 page_cache_release(spd.pages[page_nr++]);
1457
1458         if (spd.nr_pages)
1459                 error = splice_to_pipe(pipe, &spd);
1460
1461         splice_shrink_spd(pipe, &spd);
1462
1463         if (error > 0) {
1464                 *ppos += error;
1465                 file_accessed(in);
1466         }
1467         return error;
1468 }
1469
1470 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
1471 {
1472         struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
1473
1474         buf->f_type = TMPFS_MAGIC;
1475         buf->f_bsize = PAGE_CACHE_SIZE;
1476         buf->f_namelen = NAME_MAX;
1477         if (sbinfo->max_blocks) {
1478                 buf->f_blocks = sbinfo->max_blocks;
1479                 buf->f_bavail =
1480                 buf->f_bfree  = sbinfo->max_blocks -
1481                                 percpu_counter_sum(&sbinfo->used_blocks);
1482         }
1483         if (sbinfo->max_inodes) {
1484                 buf->f_files = sbinfo->max_inodes;
1485                 buf->f_ffree = sbinfo->free_inodes;
1486         }
1487         /* else leave those fields 0 like simple_statfs */
1488         return 0;
1489 }
1490
1491 /*
1492  * File creation. Allocate an inode, and we're done..
1493  */
1494 static int
1495 shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev)
1496 {
1497         struct inode *inode;
1498         int error = -ENOSPC;
1499
1500         inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
1501         if (inode) {
1502                 error = security_inode_init_security(inode, dir,
1503                                                      &dentry->d_name, NULL,
1504                                                      NULL, NULL);
1505                 if (error) {
1506                         if (error != -EOPNOTSUPP) {
1507                                 iput(inode);
1508                                 return error;
1509                         }
1510                 }
1511 #ifdef CONFIG_TMPFS_POSIX_ACL
1512                 error = generic_acl_init(inode, dir);
1513                 if (error) {
1514                         iput(inode);
1515                         return error;
1516                 }
1517 #else
1518                 error = 0;
1519 #endif
1520                 dir->i_size += BOGO_DIRENT_SIZE;
1521                 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1522                 d_instantiate(dentry, inode);
1523                 dget(dentry); /* Extra count - pin the dentry in core */
1524         }
1525         return error;
1526 }
1527
1528 static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode)
1529 {
1530         int error;
1531
1532         if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
1533                 return error;
1534         inc_nlink(dir);
1535         return 0;
1536 }
1537
1538 static int shmem_create(struct inode *dir, struct dentry *dentry, int mode,
1539                 struct nameidata *nd)
1540 {
1541         return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
1542 }
1543
1544 /*
1545  * Link a file..
1546  */
1547 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
1548 {
1549         struct inode *inode = old_dentry->d_inode;
1550         int ret;
1551
1552         /*
1553          * No ordinary (disk based) filesystem counts links as inodes;
1554          * but each new link needs a new dentry, pinning lowmem, and
1555          * tmpfs dentries cannot be pruned until they are unlinked.
1556          */
1557         ret = shmem_reserve_inode(inode->i_sb);
1558         if (ret)
1559                 goto out;
1560
1561         dir->i_size += BOGO_DIRENT_SIZE;
1562         inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1563         inc_nlink(inode);
1564         ihold(inode);   /* New dentry reference */
1565         dget(dentry);           /* Extra pinning count for the created dentry */
1566         d_instantiate(dentry, inode);
1567 out:
1568         return ret;
1569 }
1570
1571 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
1572 {
1573         struct inode *inode = dentry->d_inode;
1574
1575         if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
1576                 shmem_free_inode(inode->i_sb);
1577
1578         dir->i_size -= BOGO_DIRENT_SIZE;
1579         inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1580         drop_nlink(inode);
1581         dput(dentry);   /* Undo the count from "create" - this does all the work */
1582         return 0;
1583 }
1584
1585 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
1586 {
1587         if (!simple_empty(dentry))
1588                 return -ENOTEMPTY;
1589
1590         drop_nlink(dentry->d_inode);
1591         drop_nlink(dir);
1592         return shmem_unlink(dir, dentry);
1593 }
1594
1595 /*
1596  * The VFS layer already does all the dentry stuff for rename,
1597  * we just have to decrement the usage count for the target if
1598  * it exists so that the VFS layer correctly free's it when it
1599  * gets overwritten.
1600  */
1601 static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
1602 {
1603         struct inode *inode = old_dentry->d_inode;
1604         int they_are_dirs = S_ISDIR(inode->i_mode);
1605
1606         if (!simple_empty(new_dentry))
1607                 return -ENOTEMPTY;
1608
1609         if (new_dentry->d_inode) {
1610                 (void) shmem_unlink(new_dir, new_dentry);
1611                 if (they_are_dirs)
1612                         drop_nlink(old_dir);
1613         } else if (they_are_dirs) {
1614                 drop_nlink(old_dir);
1615                 inc_nlink(new_dir);
1616         }
1617
1618         old_dir->i_size -= BOGO_DIRENT_SIZE;
1619         new_dir->i_size += BOGO_DIRENT_SIZE;
1620         old_dir->i_ctime = old_dir->i_mtime =
1621         new_dir->i_ctime = new_dir->i_mtime =
1622         inode->i_ctime = CURRENT_TIME;
1623         return 0;
1624 }
1625
1626 static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
1627 {
1628         int error;
1629         int len;
1630         struct inode *inode;
1631         struct page *page;
1632         char *kaddr;
1633         struct shmem_inode_info *info;
1634
1635         len = strlen(symname) + 1;
1636         if (len > PAGE_CACHE_SIZE)
1637                 return -ENAMETOOLONG;
1638
1639         inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE);
1640         if (!inode)
1641                 return -ENOSPC;
1642
1643         error = security_inode_init_security(inode, dir, &dentry->d_name, NULL,
1644                                              NULL, NULL);
1645         if (error) {
1646                 if (error != -EOPNOTSUPP) {
1647                         iput(inode);
1648                         return error;
1649                 }
1650                 error = 0;
1651         }
1652
1653         info = SHMEM_I(inode);
1654         inode->i_size = len-1;
1655         if (len <= SHMEM_SYMLINK_INLINE_LEN) {
1656                 /* do it inline */
1657                 memcpy(info->inline_symlink, symname, len);
1658                 inode->i_op = &shmem_symlink_inline_operations;
1659         } else {
1660                 error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
1661                 if (error) {
1662                         iput(inode);
1663                         return error;
1664                 }
1665                 inode->i_mapping->a_ops = &shmem_aops;
1666                 inode->i_op = &shmem_symlink_inode_operations;
1667                 kaddr = kmap_atomic(page, KM_USER0);
1668                 memcpy(kaddr, symname, len);
1669                 kunmap_atomic(kaddr, KM_USER0);
1670                 set_page_dirty(page);
1671                 unlock_page(page);
1672                 page_cache_release(page);
1673         }
1674         dir->i_size += BOGO_DIRENT_SIZE;
1675         dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1676         d_instantiate(dentry, inode);
1677         dget(dentry);
1678         return 0;
1679 }
1680
1681 static void *shmem_follow_link_inline(struct dentry *dentry, struct nameidata *nd)
1682 {
1683         nd_set_link(nd, SHMEM_I(dentry->d_inode)->inline_symlink);
1684         return NULL;
1685 }
1686
1687 static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
1688 {
1689         struct page *page = NULL;
1690         int error = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
1691         nd_set_link(nd, error ? ERR_PTR(error) : kmap(page));
1692         if (page)
1693                 unlock_page(page);
1694         return page;
1695 }
1696
1697 static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
1698 {
1699         if (!IS_ERR(nd_get_link(nd))) {
1700                 struct page *page = cookie;
1701                 kunmap(page);
1702                 mark_page_accessed(page);
1703                 page_cache_release(page);
1704         }
1705 }
1706
1707 #ifdef CONFIG_TMPFS_XATTR
1708 /*
1709  * Superblocks without xattr inode operations may get some security.* xattr
1710  * support from the LSM "for free". As soon as we have any other xattrs
1711  * like ACLs, we also need to implement the security.* handlers at
1712  * filesystem level, though.
1713  */
1714
1715 static int shmem_xattr_get(struct dentry *dentry, const char *name,
1716                            void *buffer, size_t size)
1717 {
1718         struct shmem_inode_info *info;
1719         struct shmem_xattr *xattr;
1720         int ret = -ENODATA;
1721
1722         info = SHMEM_I(dentry->d_inode);
1723
1724         spin_lock(&info->lock);
1725         list_for_each_entry(xattr, &info->xattr_list, list) {
1726                 if (strcmp(name, xattr->name))
1727                         continue;
1728
1729                 ret = xattr->size;
1730                 if (buffer) {
1731                         if (size < xattr->size)
1732                                 ret = -ERANGE;
1733                         else
1734                                 memcpy(buffer, xattr->value, xattr->size);
1735                 }
1736                 break;
1737         }
1738         spin_unlock(&info->lock);
1739         return ret;
1740 }
1741
1742 static int shmem_xattr_set(struct dentry *dentry, const char *name,
1743                            const void *value, size_t size, int flags)
1744 {
1745         struct inode *inode = dentry->d_inode;
1746         struct shmem_inode_info *info = SHMEM_I(inode);
1747         struct shmem_xattr *xattr;
1748         struct shmem_xattr *new_xattr = NULL;
1749         size_t len;
1750         int err = 0;
1751
1752         /* value == NULL means remove */
1753         if (value) {
1754                 /* wrap around? */
1755                 len = sizeof(*new_xattr) + size;
1756                 if (len <= sizeof(*new_xattr))
1757                         return -ENOMEM;
1758
1759                 new_xattr = kmalloc(len, GFP_KERNEL);
1760                 if (!new_xattr)
1761                         return -ENOMEM;
1762
1763                 new_xattr->name = kstrdup(name, GFP_KERNEL);
1764                 if (!new_xattr->name) {
1765                         kfree(new_xattr);
1766                         return -ENOMEM;
1767                 }
1768
1769                 new_xattr->size = size;
1770                 memcpy(new_xattr->value, value, size);
1771         }
1772
1773         spin_lock(&info->lock);
1774         list_for_each_entry(xattr, &info->xattr_list, list) {
1775                 if (!strcmp(name, xattr->name)) {
1776                         if (flags & XATTR_CREATE) {
1777                                 xattr = new_xattr;
1778                                 err = -EEXIST;
1779                         } else if (new_xattr) {
1780                                 list_replace(&xattr->list, &new_xattr->list);
1781                         } else {
1782                                 list_del(&xattr->list);
1783                         }
1784                         goto out;
1785                 }
1786         }
1787         if (flags & XATTR_REPLACE) {
1788                 xattr = new_xattr;
1789                 err = -ENODATA;
1790         } else {
1791                 list_add(&new_xattr->list, &info->xattr_list);
1792                 xattr = NULL;
1793         }
1794 out:
1795         spin_unlock(&info->lock);
1796         if (xattr)
1797                 kfree(xattr->name);
1798         kfree(xattr);
1799         return err;
1800 }
1801
1802 static const struct xattr_handler *shmem_xattr_handlers[] = {
1803 #ifdef CONFIG_TMPFS_POSIX_ACL
1804         &generic_acl_access_handler,
1805         &generic_acl_default_handler,
1806 #endif
1807         NULL
1808 };
1809
1810 static int shmem_xattr_validate(const char *name)
1811 {
1812         struct { const char *prefix; size_t len; } arr[] = {
1813                 { XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN },
1814                 { XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN }
1815         };
1816         int i;
1817
1818         for (i = 0; i < ARRAY_SIZE(arr); i++) {
1819                 size_t preflen = arr[i].len;
1820                 if (strncmp(name, arr[i].prefix, preflen) == 0) {
1821                         if (!name[preflen])
1822                                 return -EINVAL;
1823                         return 0;
1824                 }
1825         }
1826         return -EOPNOTSUPP;
1827 }
1828
1829 static ssize_t shmem_getxattr(struct dentry *dentry, const char *name,
1830                               void *buffer, size_t size)
1831 {
1832         int err;
1833
1834         /*
1835          * If this is a request for a synthetic attribute in the system.*
1836          * namespace use the generic infrastructure to resolve a handler
1837          * for it via sb->s_xattr.
1838          */
1839         if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
1840                 return generic_getxattr(dentry, name, buffer, size);
1841
1842         err = shmem_xattr_validate(name);
1843         if (err)
1844                 return err;
1845
1846         return shmem_xattr_get(dentry, name, buffer, size);
1847 }
1848
1849 static int shmem_setxattr(struct dentry *dentry, const char *name,
1850                           const void *value, size_t size, int flags)
1851 {
1852         int err;
1853
1854         /*
1855          * If this is a request for a synthetic attribute in the system.*
1856          * namespace use the generic infrastructure to resolve a handler
1857          * for it via sb->s_xattr.
1858          */
1859         if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
1860                 return generic_setxattr(dentry, name, value, size, flags);
1861
1862         err = shmem_xattr_validate(name);
1863         if (err)
1864                 return err;
1865
1866         if (size == 0)
1867                 value = "";  /* empty EA, do not remove */
1868
1869         return shmem_xattr_set(dentry, name, value, size, flags);
1870
1871 }
1872
1873 static int shmem_removexattr(struct dentry *dentry, const char *name)
1874 {
1875         int err;
1876
1877         /*
1878          * If this is a request for a synthetic attribute in the system.*
1879          * namespace use the generic infrastructure to resolve a handler
1880          * for it via sb->s_xattr.
1881          */
1882         if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
1883                 return generic_removexattr(dentry, name);
1884
1885         err = shmem_xattr_validate(name);
1886         if (err)
1887                 return err;
1888
1889         return shmem_xattr_set(dentry, name, NULL, 0, XATTR_REPLACE);
1890 }
1891
1892 static bool xattr_is_trusted(const char *name)
1893 {
1894         return !strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN);
1895 }
1896
1897 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
1898 {
1899         bool trusted = capable(CAP_SYS_ADMIN);
1900         struct shmem_xattr *xattr;
1901         struct shmem_inode_info *info;
1902         size_t used = 0;
1903
1904         info = SHMEM_I(dentry->d_inode);
1905
1906         spin_lock(&info->lock);
1907         list_for_each_entry(xattr, &info->xattr_list, list) {
1908                 size_t len;
1909
1910                 /* skip "trusted." attributes for unprivileged callers */
1911                 if (!trusted && xattr_is_trusted(xattr->name))
1912                         continue;
1913
1914                 len = strlen(xattr->name) + 1;
1915                 used += len;
1916                 if (buffer) {
1917                         if (size < used) {
1918                                 used = -ERANGE;
1919                                 break;
1920                         }
1921                         memcpy(buffer, xattr->name, len);
1922                         buffer += len;
1923                 }
1924         }
1925         spin_unlock(&info->lock);
1926
1927         return used;
1928 }
1929 #endif /* CONFIG_TMPFS_XATTR */
1930
1931 static const struct inode_operations shmem_symlink_inline_operations = {
1932         .readlink       = generic_readlink,
1933         .follow_link    = shmem_follow_link_inline,
1934 #ifdef CONFIG_TMPFS_XATTR
1935         .setxattr       = shmem_setxattr,
1936         .getxattr       = shmem_getxattr,
1937         .listxattr      = shmem_listxattr,
1938         .removexattr    = shmem_removexattr,
1939 #endif
1940 };
1941
1942 static const struct inode_operations shmem_symlink_inode_operations = {
1943         .readlink       = generic_readlink,
1944         .follow_link    = shmem_follow_link,
1945         .put_link       = shmem_put_link,
1946 #ifdef CONFIG_TMPFS_XATTR
1947         .setxattr       = shmem_setxattr,
1948         .getxattr       = shmem_getxattr,
1949         .listxattr      = shmem_listxattr,
1950         .removexattr    = shmem_removexattr,
1951 #endif
1952 };
1953
1954 static struct dentry *shmem_get_parent(struct dentry *child)
1955 {
1956         return ERR_PTR(-ESTALE);
1957 }
1958
1959 static int shmem_match(struct inode *ino, void *vfh)
1960 {
1961         __u32 *fh = vfh;
1962         __u64 inum = fh[2];
1963         inum = (inum << 32) | fh[1];
1964         return ino->i_ino == inum && fh[0] == ino->i_generation;
1965 }
1966
1967 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
1968                 struct fid *fid, int fh_len, int fh_type)
1969 {
1970         struct inode *inode;
1971         struct dentry *dentry = NULL;
1972         u64 inum = fid->raw[2];
1973         inum = (inum << 32) | fid->raw[1];
1974
1975         if (fh_len < 3)
1976                 return NULL;
1977
1978         inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
1979                         shmem_match, fid->raw);
1980         if (inode) {
1981                 dentry = d_find_alias(inode);
1982                 iput(inode);
1983         }
1984
1985         return dentry;
1986 }
1987
1988 static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len,
1989                                 int connectable)
1990 {
1991         struct inode *inode = dentry->d_inode;
1992
1993         if (*len < 3) {
1994                 *len = 3;
1995                 return 255;
1996         }
1997
1998         if (inode_unhashed(inode)) {
1999                 /* Unfortunately insert_inode_hash is not idempotent,
2000                  * so as we hash inodes here rather than at creation
2001                  * time, we need a lock to ensure we only try
2002                  * to do it once
2003                  */
2004                 static DEFINE_SPINLOCK(lock);
2005                 spin_lock(&lock);
2006                 if (inode_unhashed(inode))
2007                         __insert_inode_hash(inode,
2008                                             inode->i_ino + inode->i_generation);
2009                 spin_unlock(&lock);
2010         }
2011
2012         fh[0] = inode->i_generation;
2013         fh[1] = inode->i_ino;
2014         fh[2] = ((__u64)inode->i_ino) >> 32;
2015
2016         *len = 3;
2017         return 1;
2018 }
2019
2020 static const struct export_operations shmem_export_ops = {
2021         .get_parent     = shmem_get_parent,
2022         .encode_fh      = shmem_encode_fh,
2023         .fh_to_dentry   = shmem_fh_to_dentry,
2024 };
2025
2026 static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
2027                                bool remount)
2028 {
2029         char *this_char, *value, *rest;
2030
2031         while (options != NULL) {
2032                 this_char = options;
2033                 for (;;) {
2034                         /*
2035                          * NUL-terminate this option: unfortunately,
2036                          * mount options form a comma-separated list,
2037                          * but mpol's nodelist may also contain commas.
2038                          */
2039                         options = strchr(options, ',');
2040                         if (options == NULL)
2041                                 break;
2042                         options++;
2043                         if (!isdigit(*options)) {
2044                                 options[-1] = '\0';
2045                                 break;
2046                         }
2047                 }
2048                 if (!*this_char)
2049                         continue;
2050                 if ((value = strchr(this_char,'=')) != NULL) {
2051                         *value++ = 0;
2052                 } else {
2053                         printk(KERN_ERR
2054                             "tmpfs: No value for mount option '%s'\n",
2055                             this_char);
2056                         return 1;
2057                 }
2058
2059                 if (!strcmp(this_char,"size")) {
2060                         unsigned long long size;
2061                         size = memparse(value,&rest);
2062                         if (*rest == '%') {
2063                                 size <<= PAGE_SHIFT;
2064                                 size *= totalram_pages;
2065                                 do_div(size, 100);
2066                                 rest++;
2067                         }
2068                         if (*rest)
2069                                 goto bad_val;
2070                         sbinfo->max_blocks =
2071                                 DIV_ROUND_UP(size, PAGE_CACHE_SIZE);
2072                 } else if (!strcmp(this_char,"nr_blocks")) {
2073                         sbinfo->max_blocks = memparse(value, &rest);
2074                         if (*rest)
2075                                 goto bad_val;
2076                 } else if (!strcmp(this_char,"nr_inodes")) {
2077                         sbinfo->max_inodes = memparse(value, &rest);
2078                         if (*rest)
2079                                 goto bad_val;
2080                 } else if (!strcmp(this_char,"mode")) {
2081                         if (remount)
2082                                 continue;
2083                         sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
2084                         if (*rest)
2085                                 goto bad_val;
2086                 } else if (!strcmp(this_char,"uid")) {
2087                         if (remount)
2088                                 continue;
2089                         sbinfo->uid = simple_strtoul(value, &rest, 0);
2090                         if (*rest)
2091                                 goto bad_val;
2092                 } else if (!strcmp(this_char,"gid")) {
2093                         if (remount)
2094                                 continue;
2095                         sbinfo->gid = simple_strtoul(value, &rest, 0);
2096                         if (*rest)
2097                                 goto bad_val;
2098                 } else if (!strcmp(this_char,"mpol")) {
2099                         if (mpol_parse_str(value, &sbinfo->mpol, 1))
2100                                 goto bad_val;
2101                 } else {
2102                         printk(KERN_ERR "tmpfs: Bad mount option %s\n",
2103                                this_char);
2104                         return 1;
2105                 }
2106         }
2107         return 0;
2108
2109 bad_val:
2110         printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
2111                value, this_char);
2112         return 1;
2113
2114 }
2115
2116 static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
2117 {
2118         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2119         struct shmem_sb_info config = *sbinfo;
2120         unsigned long inodes;
2121         int error = -EINVAL;
2122
2123         if (shmem_parse_options(data, &config, true))
2124                 return error;
2125
2126         spin_lock(&sbinfo->stat_lock);
2127         inodes = sbinfo->max_inodes - sbinfo->free_inodes;
2128         if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0)
2129                 goto out;
2130         if (config.max_inodes < inodes)
2131                 goto out;
2132         /*
2133          * Those tests disallow limited->unlimited while any are in use;
2134          * but we must separately disallow unlimited->limited, because
2135          * in that case we have no record of how much is already in use.
2136          */
2137         if (config.max_blocks && !sbinfo->max_blocks)
2138                 goto out;
2139         if (config.max_inodes && !sbinfo->max_inodes)
2140                 goto out;
2141
2142         error = 0;
2143         sbinfo->max_blocks  = config.max_blocks;
2144         sbinfo->max_inodes  = config.max_inodes;
2145         sbinfo->free_inodes = config.max_inodes - inodes;
2146
2147         mpol_put(sbinfo->mpol);
2148         sbinfo->mpol        = config.mpol;      /* transfers initial ref */
2149 out:
2150         spin_unlock(&sbinfo->stat_lock);
2151         return error;
2152 }
2153
2154 static int shmem_show_options(struct seq_file *seq, struct vfsmount *vfs)
2155 {
2156         struct shmem_sb_info *sbinfo = SHMEM_SB(vfs->mnt_sb);
2157
2158         if (sbinfo->max_blocks != shmem_default_max_blocks())
2159                 seq_printf(seq, ",size=%luk",
2160                         sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10));
2161         if (sbinfo->max_inodes != shmem_default_max_inodes())
2162                 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
2163         if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
2164                 seq_printf(seq, ",mode=%03o", sbinfo->mode);
2165         if (sbinfo->uid != 0)
2166                 seq_printf(seq, ",uid=%u", sbinfo->uid);
2167         if (sbinfo->gid != 0)
2168                 seq_printf(seq, ",gid=%u", sbinfo->gid);
2169         shmem_show_mpol(seq, sbinfo->mpol);
2170         return 0;
2171 }
2172 #endif /* CONFIG_TMPFS */
2173
2174 static void shmem_put_super(struct super_block *sb)
2175 {
2176         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2177
2178         percpu_counter_destroy(&sbinfo->used_blocks);
2179         kfree(sbinfo);
2180         sb->s_fs_info = NULL;
2181 }
2182
2183 int shmem_fill_super(struct super_block *sb, void *data, int silent)
2184 {
2185         struct inode *inode;
2186         struct dentry *root;
2187         struct shmem_sb_info *sbinfo;
2188         int err = -ENOMEM;
2189
2190         /* Round up to L1_CACHE_BYTES to resist false sharing */
2191         sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
2192                                 L1_CACHE_BYTES), GFP_KERNEL);
2193         if (!sbinfo)
2194                 return -ENOMEM;
2195
2196         sbinfo->mode = S_IRWXUGO | S_ISVTX;
2197         sbinfo->uid = current_fsuid();
2198         sbinfo->gid = current_fsgid();
2199         sb->s_fs_info = sbinfo;
2200
2201 #ifdef CONFIG_TMPFS
2202         /*
2203          * Per default we only allow half of the physical ram per
2204          * tmpfs instance, limiting inodes to one per page of lowmem;
2205          * but the internal instance is left unlimited.
2206          */
2207         if (!(sb->s_flags & MS_NOUSER)) {
2208                 sbinfo->max_blocks = shmem_default_max_blocks();
2209                 sbinfo->max_inodes = shmem_default_max_inodes();
2210                 if (shmem_parse_options(data, sbinfo, false)) {
2211                         err = -EINVAL;
2212                         goto failed;
2213                 }
2214         }
2215         sb->s_export_op = &shmem_export_ops;
2216 #else
2217         sb->s_flags |= MS_NOUSER;
2218 #endif
2219
2220         spin_lock_init(&sbinfo->stat_lock);
2221         if (percpu_counter_init(&sbinfo->used_blocks, 0))
2222                 goto failed;
2223         sbinfo->free_inodes = sbinfo->max_inodes;
2224
2225         sb->s_maxbytes = MAX_LFS_FILESIZE;
2226         sb->s_blocksize = PAGE_CACHE_SIZE;
2227         sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
2228         sb->s_magic = TMPFS_MAGIC;
2229         sb->s_op = &shmem_ops;
2230         sb->s_time_gran = 1;
2231 #ifdef CONFIG_TMPFS_XATTR
2232         sb->s_xattr = shmem_xattr_handlers;
2233 #endif
2234 #ifdef CONFIG_TMPFS_POSIX_ACL
2235         sb->s_flags |= MS_POSIXACL;
2236 #endif
2237
2238         inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
2239         if (!inode)
2240                 goto failed;
2241         inode->i_uid = sbinfo->uid;
2242         inode->i_gid = sbinfo->gid;
2243         root = d_alloc_root(inode);
2244         if (!root)
2245                 goto failed_iput;
2246         sb->s_root = root;
2247         return 0;
2248
2249 failed_iput:
2250         iput(inode);
2251 failed:
2252         shmem_put_super(sb);
2253         return err;
2254 }
2255
2256 static struct kmem_cache *shmem_inode_cachep;
2257
2258 static struct inode *shmem_alloc_inode(struct super_block *sb)
2259 {
2260         struct shmem_inode_info *info;
2261         info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
2262         if (!info)
2263                 return NULL;
2264         return &info->vfs_inode;
2265 }
2266
2267 static void shmem_destroy_callback(struct rcu_head *head)
2268 {
2269         struct inode *inode = container_of(head, struct inode, i_rcu);
2270         INIT_LIST_HEAD(&inode->i_dentry);
2271         kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
2272 }
2273
2274 static void shmem_destroy_inode(struct inode *inode)
2275 {
2276         if ((inode->i_mode & S_IFMT) == S_IFREG) {
2277                 /* only struct inode is valid if it's an inline symlink */
2278                 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
2279         }
2280         call_rcu(&inode->i_rcu, shmem_destroy_callback);
2281 }
2282
2283 static void shmem_init_inode(void *foo)
2284 {
2285         struct shmem_inode_info *info = foo;
2286         inode_init_once(&info->vfs_inode);
2287 }
2288
2289 static int shmem_init_inodecache(void)
2290 {
2291         shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
2292                                 sizeof(struct shmem_inode_info),
2293                                 0, SLAB_PANIC, shmem_init_inode);
2294         return 0;
2295 }
2296
2297 static void shmem_destroy_inodecache(void)
2298 {
2299         kmem_cache_destroy(shmem_inode_cachep);
2300 }
2301
2302 static const struct address_space_operations shmem_aops = {
2303         .writepage      = shmem_writepage,
2304         .set_page_dirty = __set_page_dirty_no_writeback,
2305 #ifdef CONFIG_TMPFS
2306         .write_begin    = shmem_write_begin,
2307         .write_end      = shmem_write_end,
2308 #endif
2309         .migratepage    = migrate_page,
2310         .error_remove_page = generic_error_remove_page,
2311 };
2312
2313 static const struct file_operations shmem_file_operations = {
2314         .mmap           = shmem_mmap,
2315 #ifdef CONFIG_TMPFS
2316         .llseek         = generic_file_llseek,
2317         .read           = do_sync_read,
2318         .write          = do_sync_write,
2319         .aio_read       = shmem_file_aio_read,
2320         .aio_write      = generic_file_aio_write,
2321         .fsync          = noop_fsync,
2322         .splice_read    = shmem_file_splice_read,
2323         .splice_write   = generic_file_splice_write,
2324 #endif
2325 };
2326
2327 static const struct inode_operations shmem_inode_operations = {
2328         .setattr        = shmem_setattr,
2329         .truncate_range = shmem_truncate_range,
2330 #ifdef CONFIG_TMPFS_XATTR
2331         .setxattr       = shmem_setxattr,
2332         .getxattr       = shmem_getxattr,
2333         .listxattr      = shmem_listxattr,
2334         .removexattr    = shmem_removexattr,
2335 #endif
2336 };
2337
2338 static const struct inode_operations shmem_dir_inode_operations = {
2339 #ifdef CONFIG_TMPFS
2340         .create         = shmem_create,
2341         .lookup         = simple_lookup,
2342         .link           = shmem_link,
2343         .unlink         = shmem_unlink,
2344         .symlink        = shmem_symlink,
2345         .mkdir          = shmem_mkdir,
2346         .rmdir          = shmem_rmdir,
2347         .mknod          = shmem_mknod,
2348         .rename         = shmem_rename,
2349 #endif
2350 #ifdef CONFIG_TMPFS_XATTR
2351         .setxattr       = shmem_setxattr,
2352         .getxattr       = shmem_getxattr,
2353         .listxattr      = shmem_listxattr,
2354         .removexattr    = shmem_removexattr,
2355 #endif
2356 #ifdef CONFIG_TMPFS_POSIX_ACL
2357         .setattr        = shmem_setattr,
2358 #endif
2359 };
2360
2361 static const struct inode_operations shmem_special_inode_operations = {
2362 #ifdef CONFIG_TMPFS_XATTR
2363         .setxattr       = shmem_setxattr,
2364         .getxattr       = shmem_getxattr,
2365         .listxattr      = shmem_listxattr,
2366         .removexattr    = shmem_removexattr,
2367 #endif
2368 #ifdef CONFIG_TMPFS_POSIX_ACL
2369         .setattr        = shmem_setattr,
2370 #endif
2371 };
2372
2373 static const struct super_operations shmem_ops = {
2374         .alloc_inode    = shmem_alloc_inode,
2375         .destroy_inode  = shmem_destroy_inode,
2376 #ifdef CONFIG_TMPFS
2377         .statfs         = shmem_statfs,
2378         .remount_fs     = shmem_remount_fs,
2379         .show_options   = shmem_show_options,
2380 #endif
2381         .evict_inode    = shmem_evict_inode,
2382         .drop_inode     = generic_delete_inode,
2383         .put_super      = shmem_put_super,
2384 };
2385
2386 static const struct vm_operations_struct shmem_vm_ops = {
2387         .fault          = shmem_fault,
2388 #ifdef CONFIG_NUMA
2389         .set_policy     = shmem_set_policy,
2390         .get_policy     = shmem_get_policy,
2391 #endif
2392 };
2393
2394 static struct dentry *shmem_mount(struct file_system_type *fs_type,
2395         int flags, const char *dev_name, void *data)
2396 {
2397         return mount_nodev(fs_type, flags, data, shmem_fill_super);
2398 }
2399
2400 static struct file_system_type shmem_fs_type = {
2401         .owner          = THIS_MODULE,
2402         .name           = "tmpfs",
2403         .mount          = shmem_mount,
2404         .kill_sb        = kill_litter_super,
2405 };
2406
2407 int __init shmem_init(void)
2408 {
2409         int error;
2410
2411         error = bdi_init(&shmem_backing_dev_info);
2412         if (error)
2413                 goto out4;
2414
2415         error = shmem_init_inodecache();
2416         if (error)
2417                 goto out3;
2418
2419         error = register_filesystem(&shmem_fs_type);
2420         if (error) {
2421                 printk(KERN_ERR "Could not register tmpfs\n");
2422                 goto out2;
2423         }
2424
2425         shm_mnt = vfs_kern_mount(&shmem_fs_type, MS_NOUSER,
2426                                  shmem_fs_type.name, NULL);
2427         if (IS_ERR(shm_mnt)) {
2428                 error = PTR_ERR(shm_mnt);
2429                 printk(KERN_ERR "Could not kern_mount tmpfs\n");
2430                 goto out1;
2431         }
2432         return 0;
2433
2434 out1:
2435         unregister_filesystem(&shmem_fs_type);
2436 out2:
2437         shmem_destroy_inodecache();
2438 out3:
2439         bdi_destroy(&shmem_backing_dev_info);
2440 out4:
2441         shm_mnt = ERR_PTR(error);
2442         return error;
2443 }
2444
2445 #else /* !CONFIG_SHMEM */
2446
2447 /*
2448  * tiny-shmem: simple shmemfs and tmpfs using ramfs code
2449  *
2450  * This is intended for small system where the benefits of the full
2451  * shmem code (swap-backed and resource-limited) are outweighed by
2452  * their complexity. On systems without swap this code should be
2453  * effectively equivalent, but much lighter weight.
2454  */
2455
2456 #include <linux/ramfs.h>
2457
2458 static struct file_system_type shmem_fs_type = {
2459         .name           = "tmpfs",
2460         .mount          = ramfs_mount,
2461         .kill_sb        = kill_litter_super,
2462 };
2463
2464 int __init shmem_init(void)
2465 {
2466         BUG_ON(register_filesystem(&shmem_fs_type) != 0);
2467
2468         shm_mnt = kern_mount(&shmem_fs_type);
2469         BUG_ON(IS_ERR(shm_mnt));
2470
2471         return 0;
2472 }
2473
2474 int shmem_unuse(swp_entry_t swap, struct page *page)
2475 {
2476         return 0;
2477 }
2478
2479 int shmem_lock(struct file *file, int lock, struct user_struct *user)
2480 {
2481         return 0;
2482 }
2483
2484 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
2485 {
2486         truncate_inode_pages_range(inode->i_mapping, lstart, lend);
2487 }
2488 EXPORT_SYMBOL_GPL(shmem_truncate_range);
2489
2490 #define shmem_vm_ops                            generic_file_vm_ops
2491 #define shmem_file_operations                   ramfs_file_operations
2492 #define shmem_get_inode(sb, dir, mode, dev, flags)      ramfs_get_inode(sb, dir, mode, dev)
2493 #define shmem_acct_size(flags, size)            0
2494 #define shmem_unacct_size(flags, size)          do {} while (0)
2495
2496 #endif /* CONFIG_SHMEM */
2497
2498 /* common code */
2499
2500 /**
2501  * shmem_file_setup - get an unlinked file living in tmpfs
2502  * @name: name for dentry (to be seen in /proc/<pid>/maps
2503  * @size: size to be set for the file
2504  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
2505  */
2506 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
2507 {
2508         int error;
2509         struct file *file;
2510         struct inode *inode;
2511         struct path path;
2512         struct dentry *root;
2513         struct qstr this;
2514
2515         if (IS_ERR(shm_mnt))
2516                 return (void *)shm_mnt;
2517
2518         if (size < 0 || size > MAX_LFS_FILESIZE)
2519                 return ERR_PTR(-EINVAL);
2520
2521         if (shmem_acct_size(flags, size))
2522                 return ERR_PTR(-ENOMEM);
2523
2524         error = -ENOMEM;
2525         this.name = name;
2526         this.len = strlen(name);
2527         this.hash = 0; /* will go */
2528         root = shm_mnt->mnt_root;
2529         path.dentry = d_alloc(root, &this);
2530         if (!path.dentry)
2531                 goto put_memory;
2532         path.mnt = mntget(shm_mnt);
2533
2534         error = -ENOSPC;
2535         inode = shmem_get_inode(root->d_sb, NULL, S_IFREG | S_IRWXUGO, 0, flags);
2536         if (!inode)
2537                 goto put_dentry;
2538
2539         d_instantiate(path.dentry, inode);
2540         inode->i_size = size;
2541         inode->i_nlink = 0;     /* It is unlinked */
2542 #ifndef CONFIG_MMU
2543         error = ramfs_nommu_expand_for_mapping(inode, size);
2544         if (error)
2545                 goto put_dentry;
2546 #endif
2547
2548         error = -ENFILE;
2549         file = alloc_file(&path, FMODE_WRITE | FMODE_READ,
2550                   &shmem_file_operations);
2551         if (!file)
2552                 goto put_dentry;
2553
2554         return file;
2555
2556 put_dentry:
2557         path_put(&path);
2558 put_memory:
2559         shmem_unacct_size(flags, size);
2560         return ERR_PTR(error);
2561 }
2562 EXPORT_SYMBOL_GPL(shmem_file_setup);
2563
2564 /**
2565  * shmem_zero_setup - setup a shared anonymous mapping
2566  * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
2567  */
2568 int shmem_zero_setup(struct vm_area_struct *vma)
2569 {
2570         struct file *file;
2571         loff_t size = vma->vm_end - vma->vm_start;
2572
2573         file = shmem_file_setup("dev/zero", size, vma->vm_flags);
2574         if (IS_ERR(file))
2575                 return PTR_ERR(file);
2576
2577         if (vma->vm_file)
2578                 fput(vma->vm_file);
2579         vma->vm_file = file;
2580         vma->vm_ops = &shmem_vm_ops;
2581         vma->vm_flags |= VM_CAN_NONLINEAR;
2582         return 0;
2583 }
2584
2585 /**
2586  * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
2587  * @mapping:    the page's address_space
2588  * @index:      the page index
2589  * @gfp:        the page allocator flags to use if allocating
2590  *
2591  * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
2592  * with any new page allocations done using the specified allocation flags.
2593  * But read_cache_page_gfp() uses the ->readpage() method: which does not
2594  * suit tmpfs, since it may have pages in swapcache, and needs to find those
2595  * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
2596  *
2597  * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
2598  * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
2599  */
2600 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
2601                                          pgoff_t index, gfp_t gfp)
2602 {
2603 #ifdef CONFIG_SHMEM
2604         struct inode *inode = mapping->host;
2605         struct page *page;
2606         int error;
2607
2608         BUG_ON(mapping->a_ops != &shmem_aops);
2609         error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, gfp, NULL);
2610         if (error)
2611                 page = ERR_PTR(error);
2612         else
2613                 unlock_page(page);
2614         return page;
2615 #else
2616         /*
2617          * The tiny !SHMEM case uses ramfs without swap
2618          */
2619         return read_cache_page_gfp(mapping, index, gfp);
2620 #endif
2621 }
2622 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);