mm: bugfix: set current->reclaim_state to NULL while returning from kswapd()
[linux-2.6.git] / mm / memory.c
index 6953d39..6105f47 100644 (file)
@@ -47,7 +47,7 @@
 #include <linux/pagemap.h>
 #include <linux/ksm.h>
 #include <linux/rmap.h>
-#include <linux/module.h>
+#include <linux/export.h>
 #include <linux/delayacct.h>
 #include <linux/init.h>
 #include <linux/writeback.h>
@@ -125,17 +125,17 @@ core_initcall(init_zero_pfn);
 
 #if defined(SPLIT_RSS_COUNTING)
 
-static void __sync_task_rss_stat(struct task_struct *task, struct mm_struct *mm)
+void sync_mm_rss(struct mm_struct *mm)
 {
        int i;
 
        for (i = 0; i < NR_MM_COUNTERS; i++) {
-               if (task->rss_stat.count[i]) {
-                       add_mm_counter(mm, i, task->rss_stat.count[i]);
-                       task->rss_stat.count[i] = 0;
+               if (current->rss_stat.count[i]) {
+                       add_mm_counter(mm, i, current->rss_stat.count[i]);
+                       current->rss_stat.count[i] = 0;
                }
        }
-       task->rss_stat.events = 0;
+       current->rss_stat.events = 0;
 }
 
 static void add_mm_counter_fast(struct mm_struct *mm, int member, int val)
@@ -157,30 +157,7 @@ static void check_sync_rss_stat(struct task_struct *task)
        if (unlikely(task != current))
                return;
        if (unlikely(task->rss_stat.events++ > TASK_RSS_EVENTS_THRESH))
-               __sync_task_rss_stat(task, task->mm);
-}
-
-unsigned long get_mm_counter(struct mm_struct *mm, int member)
-{
-       long val = 0;
-
-       /*
-        * Don't use task->mm here...for avoiding to use task_get_mm()..
-        * The caller must guarantee task->mm is not invalid.
-        */
-       val = atomic_long_read(&mm->rss_stat.count[member]);
-       /*
-        * counter is updated in asynchronous manner and may go to minus.
-        * But it's never be expected number for users.
-        */
-       if (val < 0)
-               return 0;
-       return (unsigned long)val;
-}
-
-void sync_mm_rss(struct task_struct *task, struct mm_struct *mm)
-{
-       __sync_task_rss_stat(task, mm);
+               sync_mm_rss(task->mm);
 }
 #else /* SPLIT_RSS_COUNTING */
 
@@ -293,7 +270,7 @@ int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
 {
        struct mmu_gather_batch *batch;
 
-       tlb->need_flush = 1;
+       VM_BUG_ON(!tlb->need_flush);
 
        if (tlb_fast_mode(tlb)) {
                free_page_and_swap_cache(page);
@@ -305,6 +282,7 @@ int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
        if (batch->nr == batch->max) {
                if (!tlb_next_batch(tlb))
                        return 0;
+               batch = tlb->active;
        }
        VM_BUG_ON(batch->nr > batch->max);
 
@@ -660,7 +638,7 @@ static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss)
        int i;
 
        if (current->mm == mm)
-               sync_mm_rss(current, mm);
+               sync_mm_rss(mm);
        for (i = 0; i < NR_MM_COUNTERS; i++)
                if (rss[i])
                        add_mm_counter(mm, i, rss[i]);
@@ -877,15 +855,24 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
                        }
                        if (likely(!non_swap_entry(entry)))
                                rss[MM_SWAPENTS]++;
-                       else if (is_write_migration_entry(entry) &&
-                                       is_cow_mapping(vm_flags)) {
-                               /*
-                                * COW mappings require pages in both parent
-                                * and child to be set to read.
-                                */
-                               make_migration_entry_read(&entry);
-                               pte = swp_entry_to_pte(entry);
-                               set_pte_at(src_mm, addr, src_pte, pte);
+                       else if (is_migration_entry(entry)) {
+                               page = migration_entry_to_page(entry);
+
+                               if (PageAnon(page))
+                                       rss[MM_ANONPAGES]++;
+                               else
+                                       rss[MM_FILEPAGES]++;
+
+                               if (is_write_migration_entry(entry) &&
+                                   is_cow_mapping(vm_flags)) {
+                                       /*
+                                        * COW mappings require pages in both
+                                        * parent and child to be set to read.
+                                        */
+                                       make_migration_entry_read(&entry);
+                                       pte = swp_entry_to_pte(entry);
+                                       set_pte_at(src_mm, addr, src_pte, pte);
+                               }
                        }
                }
                goto out_set_pte;
@@ -1112,11 +1099,13 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
        int force_flush = 0;
        int rss[NR_MM_COUNTERS];
        spinlock_t *ptl;
+       pte_t *start_pte;
        pte_t *pte;
 
 again:
        init_rss_vec(rss);
-       pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
+       start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
+       pte = start_pte;
        arch_enter_lazy_mmu_mode();
        do {
                pte_t ptent = *pte;
@@ -1188,6 +1177,16 @@ again:
 
                        if (!non_swap_entry(entry))
                                rss[MM_SWAPENTS]--;
+                       else if (is_migration_entry(entry)) {
+                               struct page *page;
+
+                               page = migration_entry_to_page(entry);
+
+                               if (PageAnon(page))
+                                       rss[MM_ANONPAGES]--;
+                               else
+                                       rss[MM_FILEPAGES]--;
+                       }
                        if (unlikely(!free_swap_and_cache(entry)))
                                print_bad_pte(vma, addr, ptent, NULL);
                }
@@ -1196,7 +1195,7 @@ again:
 
        add_mm_rss_vec(mm, rss);
        arch_leave_lazy_mmu_mode();
-       pte_unmap_unlock(pte - 1, ptl);
+       pte_unmap_unlock(start_pte, ptl);
 
        /*
         * mmu_gather ran out of room to batch pages, we break out of
@@ -1225,16 +1224,24 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
        do {
                next = pmd_addr_end(addr, end);
                if (pmd_trans_huge(*pmd)) {
-                       if (next-addr != HPAGE_PMD_SIZE) {
+                       if (next - addr != HPAGE_PMD_SIZE) {
                                VM_BUG_ON(!rwsem_is_locked(&tlb->mm->mmap_sem));
                                split_huge_page_pmd(vma->vm_mm, pmd);
-                       } else if (zap_huge_pmd(tlb, vma, pmd))
-                               continue;
+                       } else if (zap_huge_pmd(tlb, vma, pmd, addr))
+                               goto next;
                        /* fall through */
                }
-               if (pmd_none_or_clear_bad(pmd))
-                       continue;
+               /*
+                * Here there can be other concurrent MADV_DONTNEED or
+                * trans huge page faults running, and if the pmd is
+                * none or trans huge it can change under us. This is
+                * because MADV_DONTNEED holds the mmap_sem in read
+                * mode.
+                */
+               if (pmd_none_or_trans_huge_or_clear_bad(pmd))
+                       goto next;
                next = zap_pte_range(tlb, vma, pmd, addr, next, details);
+next:
                cond_resched();
        } while (pmd++, addr = next, addr != end);
 
@@ -1260,10 +1267,10 @@ static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
        return addr;
 }
 
-static unsigned long unmap_page_range(struct mmu_gather *tlb,
-                               struct vm_area_struct *vma,
-                               unsigned long addr, unsigned long end,
-                               struct zap_details *details)
+static void unmap_page_range(struct mmu_gather *tlb,
+                            struct vm_area_struct *vma,
+                            unsigned long addr, unsigned long end,
+                            struct zap_details *details)
 {
        pgd_t *pgd;
        unsigned long next;
@@ -1283,34 +1290,60 @@ static unsigned long unmap_page_range(struct mmu_gather *tlb,
        } while (pgd++, addr = next, addr != end);
        tlb_end_vma(tlb, vma);
        mem_cgroup_uncharge_end();
-
-       return addr;
 }
 
-#ifdef CONFIG_PREEMPT
-# define ZAP_BLOCK_SIZE        (8 * PAGE_SIZE)
-#else
-/* No preempt: go for improved straight-line efficiency */
-# define ZAP_BLOCK_SIZE        (1024 * PAGE_SIZE)
-#endif
+
+static void unmap_single_vma(struct mmu_gather *tlb,
+               struct vm_area_struct *vma, unsigned long start_addr,
+               unsigned long end_addr, unsigned long *nr_accounted,
+               struct zap_details *details)
+{
+       unsigned long start = max(vma->vm_start, start_addr);
+       unsigned long end;
+
+       if (start >= vma->vm_end)
+               return;
+       end = min(vma->vm_end, end_addr);
+       if (end <= vma->vm_start)
+               return;
+
+       if (vma->vm_flags & VM_ACCOUNT)
+               *nr_accounted += (end - start) >> PAGE_SHIFT;
+
+       if (unlikely(is_pfn_mapping(vma)))
+               untrack_pfn_vma(vma, 0, 0);
+
+       if (start != end) {
+               if (unlikely(is_vm_hugetlb_page(vma))) {
+                       /*
+                        * It is undesirable to test vma->vm_file as it
+                        * should be non-null for valid hugetlb area.
+                        * However, vm_file will be NULL in the error
+                        * cleanup path of do_mmap_pgoff. When
+                        * hugetlbfs ->mmap method fails,
+                        * do_mmap_pgoff() nullifies vma->vm_file
+                        * before calling this function to clean up.
+                        * Since no pte has actually been setup, it is
+                        * safe to do nothing in this case.
+                        */
+                       if (vma->vm_file)
+                               unmap_hugepage_range(vma, start, end, NULL);
+               } else
+                       unmap_page_range(tlb, vma, start, end, details);
+       }
+}
 
 /**
  * unmap_vmas - unmap a range of memory covered by a list of vma's
- * @tlbp: address of the caller's struct mmu_gather
+ * @tlb: address of the caller's struct mmu_gather
  * @vma: the starting vma
  * @start_addr: virtual address at which to start unmapping
  * @end_addr: virtual address at which to end unmapping
  * @nr_accounted: Place number of unmapped pages in vm-accountable vma's here
  * @details: details of nonlinear truncation or shared cache invalidation
  *
- * Returns the end address of the unmapping (restart addr if interrupted).
- *
  * Unmap all pages in the vma list.
  *
- * We aim to not hold locks for too long (for scheduling latency reasons).
- * So zap pages in ZAP_BLOCK_SIZE bytecounts.  This means we need to
- * return the ending mmu_gather to the caller.
- *
  * Only addresses between `start' and `end' will be unmapped.
  *
  * The VMA list must be sorted in ascending virtual address order.
@@ -1320,55 +1353,18 @@ static unsigned long unmap_page_range(struct mmu_gather *tlb,
  * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
  * drops the lock and schedules.
  */
-unsigned long unmap_vmas(struct mmu_gather *tlb,
+void unmap_vmas(struct mmu_gather *tlb,
                struct vm_area_struct *vma, unsigned long start_addr,
                unsigned long end_addr, unsigned long *nr_accounted,
                struct zap_details *details)
 {
-       unsigned long start = start_addr;
        struct mm_struct *mm = vma->vm_mm;
 
        mmu_notifier_invalidate_range_start(mm, start_addr, end_addr);
-       for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) {
-               unsigned long end;
-
-               start = max(vma->vm_start, start_addr);
-               if (start >= vma->vm_end)
-                       continue;
-               end = min(vma->vm_end, end_addr);
-               if (end <= vma->vm_start)
-                       continue;
-
-               if (vma->vm_flags & VM_ACCOUNT)
-                       *nr_accounted += (end - start) >> PAGE_SHIFT;
-
-               if (unlikely(is_pfn_mapping(vma)))
-                       untrack_pfn_vma(vma, 0, 0);
-
-               while (start != end) {
-                       if (unlikely(is_vm_hugetlb_page(vma))) {
-                               /*
-                                * It is undesirable to test vma->vm_file as it
-                                * should be non-null for valid hugetlb area.
-                                * However, vm_file will be NULL in the error
-                                * cleanup path of do_mmap_pgoff. When
-                                * hugetlbfs ->mmap method fails,
-                                * do_mmap_pgoff() nullifies vma->vm_file
-                                * before calling this function to clean up.
-                                * Since no pte has actually been setup, it is
-                                * safe to do nothing in this case.
-                                */
-                               if (vma->vm_file)
-                                       unmap_hugepage_range(vma, start, end, NULL);
-
-                               start = end;
-                       } else
-                               start = unmap_page_range(tlb, vma, start, end, details);
-               }
-       }
-
+       for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next)
+               unmap_single_vma(tlb, vma, start_addr, end_addr, nr_accounted,
+                                details);
        mmu_notifier_invalidate_range_end(mm, start_addr, end_addr);
-       return start;   /* which is now the end (or restart) address */
 }
 
 /**
@@ -1377,8 +1373,34 @@ unsigned long unmap_vmas(struct mmu_gather *tlb,
  * @address: starting address of pages to zap
  * @size: number of bytes to zap
  * @details: details of nonlinear truncation or shared cache invalidation
+ *
+ * Caller must protect the VMA list
+ */
+void zap_page_range(struct vm_area_struct *vma, unsigned long address,
+               unsigned long size, struct zap_details *details)
+{
+       struct mm_struct *mm = vma->vm_mm;
+       struct mmu_gather tlb;
+       unsigned long end = address + size;
+       unsigned long nr_accounted = 0;
+
+       lru_add_drain();
+       tlb_gather_mmu(&tlb, mm, 0);
+       update_hiwater_rss(mm);
+       unmap_vmas(&tlb, vma, address, end, &nr_accounted, details);
+       tlb_finish_mmu(&tlb, address, end);
+}
+
+/**
+ * zap_page_range_single - remove user pages in a given range
+ * @vma: vm_area_struct holding the applicable pages
+ * @address: starting address of pages to zap
+ * @size: number of bytes to zap
+ * @details: details of nonlinear truncation or shared cache invalidation
+ *
+ * The range must fit into one VMA.
  */
-unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address,
+static void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
                unsigned long size, struct zap_details *details)
 {
        struct mm_struct *mm = vma->vm_mm;
@@ -1389,9 +1411,10 @@ unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address,
        lru_add_drain();
        tlb_gather_mmu(&tlb, mm, 0);
        update_hiwater_rss(mm);
-       end = unmap_vmas(&tlb, vma, address, end, &nr_accounted, details);
+       mmu_notifier_invalidate_range_start(mm, address, end);
+       unmap_single_vma(&tlb, vma, address, end, &nr_accounted, details);
+       mmu_notifier_invalidate_range_end(mm, address, end);
        tlb_finish_mmu(&tlb, address, end);
-       return end;
 }
 
 /**
@@ -1412,7 +1435,7 @@ int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
        if (address < vma->vm_start || address + size > vma->vm_end ||
                        !(vma->vm_flags & VM_PFNMAP))
                return -1;
-       zap_page_range(vma, address, size, NULL);
+       zap_page_range_single(vma, address, size, NULL);
        return 0;
 }
 EXPORT_SYMBOL_GPL(zap_vma_ptes);
@@ -1511,7 +1534,7 @@ split_fallthrough:
        }
 
        if (flags & FOLL_GET)
-               get_page(page);
+               get_page_foll(page);
        if (flags & FOLL_TOUCH) {
                if ((flags & FOLL_WRITE) &&
                    !pte_dirty(pte) && !PageDirty(page))
@@ -1813,7 +1836,63 @@ next_page:
 }
 EXPORT_SYMBOL(__get_user_pages);
 
-/**
+/*
+ * fixup_user_fault() - manually resolve a user page fault
+ * @tsk:       the task_struct to use for page fault accounting, or
+ *             NULL if faults are not to be recorded.
+ * @mm:                mm_struct of target mm
+ * @address:   user address
+ * @fault_flags:flags to pass down to handle_mm_fault()
+ *
+ * This is meant to be called in the specific scenario where for locking reasons
+ * we try to access user memory in atomic context (within a pagefault_disable()
+ * section), this returns -EFAULT, and we want to resolve the user fault before
+ * trying again.
+ *
+ * Typically this is meant to be used by the futex code.
+ *
+ * The main difference with get_user_pages() is that this function will
+ * unconditionally call handle_mm_fault() which will in turn perform all the
+ * necessary SW fixup of the dirty and young bits in the PTE, while
+ * handle_mm_fault() only guarantees to update these in the struct page.
+ *
+ * This is important for some architectures where those bits also gate the
+ * access permission to the page because they are maintained in software.  On
+ * such architectures, gup() will not be enough to make a subsequent access
+ * succeed.
+ *
+ * This should be called with the mm_sem held for read.
+ */
+int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
+                    unsigned long address, unsigned int fault_flags)
+{
+       struct vm_area_struct *vma;
+       int ret;
+
+       vma = find_extend_vma(mm, address);
+       if (!vma || address < vma->vm_start)
+               return -EFAULT;
+
+       ret = handle_mm_fault(mm, vma, address, fault_flags);
+       if (ret & VM_FAULT_ERROR) {
+               if (ret & VM_FAULT_OOM)
+                       return -ENOMEM;
+               if (ret & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE))
+                       return -EHWPOISON;
+               if (ret & VM_FAULT_SIGBUS)
+                       return -EFAULT;
+               BUG();
+       }
+       if (tsk) {
+               if (ret & VM_FAULT_MAJOR)
+                       tsk->maj_flt++;
+               else
+                       tsk->min_flt++;
+       }
+       return 0;
+}
+
+/*
  * get_user_pages() - pin user pages in memory
  * @tsk:       the task_struct to use for page fault accounting, or
  *             NULL if faults are not to be recorded.
@@ -2380,7 +2459,7 @@ static inline void cow_user_page(struct page *dst, struct page *src, unsigned lo
         * fails, we just zero-fill it. Live with it.
         */
        if (unlikely(!src)) {
-               void *kaddr = kmap_atomic(dst, KM_USER0);
+               void *kaddr = kmap_atomic(dst);
                void __user *uaddr = (void __user *)(va & PAGE_MASK);
 
                /*
@@ -2391,7 +2470,7 @@ static inline void cow_user_page(struct page *dst, struct page *src, unsigned lo
                 */
                if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE))
                        clear_page(kaddr);
-               kunmap_atomic(kaddr, KM_USER0);
+               kunmap_atomic(kaddr);
                flush_dcache_page(dst);
        } else
                copy_user_highpage(dst, src, va, vma);
@@ -2703,7 +2782,7 @@ static void unmap_mapping_range_vma(struct vm_area_struct *vma,
                unsigned long start_addr, unsigned long end_addr,
                struct zap_details *details)
 {
-       zap_page_range(vma, start_addr, end_addr - start_addr, details);
+       zap_page_range_single(vma, start_addr, end_addr - start_addr, details);
 }
 
 static inline void unmap_mapping_range_tree(struct prio_tree_root *root,
@@ -2796,30 +2875,6 @@ void unmap_mapping_range(struct address_space *mapping,
 }
 EXPORT_SYMBOL(unmap_mapping_range);
 
-int vmtruncate_range(struct inode *inode, loff_t offset, loff_t end)
-{
-       struct address_space *mapping = inode->i_mapping;
-
-       /*
-        * If the underlying filesystem is not going to provide
-        * a way to truncate a range of blocks (punch a hole) -
-        * we should return failure right now.
-        */
-       if (!inode->i_op->truncate_range)
-               return -ENOSYS;
-
-       mutex_lock(&inode->i_mutex);
-       down_write(&inode->i_alloc_sem);
-       unmap_mapping_range(mapping, offset, (end - offset), 1);
-       truncate_inode_pages_range(mapping, offset, end);
-       unmap_mapping_range(mapping, offset, (end - offset), 1);
-       inode->i_op->truncate_range(inode, offset, end);
-       up_write(&inode->i_alloc_sem);
-       mutex_unlock(&inode->i_mutex);
-
-       return 0;
-}
-
 /*
  * We enter with non-exclusive mmap_sem (to exclude vma changes,
  * but allow concurrent faults), and pte mapped but not yet locked.
@@ -3125,14 +3180,34 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
        pte_t *page_table;
        spinlock_t *ptl;
        struct page *page;
+       struct page *cow_page;
        pte_t entry;
        int anon = 0;
-       int charged = 0;
        struct page *dirty_page = NULL;
        struct vm_fault vmf;
        int ret;
        int page_mkwrite = 0;
 
+       /*
+        * If we do COW later, allocate page befor taking lock_page()
+        * on the file cache page. This will reduce lock holding time.
+        */
+       if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
+
+               if (unlikely(anon_vma_prepare(vma)))
+                       return VM_FAULT_OOM;
+
+               cow_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
+               if (!cow_page)
+                       return VM_FAULT_OOM;
+
+               if (mem_cgroup_newpage_charge(cow_page, mm, GFP_KERNEL)) {
+                       page_cache_release(cow_page);
+                       return VM_FAULT_OOM;
+               }
+       } else
+               cow_page = NULL;
+
        vmf.virtual_address = (void __user *)(address & PAGE_MASK);
        vmf.pgoff = pgoff;
        vmf.flags = flags;
@@ -3141,12 +3216,13 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
        ret = vma->vm_ops->fault(vma, &vmf);
        if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
                            VM_FAULT_RETRY)))
-               return ret;
+               goto uncharge_out;
 
        if (unlikely(PageHWPoison(vmf.page))) {
                if (ret & VM_FAULT_LOCKED)
                        unlock_page(vmf.page);
-               return VM_FAULT_HWPOISON;
+               ret = VM_FAULT_HWPOISON;
+               goto uncharge_out;
        }
 
        /*
@@ -3164,23 +3240,8 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
        page = vmf.page;
        if (flags & FAULT_FLAG_WRITE) {
                if (!(vma->vm_flags & VM_SHARED)) {
+                       page = cow_page;
                        anon = 1;
-                       if (unlikely(anon_vma_prepare(vma))) {
-                               ret = VM_FAULT_OOM;
-                               goto out;
-                       }
-                       page = alloc_page_vma(GFP_HIGHUSER_MOVABLE,
-                                               vma, address);
-                       if (!page) {
-                               ret = VM_FAULT_OOM;
-                               goto out;
-                       }
-                       if (mem_cgroup_newpage_charge(page, mm, GFP_KERNEL)) {
-                               ret = VM_FAULT_OOM;
-                               page_cache_release(page);
-                               goto out;
-                       }
-                       charged = 1;
                        copy_user_highpage(page, vmf.page, address, vma);
                        __SetPageUptodate(page);
                } else {
@@ -3249,8 +3310,8 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
                /* no need to invalidate: a not-present page won't be cached */
                update_mmu_cache(vma, address, page_table);
        } else {
-               if (charged)
-                       mem_cgroup_uncharge_page(page);
+               if (cow_page)
+                       mem_cgroup_uncharge_page(cow_page);
                if (anon)
                        page_cache_release(page);
                else
@@ -3259,7 +3320,6 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
 
        pte_unmap_unlock(page_table, ptl);
 
-out:
        if (dirty_page) {
                struct address_space *mapping = page->mapping;
 
@@ -3289,6 +3349,13 @@ out:
 unwritable_page:
        page_cache_release(page);
        return ret;
+uncharge_out:
+       /* fs's fault handler get error */
+       if (cow_page) {
+               mem_cgroup_uncharge_page(cow_page);
+               page_cache_release(cow_page);
+       }
+       return ret;
 }
 
 static int do_linear_fault(struct mm_struct *mm, struct vm_area_struct *vma,
@@ -3556,13 +3623,7 @@ static int __init gate_vma_init(void)
        gate_vma.vm_end = FIXADDR_USER_END;
        gate_vma.vm_flags = VM_READ | VM_MAYREAD | VM_EXEC | VM_MAYEXEC;
        gate_vma.vm_page_prot = __P101;
-       /*
-        * Make sure the vDSO gets into every core dump.
-        * Dumping its contents makes post-mortem fully interpretable later
-        * without matching up the same kernel and hardware config to see
-        * what PC values meant.
-        */
-       gate_vma.vm_flags |= VM_ALWAYSDUMP;
+
        return 0;
 }
 __initcall(gate_vma_init);