mm: vmscan: fix force-scanning small targets without swap
[linux-2.6.git] / mm / mmap.c
index ee22989..a65efd4 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -28,6 +28,8 @@
 #include <linux/rmap.h>
 #include <linux/mmu_notifier.h>
 #include <linux/perf_event.h>
+#include <linux/audit.h>
+#include <linux/khugepaged.h>
 
 #include <asm/uaccess.h>
 #include <asm/cacheflush.h>
@@ -82,10 +84,14 @@ pgprot_t vm_get_page_prot(unsigned long vm_flags)
 }
 EXPORT_SYMBOL(vm_get_page_prot);
 
-int sysctl_overcommit_memory = OVERCOMMIT_GUESS;  /* heuristic overcommit */
-int sysctl_overcommit_ratio = 50;      /* default is 50% */
+int sysctl_overcommit_memory __read_mostly = OVERCOMMIT_GUESS;  /* heuristic overcommit */
+int sysctl_overcommit_ratio __read_mostly = 50;        /* default is 50% */
 int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
-struct percpu_counter vm_committed_as;
+/*
+ * Make sure vm_committed_as in one cacheline and not cacheline shared with
+ * other variables. It can be updated by several CPUs frequently.
+ */
+struct percpu_counter vm_committed_as ____cacheline_aligned_in_smp;
 
 /*
  * Check that a process has enough memory to allocate a new virtual
@@ -116,9 +122,17 @@ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
                return 0;
 
        if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
-               unsigned long n;
+               free = global_page_state(NR_FREE_PAGES);
+               free += global_page_state(NR_FILE_PAGES);
+
+               /*
+                * shmem pages shouldn't be counted as free in this
+                * case, they can't be purged, only swapped out, and
+                * that won't affect the overall amount of available
+                * memory in the system.
+                */
+               free -= global_page_state(NR_SHMEM);
 
-               free = global_page_state(NR_FILE_PAGES);
                free += nr_swap_pages;
 
                /*
@@ -130,34 +144,18 @@ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
                free += global_page_state(NR_SLAB_RECLAIMABLE);
 
                /*
-                * Leave the last 3% for root
-                */
-               if (!cap_sys_admin)
-                       free -= free / 32;
-
-               if (free > pages)
-                       return 0;
-
-               /*
-                * nr_free_pages() is very expensive on large systems,
-                * only call if we're about to fail.
-                */
-               n = nr_free_pages();
-
-               /*
                 * Leave reserved pages. The pages are not for anonymous pages.
                 */
-               if (n <= totalreserve_pages)
+               if (free <= totalreserve_pages)
                        goto error;
                else
-                       n -= totalreserve_pages;
+                       free -= totalreserve_pages;
 
                /*
                 * Leave the last 3% for root
                 */
                if (!cap_sys_admin)
-                       n -= n / 32;
-               free += n;
+                       free -= free / 32;
 
                if (free > pages)
                        return 0;
@@ -188,7 +186,7 @@ error:
 }
 
 /*
- * Requires inode->i_mapping->i_mmap_lock
+ * Requires inode->i_mapping->i_mmap_mutex
  */
 static void __remove_shared_vm_struct(struct vm_area_struct *vma,
                struct file *file, struct address_space *mapping)
@@ -216,9 +214,9 @@ void unlink_file_vma(struct vm_area_struct *vma)
 
        if (file) {
                struct address_space *mapping = file->f_mapping;
-               spin_lock(&mapping->i_mmap_lock);
+               mutex_lock(&mapping->i_mmap_mutex);
                __remove_shared_vm_struct(vma, file, mapping);
-               spin_unlock(&mapping->i_mmap_lock);
+               mutex_unlock(&mapping->i_mmap_mutex);
        }
 }
 
@@ -252,7 +250,15 @@ SYSCALL_DEFINE1(brk, unsigned long, brk)
        down_write(&mm->mmap_sem);
 
 #ifdef CONFIG_COMPAT_BRK
-       min_brk = mm->end_code;
+       /*
+        * CONFIG_COMPAT_BRK can still be overridden by setting
+        * randomize_va_space to 2, which will still cause mm->start_brk
+        * to be arbitrarily shifted
+        */
+       if (current->brk_randomized)
+               min_brk = mm->start_brk;
+       else
+               min_brk = mm->end_data;
 #else
        min_brk = mm->start_brk;
 #endif
@@ -265,7 +271,7 @@ SYSCALL_DEFINE1(brk, unsigned long, brk)
         * segment grow beyond its set limit the in case where the limit is
         * not page aligned -Ram Gupta
         */
-       rlim = current->signal->rlim[RLIMIT_DATA].rlim_cur;
+       rlim = rlimit(RLIMIT_DATA);
        if (rlim < RLIM_INFINITY && (brk - mm->start_brk) +
                        (mm->end_data - mm->start_data) > rlim)
                goto out;
@@ -384,23 +390,6 @@ find_vma_prepare(struct mm_struct *mm, unsigned long addr,
        return vma;
 }
 
-static inline void
-__vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
-               struct vm_area_struct *prev, struct rb_node *rb_parent)
-{
-       if (prev) {
-               vma->vm_next = prev->vm_next;
-               prev->vm_next = vma;
-       } else {
-               mm->mmap = vma;
-               if (rb_parent)
-                       vma->vm_next = rb_entry(rb_parent,
-                                       struct vm_area_struct, vm_rb);
-               else
-                       vma->vm_next = NULL;
-       }
-}
-
 void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
                struct rb_node **rb_link, struct rb_node *rb_parent)
 {
@@ -437,7 +426,6 @@ __vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
 {
        __vma_link_list(mm, vma, prev, rb_parent);
        __vma_link_rb(mm, vma, rb_link, rb_parent);
-       __anon_vma_link(vma);
 }
 
 static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
@@ -449,18 +437,14 @@ static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
        if (vma->vm_file)
                mapping = vma->vm_file->f_mapping;
 
-       if (mapping) {
-               spin_lock(&mapping->i_mmap_lock);
-               vma->vm_truncate_count = mapping->truncate_count;
-       }
-       anon_vma_lock(vma);
+       if (mapping)
+               mutex_lock(&mapping->i_mmap_mutex);
 
        __vma_link(mm, vma, prev, rb_link, rb_parent);
        __vma_link_file(vma);
 
-       anon_vma_unlock(vma);
        if (mapping)
-               spin_unlock(&mapping->i_mmap_lock);
+               mutex_unlock(&mapping->i_mmap_mutex);
 
        mm->map_count++;
        validate_mm(mm);
@@ -486,7 +470,11 @@ static inline void
 __vma_unlink(struct mm_struct *mm, struct vm_area_struct *vma,
                struct vm_area_struct *prev)
 {
-       prev->vm_next = vma->vm_next;
+       struct vm_area_struct *next = vma->vm_next;
+
+       prev->vm_next = next;
+       if (next)
+               next->vm_prev = prev;
        rb_erase(&vma->vm_rb, &mm->mm_rb);
        if (mm->mmap_cache == vma)
                mm->mmap_cache = prev;
@@ -499,7 +487,7 @@ __vma_unlink(struct mm_struct *mm, struct vm_area_struct *vma,
  * are necessary.  The "insert" vma (if any) is to be inserted
  * before we drop the necessary locks.
  */
-void vma_adjust(struct vm_area_struct *vma, unsigned long start,
+int vma_adjust(struct vm_area_struct *vma, unsigned long start,
        unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
 {
        struct mm_struct *mm = vma->vm_mm;
@@ -507,12 +495,14 @@ void vma_adjust(struct vm_area_struct *vma, unsigned long start,
        struct vm_area_struct *importer = NULL;
        struct address_space *mapping = NULL;
        struct prio_tree_root *root = NULL;
-       struct file *file = vma->vm_file;
        struct anon_vma *anon_vma = NULL;
+       struct file *file = vma->vm_file;
        long adjust_next = 0;
        int remove_next = 0;
 
        if (next && !insert) {
+               struct vm_area_struct *exporter = NULL;
+
                if (end >= next->vm_end) {
                        /*
                         * vma expands, overlapping all the next, and
@@ -520,7 +510,7 @@ void vma_adjust(struct vm_area_struct *vma, unsigned long start,
                         */
 again:                 remove_next = 1 + (end > next->vm_end);
                        end = next->vm_end;
-                       anon_vma = next->anon_vma;
+                       exporter = next;
                        importer = vma;
                } else if (end > next->vm_start) {
                        /*
@@ -528,7 +518,7 @@ again:                      remove_next = 1 + (end > next->vm_end);
                         * mprotect case 5 shifting the boundary up.
                         */
                        adjust_next = (end - next->vm_start) >> PAGE_SHIFT;
-                       anon_vma = next->anon_vma;
+                       exporter = next;
                        importer = vma;
                } else if (end < vma->vm_end) {
                        /*
@@ -537,26 +527,28 @@ again:                    remove_next = 1 + (end > next->vm_end);
                         * mprotect case 4 shifting the boundary down.
                         */
                        adjust_next = - ((vma->vm_end - end) >> PAGE_SHIFT);
-                       anon_vma = next->anon_vma;
+                       exporter = vma;
                        importer = next;
                }
+
+               /*
+                * Easily overlooked: when mprotect shifts the boundary,
+                * make sure the expanding vma has anon_vma set if the
+                * shrinking vma had, to cover any anon pages imported.
+                */
+               if (exporter && exporter->anon_vma && !importer->anon_vma) {
+                       if (anon_vma_clone(importer, exporter))
+                               return -ENOMEM;
+                       importer->anon_vma = exporter->anon_vma;
+               }
        }
 
        if (file) {
                mapping = file->f_mapping;
                if (!(vma->vm_flags & VM_NONLINEAR))
                        root = &mapping->i_mmap;
-               spin_lock(&mapping->i_mmap_lock);
-               if (importer &&
-                   vma->vm_truncate_count != next->vm_truncate_count) {
-                       /*
-                        * unmap_mapping_range might be in progress:
-                        * ensure that the expanding vma is rescanned.
-                        */
-                       importer->vm_truncate_count = 0;
-               }
+               mutex_lock(&mapping->i_mmap_mutex);
                if (insert) {
-                       insert->vm_truncate_count = vma->vm_truncate_count;
                        /*
                         * Put into prio_tree now, so instantiated pages
                         * are visible to arm/parisc __flush_dcache_page
@@ -567,23 +559,17 @@ again:                    remove_next = 1 + (end > next->vm_end);
                }
        }
 
+       vma_adjust_trans_huge(vma, start, end, adjust_next);
+
        /*
-        * When changing only vma->vm_end, we don't really need
-        * anon_vma lock.
+        * When changing only vma->vm_end, we don't really need anon_vma
+        * lock. This is a fairly rare case by itself, but the anon_vma
+        * lock may be shared between many sibling processes.  Skipping
+        * the lock for brk adjustments makes a difference sometimes.
         */
-       if (vma->anon_vma && (insert || importer || start != vma->vm_start))
+       if (vma->anon_vma && (importer || start != vma->vm_start)) {
                anon_vma = vma->anon_vma;
-       if (anon_vma) {
-               spin_lock(&anon_vma->lock);
-               /*
-                * Easily overlooked: when mprotect shifts the boundary,
-                * make sure the expanding vma has anon_vma set if the
-                * shrinking vma had, to cover any anon pages imported.
-                */
-               if (importer && !importer->anon_vma) {
-                       importer->anon_vma = anon_vma;
-                       __anon_vma_link(importer);
-               }
+               anon_vma_lock(anon_vma);
        }
 
        if (root) {
@@ -616,8 +602,6 @@ again:                      remove_next = 1 + (end > next->vm_end);
                __vma_unlink(mm, next, vma);
                if (file)
                        __remove_shared_vm_struct(next, file, mapping);
-               if (next->anon_vma)
-                       __anon_vma_merge(vma, next);
        } else if (insert) {
                /*
                 * split_vma has split insert from vma, and needs
@@ -628,9 +612,9 @@ again:                      remove_next = 1 + (end > next->vm_end);
        }
 
        if (anon_vma)
-               spin_unlock(&anon_vma->lock);
+               anon_vma_unlock(anon_vma);
        if (mapping)
-               spin_unlock(&mapping->i_mmap_lock);
+               mutex_unlock(&mapping->i_mmap_mutex);
 
        if (remove_next) {
                if (file) {
@@ -638,6 +622,8 @@ again:                      remove_next = 1 + (end > next->vm_end);
                        if (next->vm_flags & VM_EXECUTABLE)
                                removed_exe_file_vma(mm);
                }
+               if (next->anon_vma)
+                       anon_vma_merge(vma, next);
                mm->map_count--;
                mpol_put(vma_policy(next));
                kmem_cache_free(vm_area_cachep, next);
@@ -653,6 +639,8 @@ again:                      remove_next = 1 + (end > next->vm_end);
        }
 
        validate_mm(mm);
+
+       return 0;
 }
 
 /*
@@ -673,9 +661,17 @@ static inline int is_mergeable_vma(struct vm_area_struct *vma,
 }
 
 static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1,
-                                       struct anon_vma *anon_vma2)
+                                       struct anon_vma *anon_vma2,
+                                       struct vm_area_struct *vma)
 {
-       return !anon_vma1 || !anon_vma2 || (anon_vma1 == anon_vma2);
+       /*
+        * The list_is_singular() test is to avoid merging VMA cloned from
+        * parents. This can improve scalability caused by anon_vma lock.
+        */
+       if ((!anon_vma1 || !anon_vma2) && (!vma ||
+               list_is_singular(&vma->anon_vma_chain)))
+               return 1;
+       return anon_vma1 == anon_vma2;
 }
 
 /*
@@ -694,7 +690,7 @@ can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
        struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
 {
        if (is_mergeable_vma(vma, file, vm_flags) &&
-           is_mergeable_anon_vma(anon_vma, vma->anon_vma)) {
+           is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
                if (vma->vm_pgoff == vm_pgoff)
                        return 1;
        }
@@ -713,7 +709,7 @@ can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
        struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
 {
        if (is_mergeable_vma(vma, file, vm_flags) &&
-           is_mergeable_anon_vma(anon_vma, vma->anon_vma)) {
+           is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
                pgoff_t vm_pglen;
                vm_pglen = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
                if (vma->vm_pgoff + vm_pglen == vm_pgoff)
@@ -759,6 +755,7 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
 {
        pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
        struct vm_area_struct *area, *next;
+       int err;
 
        /*
         * We later require that vma->vm_flags == vm_flags,
@@ -790,13 +787,16 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
                                can_vma_merge_before(next, vm_flags,
                                        anon_vma, file, pgoff+pglen) &&
                                is_mergeable_anon_vma(prev->anon_vma,
-                                                     next->anon_vma)) {
+                                                     next->anon_vma, NULL)) {
                                                        /* cases 1, 6 */
-                       vma_adjust(prev, prev->vm_start,
+                       err = vma_adjust(prev, prev->vm_start,
                                next->vm_end, prev->vm_pgoff, NULL);
                } else                                  /* cases 2, 5, 7 */
-                       vma_adjust(prev, prev->vm_start,
+                       err = vma_adjust(prev, prev->vm_start,
                                end, prev->vm_pgoff, NULL);
+               if (err)
+                       return NULL;
+               khugepaged_enter_vma_merge(prev);
                return prev;
        }
 
@@ -808,11 +808,14 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
                        can_vma_merge_before(next, vm_flags,
                                        anon_vma, file, pgoff+pglen)) {
                if (prev && addr < prev->vm_end)        /* case 4 */
-                       vma_adjust(prev, prev->vm_start,
+                       err = vma_adjust(prev, prev->vm_start,
                                addr, prev->vm_pgoff, NULL);
                else                                    /* cases 3, 8 */
-                       vma_adjust(area, addr, next->vm_end,
+                       err = vma_adjust(area, addr, next->vm_end,
                                next->vm_pgoff - pglen, NULL);
+               if (err)
+                       return NULL;
+               khugepaged_enter_vma_merge(area);
                return area;
        }
 
@@ -820,6 +823,61 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
 }
 
 /*
+ * Rough compatbility check to quickly see if it's even worth looking
+ * at sharing an anon_vma.
+ *
+ * They need to have the same vm_file, and the flags can only differ
+ * in things that mprotect may change.
+ *
+ * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
+ * we can merge the two vma's. For example, we refuse to merge a vma if
+ * there is a vm_ops->close() function, because that indicates that the
+ * driver is doing some kind of reference counting. But that doesn't
+ * really matter for the anon_vma sharing case.
+ */
+static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
+{
+       return a->vm_end == b->vm_start &&
+               mpol_equal(vma_policy(a), vma_policy(b)) &&
+               a->vm_file == b->vm_file &&
+               !((a->vm_flags ^ b->vm_flags) & ~(VM_READ|VM_WRITE|VM_EXEC)) &&
+               b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
+}
+
+/*
+ * Do some basic sanity checking to see if we can re-use the anon_vma
+ * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
+ * the same as 'old', the other will be the new one that is trying
+ * to share the anon_vma.
+ *
+ * NOTE! This runs with mm_sem held for reading, so it is possible that
+ * the anon_vma of 'old' is concurrently in the process of being set up
+ * by another page fault trying to merge _that_. But that's ok: if it
+ * is being set up, that automatically means that it will be a singleton
+ * acceptable for merging, so we can do all of this optimistically. But
+ * we do that ACCESS_ONCE() to make sure that we never re-load the pointer.
+ *
+ * IOW: that the "list_is_singular()" test on the anon_vma_chain only
+ * matters for the 'stable anon_vma' case (ie the thing we want to avoid
+ * is to return an anon_vma that is "complex" due to having gone through
+ * a fork).
+ *
+ * We also make sure that the two vma's are compatible (adjacent,
+ * and with the same memory policies). That's all stable, even with just
+ * a read lock on the mm_sem.
+ */
+static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b)
+{
+       if (anon_vma_compatible(a, b)) {
+               struct anon_vma *anon_vma = ACCESS_ONCE(old->anon_vma);
+
+               if (anon_vma && list_is_singular(&old->anon_vma_chain))
+                       return anon_vma;
+       }
+       return NULL;
+}
+
+/*
  * find_mergeable_anon_vma is used by anon_vma_prepare, to check
  * neighbouring vmas for a suitable anon_vma, before it goes off
  * to allocate a new anon_vma.  It checks because a repetitive
@@ -829,48 +887,24 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
  */
 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
 {
+       struct anon_vma *anon_vma;
        struct vm_area_struct *near;
-       unsigned long vm_flags;
 
        near = vma->vm_next;
        if (!near)
                goto try_prev;
 
-       /*
-        * Since only mprotect tries to remerge vmas, match flags
-        * which might be mprotected into each other later on.
-        * Neither mlock nor madvise tries to remerge at present,
-        * so leave their flags as obstructing a merge.
-        */
-       vm_flags = vma->vm_flags & ~(VM_READ|VM_WRITE|VM_EXEC);
-       vm_flags |= near->vm_flags & (VM_READ|VM_WRITE|VM_EXEC);
-
-       if (near->anon_vma && vma->vm_end == near->vm_start &&
-                       mpol_equal(vma_policy(vma), vma_policy(near)) &&
-                       can_vma_merge_before(near, vm_flags,
-                               NULL, vma->vm_file, vma->vm_pgoff +
-                               ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT)))
-               return near->anon_vma;
+       anon_vma = reusable_anon_vma(near, vma, near);
+       if (anon_vma)
+               return anon_vma;
 try_prev:
-       /*
-        * It is potentially slow to have to call find_vma_prev here.
-        * But it's only on the first write fault on the vma, not
-        * every time, and we could devise a way to avoid it later
-        * (e.g. stash info in next's anon_vma_node when assigning
-        * an anon_vma, or when trying vma_merge).  Another time.
-        */
-       BUG_ON(find_vma_prev(vma->vm_mm, vma->vm_start, &near) != vma);
+       near = vma->vm_prev;
        if (!near)
                goto none;
 
-       vm_flags = vma->vm_flags & ~(VM_READ|VM_WRITE|VM_EXEC);
-       vm_flags |= near->vm_flags & (VM_READ|VM_WRITE|VM_EXEC);
-
-       if (near->anon_vma && near->vm_end == vma->vm_start &&
-                       mpol_equal(vma_policy(near), vma_policy(vma)) &&
-                       can_vma_merge_after(near, vm_flags,
-                               NULL, vma->vm_file, vma->vm_pgoff))
-               return near->anon_vma;
+       anon_vma = reusable_anon_vma(near, near, vma);
+       if (anon_vma)
+               return anon_vma;
 none:
        /*
         * There's no absolute need to look only at touching neighbours:
@@ -911,7 +945,7 @@ unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
 {
        struct mm_struct * mm = current->mm;
        struct inode *inode;
-       unsigned int vm_flags;
+       vm_flags_t vm_flags;
        int error;
        unsigned long reqprot = prot;
 
@@ -967,7 +1001,7 @@ unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
                unsigned long locked, lock_limit;
                locked = len >> PAGE_SHIFT;
                locked += mm->locked_vm;
-               lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
+               lock_limit = rlimit(RLIMIT_MEMLOCK);
                lock_limit >>= PAGE_SHIFT;
                if (locked > lock_limit && !capable(CAP_IPC_LOCK))
                        return -EAGAIN;
@@ -1051,6 +1085,7 @@ SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
        unsigned long retval = -EBADF;
 
        if (!(flags & MAP_ANONYMOUS)) {
+               audit_mmap_fd(fd, flags);
                if (unlikely(flags & MAP_HUGETLB))
                        return -EINVAL;
                file = fget(fd);
@@ -1083,6 +1118,30 @@ out:
        return retval;
 }
 
+#ifdef __ARCH_WANT_SYS_OLD_MMAP
+struct mmap_arg_struct {
+       unsigned long addr;
+       unsigned long len;
+       unsigned long prot;
+       unsigned long flags;
+       unsigned long fd;
+       unsigned long offset;
+};
+
+SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
+{
+       struct mmap_arg_struct a;
+
+       if (copy_from_user(&a, arg, sizeof(a)))
+               return -EFAULT;
+       if (a.offset & ~PAGE_MASK)
+               return -EINVAL;
+
+       return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
+                             a.offset >> PAGE_SHIFT);
+}
+#endif /* __ARCH_WANT_SYS_OLD_MMAP */
+
 /*
  * Some shared mappigns will want the pages marked read-only
  * to track write events. If so, we'll downgrade vm_page_prot
@@ -1091,7 +1150,7 @@ out:
  */
 int vma_wants_writenotify(struct vm_area_struct *vma)
 {
-       unsigned int vm_flags = vma->vm_flags;
+       vm_flags_t vm_flags = vma->vm_flags;
 
        /* If it was private or non-writable, the write bit is already clear */
        if ((vm_flags & (VM_WRITE|VM_SHARED)) != ((VM_WRITE|VM_SHARED)))
@@ -1119,7 +1178,7 @@ int vma_wants_writenotify(struct vm_area_struct *vma)
  * We account for memory if it's a private writeable mapping,
  * not hugepages and VM_NORESERVE wasn't set.
  */
-static inline int accountable_mapping(struct file *file, unsigned int vm_flags)
+static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags)
 {
        /*
         * hugetlb has its own accounting separate from the core VM
@@ -1133,7 +1192,7 @@ static inline int accountable_mapping(struct file *file, unsigned int vm_flags)
 
 unsigned long mmap_region(struct file *file, unsigned long addr,
                          unsigned long len, unsigned long flags,
-                         unsigned int vm_flags, unsigned long pgoff)
+                         vm_flags_t vm_flags, unsigned long pgoff)
 {
        struct mm_struct *mm = current->mm;
        struct vm_area_struct *vma, *prev;
@@ -1205,6 +1264,7 @@ munmap_back:
        vma->vm_flags = vm_flags;
        vma->vm_page_prot = vm_get_page_prot(vm_flags);
        vma->vm_pgoff = pgoff;
+       INIT_LIST_HEAD(&vma->anon_vma_chain);
 
        if (file) {
                error = -EINVAL;
@@ -1265,13 +1325,8 @@ out:
        mm->total_vm += len >> PAGE_SHIFT;
        vm_stat_account(mm, vm_flags, file, len >> PAGE_SHIFT);
        if (vm_flags & VM_LOCKED) {
-               /*
-                * makes pages present; downgrades, drops, reacquires mmap_sem
-                */
-               long nr_pages = mlock_vma_pages_range(vma, addr, addr + len);
-               if (nr_pages < 0)
-                       return nr_pages;        /* vma gone! */
-               mm->locked_vm += (len >> PAGE_SHIFT) - nr_pages;
+               if (!mlock_vma_pages_range(vma, addr, addr + len))
+                       mm->locked_vm += (len >> PAGE_SHIFT);
        } else if ((flags & MAP_POPULATE) && !(flags & MAP_NONBLOCK))
                make_pages_present(addr, addr + len);
        return addr;
@@ -1599,7 +1654,7 @@ static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, uns
                return -ENOMEM;
 
        /* Stack limit test */
-       if (size > rlim[RLIMIT_STACK].rlim_cur)
+       if (size > ACCESS_ONCE(rlim[RLIMIT_STACK].rlim_cur))
                return -ENOMEM;
 
        /* mlock limit tests */
@@ -1607,7 +1662,8 @@ static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, uns
                unsigned long locked;
                unsigned long limit;
                locked = mm->locked_vm + grow;
-               limit = rlim[RLIMIT_MEMLOCK].rlim_cur >> PAGE_SHIFT;
+               limit = ACCESS_ONCE(rlim[RLIMIT_MEMLOCK].rlim_cur);
+               limit >>= PAGE_SHIFT;
                if (locked > limit && !capable(CAP_IPC_LOCK))
                        return -ENOMEM;
        }
@@ -1638,9 +1694,6 @@ static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, uns
  * PA-RISC uses this for its stack; IA64 for its Register Backing Store.
  * vma is the last one with address > vma->vm_end.  Have to extend vma.
  */
-#ifndef CONFIG_IA64
-static
-#endif
 int expand_upwards(struct vm_area_struct *vma, unsigned long address)
 {
        int error;
@@ -1654,7 +1707,7 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address)
         */
        if (unlikely(anon_vma_prepare(vma)))
                return -ENOMEM;
-       anon_vma_lock(vma);
+       vma_lock_anon_vma(vma);
 
        /*
         * vma->vm_start/vm_end cannot change under us because the caller
@@ -1665,7 +1718,7 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address)
        if (address < PAGE_ALIGN(address+4))
                address = PAGE_ALIGN(address+4);
        else {
-               anon_vma_unlock(vma);
+               vma_unlock_anon_vma(vma);
                return -ENOMEM;
        }
        error = 0;
@@ -1677,11 +1730,17 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address)
                size = address - vma->vm_start;
                grow = (address - vma->vm_end) >> PAGE_SHIFT;
 
-               error = acct_stack_growth(vma, size, grow);
-               if (!error)
-                       vma->vm_end = address;
+               error = -ENOMEM;
+               if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) {
+                       error = acct_stack_growth(vma, size, grow);
+                       if (!error) {
+                               vma->vm_end = address;
+                               perf_event_mmap(vma);
+                       }
+               }
        }
-       anon_vma_unlock(vma);
+       vma_unlock_anon_vma(vma);
+       khugepaged_enter_vma_merge(vma);
        return error;
 }
 #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */
@@ -1689,7 +1748,7 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address)
 /*
  * vma is the first one with address < vma->vm_start.  Have to extend vma.
  */
-static int expand_downwards(struct vm_area_struct *vma,
+int expand_downwards(struct vm_area_struct *vma,
                                   unsigned long address)
 {
        int error;
@@ -1706,7 +1765,7 @@ static int expand_downwards(struct vm_area_struct *vma,
        if (error)
                return error;
 
-       anon_vma_lock(vma);
+       vma_lock_anon_vma(vma);
 
        /*
         * vma->vm_start/vm_end cannot change under us because the caller
@@ -1721,21 +1780,21 @@ static int expand_downwards(struct vm_area_struct *vma,
                size = vma->vm_end - address;
                grow = (vma->vm_start - address) >> PAGE_SHIFT;
 
-               error = acct_stack_growth(vma, size, grow);
-               if (!error) {
-                       vma->vm_start = address;
-                       vma->vm_pgoff -= grow;
+               error = -ENOMEM;
+               if (grow <= vma->vm_pgoff) {
+                       error = acct_stack_growth(vma, size, grow);
+                       if (!error) {
+                               vma->vm_start = address;
+                               vma->vm_pgoff -= grow;
+                               perf_event_mmap(vma);
+                       }
                }
        }
-       anon_vma_unlock(vma);
+       vma_unlock_anon_vma(vma);
+       khugepaged_enter_vma_merge(vma);
        return error;
 }
 
-int expand_stack_downwards(struct vm_area_struct *vma, unsigned long address)
-{
-       return expand_downwards(vma, address);
-}
-
 #ifdef CONFIG_STACK_GROWSUP
 int expand_stack(struct vm_area_struct *vma, unsigned long address)
 {
@@ -1754,8 +1813,7 @@ find_extend_vma(struct mm_struct *mm, unsigned long addr)
        if (!prev || expand_stack(prev, addr))
                return NULL;
        if (prev->vm_flags & VM_LOCKED) {
-               if (mlock_vma_pages_range(prev, addr, prev->vm_end) < 0)
-                       return NULL;    /* vma gone! */
+               mlock_vma_pages_range(prev, addr, prev->vm_end);
        }
        return prev;
 }
@@ -1783,8 +1841,7 @@ find_extend_vma(struct mm_struct * mm, unsigned long addr)
        if (expand_stack(vma, addr))
                return NULL;
        if (vma->vm_flags & VM_LOCKED) {
-               if (mlock_vma_pages_range(vma, addr, start) < 0)
-                       return NULL;    /* vma gone! */
+               mlock_vma_pages_range(vma, addr, start);
        }
        return vma;
 }
@@ -1820,17 +1877,17 @@ static void unmap_region(struct mm_struct *mm,
                unsigned long start, unsigned long end)
 {
        struct vm_area_struct *next = prev? prev->vm_next: mm->mmap;
-       struct mmu_gather *tlb;
+       struct mmu_gather tlb;
        unsigned long nr_accounted = 0;
 
        lru_add_drain();
-       tlb = tlb_gather_mmu(mm, 0);
+       tlb_gather_mmu(&tlb, mm, 0);
        update_hiwater_rss(mm);
        unmap_vmas(&tlb, vma, start, end, &nr_accounted, NULL);
        vm_unacct_memory(nr_accounted);
-       free_pgtables(tlb, vma, prev? prev->vm_end: FIRST_USER_ADDRESS,
-                                next? next->vm_start: 0);
-       tlb_finish_mmu(tlb, start, end);
+       free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
+                                next ? next->vm_start : 0);
+       tlb_finish_mmu(&tlb, start, end);
 }
 
 /*
@@ -1846,6 +1903,7 @@ detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
        unsigned long addr;
 
        insertion_point = (prev ? &prev->vm_next : &mm->mmap);
+       vma->vm_prev = NULL;
        do {
                rb_erase(&vma->vm_rb, &mm->mm_rb);
                mm->map_count--;
@@ -1853,6 +1911,8 @@ detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
                vma = vma->vm_next;
        } while (vma && vma->vm_start < end);
        *insertion_point = vma;
+       if (vma)
+               vma->vm_prev = prev;
        tail_vma->vm_next = NULL;
        if (mm->unmap_area == arch_unmap_area)
                addr = prev ? prev->vm_end : mm->mmap_base;
@@ -1871,6 +1931,7 @@ static int __split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
 {
        struct mempolicy *pol;
        struct vm_area_struct *new;
+       int err = -ENOMEM;
 
        if (is_vm_hugetlb_page(vma) && (addr &
                                        ~(huge_page_mask(hstate_vma(vma)))))
@@ -1878,11 +1939,13 @@ static int __split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
 
        new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
        if (!new)
-               return -ENOMEM;
+               goto out_err;
 
        /* most fields are the same, copy all, and then fixup */
        *new = *vma;
 
+       INIT_LIST_HEAD(&new->anon_vma_chain);
+
        if (new_below)
                new->vm_end = addr;
        else {
@@ -1892,11 +1955,14 @@ static int __split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
 
        pol = mpol_dup(vma_policy(vma));
        if (IS_ERR(pol)) {
-               kmem_cache_free(vm_area_cachep, new);
-               return PTR_ERR(pol);
+               err = PTR_ERR(pol);
+               goto out_free_vma;
        }
        vma_set_policy(new, pol);
 
+       if (anon_vma_clone(new, vma))
+               goto out_free_mpol;
+
        if (new->vm_file) {
                get_file(new->vm_file);
                if (vma->vm_flags & VM_EXECUTABLE)
@@ -1907,12 +1973,30 @@ static int __split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
                new->vm_ops->open(new);
 
        if (new_below)
-               vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
+               err = vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
                        ((addr - new->vm_start) >> PAGE_SHIFT), new);
        else
-               vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
+               err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
 
-       return 0;
+       /* Success. */
+       if (!err)
+               return 0;
+
+       /* Clean everything up if vma_adjust failed. */
+       if (new->vm_ops && new->vm_ops->close)
+               new->vm_ops->close(new);
+       if (new->vm_file) {
+               if (vma->vm_flags & VM_EXECUTABLE)
+                       removed_exe_file_vma(mm);
+               fput(new->vm_file);
+       }
+       unlink_anon_vmas(new);
+ out_free_mpol:
+       mpol_put(pol);
+ out_free_vma:
+       kmem_cache_free(vm_area_cachep, new);
+ out_err:
+       return err;
 }
 
 /*
@@ -1945,9 +2029,10 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
                return -EINVAL;
 
        /* Find the first overlapping VMA */
-       vma = find_vma_prev(mm, start, &prev);
+       vma = find_vma(mm, start);
        if (!vma)
                return 0;
+       prev = vma->vm_prev;
        /* we have  start < vma->vm_end  */
 
        /* if it doesn't overlap, we have nothing.. */
@@ -2074,7 +2159,7 @@ unsigned long do_brk(unsigned long addr, unsigned long len)
                unsigned long locked, lock_limit;
                locked = len >> PAGE_SHIFT;
                locked += mm->locked_vm;
-               lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
+               lock_limit = rlimit(RLIMIT_MEMLOCK);
                lock_limit >>= PAGE_SHIFT;
                if (locked > lock_limit && !capable(CAP_IPC_LOCK))
                        return -EAGAIN;
@@ -2122,6 +2207,7 @@ unsigned long do_brk(unsigned long addr, unsigned long len)
                return -ENOMEM;
        }
 
+       INIT_LIST_HEAD(&vma->anon_vma_chain);
        vma->vm_mm = mm;
        vma->vm_start = addr;
        vma->vm_end = addr + len;
@@ -2130,6 +2216,7 @@ unsigned long do_brk(unsigned long addr, unsigned long len)
        vma->vm_page_prot = vm_get_page_prot(flags);
        vma_link(mm, vma, prev, rb_link, rb_parent);
 out:
+       perf_event_mmap(vma);
        mm->total_vm += len >> PAGE_SHIFT;
        if (flags & VM_LOCKED) {
                if (!mlock_vma_pages_range(vma, addr, addr + len))
@@ -2143,7 +2230,7 @@ EXPORT_SYMBOL(do_brk);
 /* Release all mmaps. */
 void exit_mmap(struct mm_struct *mm)
 {
-       struct mmu_gather *tlb;
+       struct mmu_gather tlb;
        struct vm_area_struct *vma;
        unsigned long nr_accounted = 0;
        unsigned long end;
@@ -2168,14 +2255,14 @@ void exit_mmap(struct mm_struct *mm)
 
        lru_add_drain();
        flush_cache_mm(mm);
-       tlb = tlb_gather_mmu(mm, 1);
+       tlb_gather_mmu(&tlb, mm, 1);
        /* update_hiwater_rss(mm) here? but nobody should be looking */
        /* Use -1 here to ensure all VMAs in the mm are unmapped */
        end = unmap_vmas(&tlb, vma, 0, -1, &nr_accounted, NULL);
        vm_unacct_memory(nr_accounted);
 
-       free_pgtables(tlb, vma, FIRST_USER_ADDRESS, 0);
-       tlb_finish_mmu(tlb, 0, end);
+       free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, 0);
+       tlb_finish_mmu(&tlb, 0, end);
 
        /*
         * Walk the list again, actually closing and freeing it,
@@ -2189,7 +2276,7 @@ void exit_mmap(struct mm_struct *mm)
 
 /* Insert vm structure into process list sorted by address
  * and into the inode's i_mmap tree.  If vm_file is non-NULL
- * then i_mmap_lock is taken here.
+ * then i_mmap_mutex is taken here.
  */
 int insert_vm_struct(struct mm_struct * mm, struct vm_area_struct * vma)
 {
@@ -2258,10 +2345,11 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
                if (new_vma) {
                        *new_vma = *vma;
                        pol = mpol_dup(vma_policy(vma));
-                       if (IS_ERR(pol)) {
-                               kmem_cache_free(vm_area_cachep, new_vma);
-                               return NULL;
-                       }
+                       if (IS_ERR(pol))
+                               goto out_free_vma;
+                       INIT_LIST_HEAD(&new_vma->anon_vma_chain);
+                       if (anon_vma_clone(new_vma, vma))
+                               goto out_free_mempol;
                        vma_set_policy(new_vma, pol);
                        new_vma->vm_start = addr;
                        new_vma->vm_end = addr + len;
@@ -2277,6 +2365,12 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
                }
        }
        return new_vma;
+
+ out_free_mempol:
+       mpol_put(pol);
+ out_free_vma:
+       kmem_cache_free(vm_area_cachep, new_vma);
+       return NULL;
 }
 
 /*
@@ -2288,7 +2382,7 @@ int may_expand_vm(struct mm_struct *mm, unsigned long npages)
        unsigned long cur = mm->total_vm;       /* pages */
        unsigned long lim;
 
-       lim = current->signal->rlim[RLIMIT_AS].rlim_cur >> PAGE_SHIFT;
+       lim = rlimit(RLIMIT_AS) >> PAGE_SHIFT;
 
        if (cur + npages > lim)
                return 0;
@@ -2348,12 +2442,14 @@ int install_special_mapping(struct mm_struct *mm,
                            unsigned long addr, unsigned long len,
                            unsigned long vm_flags, struct page **pages)
 {
+       int ret;
        struct vm_area_struct *vma;
 
        vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
        if (unlikely(vma == NULL))
                return -ENOMEM;
 
+       INIT_LIST_HEAD(&vma->anon_vma_chain);
        vma->vm_mm = mm;
        vma->vm_start = addr;
        vma->vm_end = addr + len;
@@ -2364,39 +2460,46 @@ int install_special_mapping(struct mm_struct *mm,
        vma->vm_ops = &special_mapping_vmops;
        vma->vm_private_data = pages;
 
-       if (unlikely(insert_vm_struct(mm, vma))) {
-               kmem_cache_free(vm_area_cachep, vma);
-               return -ENOMEM;
-       }
+       ret = security_file_mmap(NULL, 0, 0, 0, vma->vm_start, 1);
+       if (ret)
+               goto out;
+
+       ret = insert_vm_struct(mm, vma);
+       if (ret)
+               goto out;
 
        mm->total_vm += len >> PAGE_SHIFT;
 
        perf_event_mmap(vma);
 
        return 0;
+
+out:
+       kmem_cache_free(vm_area_cachep, vma);
+       return ret;
 }
 
 static DEFINE_MUTEX(mm_all_locks_mutex);
 
 static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
 {
-       if (!test_bit(0, (unsigned long *) &anon_vma->head.next)) {
+       if (!test_bit(0, (unsigned long *) &anon_vma->root->head.next)) {
                /*
                 * The LSB of head.next can't change from under us
                 * because we hold the mm_all_locks_mutex.
                 */
-               spin_lock_nest_lock(&anon_vma->lock, &mm->mmap_sem);
+               mutex_lock_nest_lock(&anon_vma->root->mutex, &mm->mmap_sem);
                /*
                 * We can safely modify head.next after taking the
-                * anon_vma->lock. If some other vma in this mm shares
+                * anon_vma->root->mutex. If some other vma in this mm shares
                 * the same anon_vma we won't take it again.
                 *
                 * No need of atomic instructions here, head.next
                 * can't change from under us thanks to the
-                * anon_vma->lock.
+                * anon_vma->root->mutex.
                 */
                if (__test_and_set_bit(0, (unsigned long *)
-                                      &anon_vma->head.next))
+                                      &anon_vma->root->head.next))
                        BUG();
        }
 }
@@ -2415,7 +2518,7 @@ static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
                 */
                if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
                        BUG();
-               spin_lock_nest_lock(&mapping->i_mmap_lock, &mm->mmap_sem);
+               mutex_lock_nest_lock(&mapping->i_mmap_mutex, &mm->mmap_sem);
        }
 }
 
@@ -2442,7 +2545,7 @@ static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
  * vma in this mm is backed by the same anon_vma or address_space.
  *
  * We can take all the locks in random order because the VM code
- * taking i_mmap_lock or anon_vma->lock outside the mmap_sem never
+ * taking i_mmap_mutex or anon_vma->mutex outside the mmap_sem never
  * takes more than one of them in a row. Secondly we're protected
  * against a concurrent mm_take_all_locks() by the mm_all_locks_mutex.
  *
@@ -2454,6 +2557,7 @@ static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
 int mm_take_all_locks(struct mm_struct *mm)
 {
        struct vm_area_struct *vma;
+       struct anon_vma_chain *avc;
        int ret = -EINTR;
 
        BUG_ON(down_read_trylock(&mm->mmap_sem));
@@ -2471,7 +2575,8 @@ int mm_take_all_locks(struct mm_struct *mm)
                if (signal_pending(current))
                        goto out_unlock;
                if (vma->anon_vma)
-                       vm_lock_anon_vma(mm, vma->anon_vma);
+                       list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
+                               vm_lock_anon_vma(mm, avc->anon_vma);
        }
 
        ret = 0;
@@ -2485,7 +2590,7 @@ out_unlock:
 
 static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
 {
-       if (test_bit(0, (unsigned long *) &anon_vma->head.next)) {
+       if (test_bit(0, (unsigned long *) &anon_vma->root->head.next)) {
                /*
                 * The LSB of head.next can't change to 0 from under
                 * us because we hold the mm_all_locks_mutex.
@@ -2496,12 +2601,12 @@ static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
                 *
                 * No need of atomic instructions here, head.next
                 * can't change from under us until we release the
-                * anon_vma->lock.
+                * anon_vma->root->mutex.
                 */
                if (!__test_and_clear_bit(0, (unsigned long *)
-                                         &anon_vma->head.next))
+                                         &anon_vma->root->head.next))
                        BUG();
-               spin_unlock(&anon_vma->lock);
+               anon_vma_unlock(anon_vma);
        }
 }
 
@@ -2512,7 +2617,7 @@ static void vm_unlock_mapping(struct address_space *mapping)
                 * AS_MM_ALL_LOCKS can't change to 0 from under us
                 * because we hold the mm_all_locks_mutex.
                 */
-               spin_unlock(&mapping->i_mmap_lock);
+               mutex_unlock(&mapping->i_mmap_mutex);
                if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
                                        &mapping->flags))
                        BUG();
@@ -2526,13 +2631,15 @@ static void vm_unlock_mapping(struct address_space *mapping)
 void mm_drop_all_locks(struct mm_struct *mm)
 {
        struct vm_area_struct *vma;
+       struct anon_vma_chain *avc;
 
        BUG_ON(down_read_trylock(&mm->mmap_sem));
        BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
 
        for (vma = mm->mmap; vma; vma = vma->vm_next) {
                if (vma->anon_vma)
-                       vm_unlock_anon_vma(vma->anon_vma);
+                       list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
+                               vm_unlock_anon_vma(avc->anon_vma);
                if (vma->vm_file && vma->vm_file->f_mapping)
                        vm_unlock_mapping(vma->vm_file->f_mapping);
        }