thp: add compound_trans_head() helper
[linux-2.6.git] / include / linux / huge_mm.h
1 #ifndef _LINUX_HUGE_MM_H
2 #define _LINUX_HUGE_MM_H
3
4 extern int do_huge_pmd_anonymous_page(struct mm_struct *mm,
5                                       struct vm_area_struct *vma,
6                                       unsigned long address, pmd_t *pmd,
7                                       unsigned int flags);
8 extern int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
9                          pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
10                          struct vm_area_struct *vma);
11 extern int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
12                                unsigned long address, pmd_t *pmd,
13                                pmd_t orig_pmd);
14 extern pgtable_t get_pmd_huge_pte(struct mm_struct *mm);
15 extern struct page *follow_trans_huge_pmd(struct mm_struct *mm,
16                                           unsigned long addr,
17                                           pmd_t *pmd,
18                                           unsigned int flags);
19 extern int zap_huge_pmd(struct mmu_gather *tlb,
20                         struct vm_area_struct *vma,
21                         pmd_t *pmd);
22 extern int mincore_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
23                         unsigned long addr, unsigned long end,
24                         unsigned char *vec);
25 extern int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
26                         unsigned long addr, pgprot_t newprot);
27
28 enum transparent_hugepage_flag {
29         TRANSPARENT_HUGEPAGE_FLAG,
30         TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
31         TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
32         TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
33         TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG,
34 #ifdef CONFIG_DEBUG_VM
35         TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG,
36 #endif
37 };
38
39 enum page_check_address_pmd_flag {
40         PAGE_CHECK_ADDRESS_PMD_FLAG,
41         PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG,
42         PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG,
43 };
44 extern pmd_t *page_check_address_pmd(struct page *page,
45                                      struct mm_struct *mm,
46                                      unsigned long address,
47                                      enum page_check_address_pmd_flag flag);
48
49 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
50 #define HPAGE_PMD_SHIFT HPAGE_SHIFT
51 #define HPAGE_PMD_MASK HPAGE_MASK
52 #define HPAGE_PMD_SIZE HPAGE_SIZE
53
54 #define transparent_hugepage_enabled(__vma)                             \
55         ((transparent_hugepage_flags &                                  \
56           (1<<TRANSPARENT_HUGEPAGE_FLAG) ||                             \
57           (transparent_hugepage_flags &                                 \
58            (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG) &&                   \
59            ((__vma)->vm_flags & VM_HUGEPAGE))) &&                       \
60          !((__vma)->vm_flags & VM_NOHUGEPAGE))
61 #define transparent_hugepage_defrag(__vma)                              \
62         ((transparent_hugepage_flags &                                  \
63           (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)) ||                     \
64          (transparent_hugepage_flags &                                  \
65           (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG) &&             \
66           (__vma)->vm_flags & VM_HUGEPAGE))
67 #ifdef CONFIG_DEBUG_VM
68 #define transparent_hugepage_debug_cow()                                \
69         (transparent_hugepage_flags &                                   \
70          (1<<TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG))
71 #else /* CONFIG_DEBUG_VM */
72 #define transparent_hugepage_debug_cow() 0
73 #endif /* CONFIG_DEBUG_VM */
74
75 extern unsigned long transparent_hugepage_flags;
76 extern int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
77                           pmd_t *dst_pmd, pmd_t *src_pmd,
78                           struct vm_area_struct *vma,
79                           unsigned long addr, unsigned long end);
80 extern int handle_pte_fault(struct mm_struct *mm,
81                             struct vm_area_struct *vma, unsigned long address,
82                             pte_t *pte, pmd_t *pmd, unsigned int flags);
83 extern int split_huge_page(struct page *page);
84 extern void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd);
85 #define split_huge_page_pmd(__mm, __pmd)                                \
86         do {                                                            \
87                 pmd_t *____pmd = (__pmd);                               \
88                 if (unlikely(pmd_trans_huge(*____pmd)))                 \
89                         __split_huge_page_pmd(__mm, ____pmd);           \
90         }  while (0)
91 #define wait_split_huge_page(__anon_vma, __pmd)                         \
92         do {                                                            \
93                 pmd_t *____pmd = (__pmd);                               \
94                 spin_unlock_wait(&(__anon_vma)->root->lock);            \
95                 /*                                                      \
96                  * spin_unlock_wait() is just a loop in C and so the    \
97                  * CPU can reorder anything around it.                  \
98                  */                                                     \
99                 smp_mb();                                               \
100                 BUG_ON(pmd_trans_splitting(*____pmd) ||                 \
101                        pmd_trans_huge(*____pmd));                       \
102         } while (0)
103 #define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
104 #define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)
105 #if HPAGE_PMD_ORDER > MAX_ORDER
106 #error "hugepages can't be allocated by the buddy allocator"
107 #endif
108 extern int hugepage_madvise(struct vm_area_struct *vma,
109                             unsigned long *vm_flags, int advice);
110 extern void __vma_adjust_trans_huge(struct vm_area_struct *vma,
111                                     unsigned long start,
112                                     unsigned long end,
113                                     long adjust_next);
114 static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
115                                          unsigned long start,
116                                          unsigned long end,
117                                          long adjust_next)
118 {
119         if (!vma->anon_vma || vma->vm_ops || vma->vm_file)
120                 return;
121         __vma_adjust_trans_huge(vma, start, end, adjust_next);
122 }
123 static inline int hpage_nr_pages(struct page *page)
124 {
125         if (unlikely(PageTransHuge(page)))
126                 return HPAGE_PMD_NR;
127         return 1;
128 }
129 static inline struct page *compound_trans_head(struct page *page)
130 {
131         if (PageTail(page)) {
132                 struct page *head;
133                 head = page->first_page;
134                 smp_rmb();
135                 /*
136                  * head may be a dangling pointer.
137                  * __split_huge_page_refcount clears PageTail before
138                  * overwriting first_page, so if PageTail is still
139                  * there it means the head pointer isn't dangling.
140                  */
141                 if (PageTail(page))
142                         return head;
143         }
144         return page;
145 }
146 #else /* CONFIG_TRANSPARENT_HUGEPAGE */
147 #define HPAGE_PMD_SHIFT ({ BUG(); 0; })
148 #define HPAGE_PMD_MASK ({ BUG(); 0; })
149 #define HPAGE_PMD_SIZE ({ BUG(); 0; })
150
151 #define hpage_nr_pages(x) 1
152
153 #define transparent_hugepage_enabled(__vma) 0
154
155 #define transparent_hugepage_flags 0UL
156 static inline int split_huge_page(struct page *page)
157 {
158         return 0;
159 }
160 #define split_huge_page_pmd(__mm, __pmd)        \
161         do { } while (0)
162 #define wait_split_huge_page(__anon_vma, __pmd) \
163         do { } while (0)
164 #define compound_trans_head(page) compound_head(page)
165 static inline int hugepage_madvise(struct vm_area_struct *vma,
166                                    unsigned long *vm_flags, int advice)
167 {
168         BUG();
169         return 0;
170 }
171 static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
172                                          unsigned long start,
173                                          unsigned long end,
174                                          long adjust_next)
175 {
176 }
177 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
178
179 #endif /* _LINUX_HUGE_MM_H */