blob: 38b6daf72debc6396347de943762b2e1cda30f54 [file] [log] [blame]
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -07001#include <linux/mm.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +09002#include <linux/gfp.h>
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -07003#include <asm/pgalloc.h>
Jeremy Fitzhardingeee5aa8d2008-03-17 16:37:03 -07004#include <asm/pgtable.h>
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -07005#include <asm/tlb.h>
Ingo Molnara1d5a862008-06-20 15:34:46 +02006#include <asm/fixmap.h>
Toshi Kani6b637832015-04-14 15:47:32 -07007#include <asm/mtrr.h>
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -07008
Vladimir Davydov3e79ec72016-07-26 15:24:30 -07009#define PGALLOC_GFP (GFP_KERNEL_ACCOUNT | __GFP_NOTRACK | __GFP_ZERO)
Vegard Nossum9e730232009-02-22 11:28:25 +010010
Ian Campbell14315592010-02-17 10:38:10 +000011#ifdef CONFIG_HIGHPTE
12#define PGALLOC_USER_GFP __GFP_HIGHMEM
13#else
14#define PGALLOC_USER_GFP 0
15#endif
16
17gfp_t __userpte_alloc_gfp = PGALLOC_GFP | PGALLOC_USER_GFP;
18
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -070019pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
20{
Vladimir Davydov3e79ec72016-07-26 15:24:30 -070021 return (pte_t *)__get_free_page(PGALLOC_GFP & ~__GFP_ACCOUNT);
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -070022}
23
24pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
25{
26 struct page *pte;
27
Ian Campbell14315592010-02-17 10:38:10 +000028 pte = alloc_pages(__userpte_alloc_gfp, 0);
Kirill A. Shutemovcecbd1b2013-11-14 14:31:47 -080029 if (!pte)
30 return NULL;
31 if (!pgtable_page_ctor(pte)) {
32 __free_page(pte);
33 return NULL;
34 }
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -070035 return pte;
36}
37
Ian Campbell14315592010-02-17 10:38:10 +000038static int __init setup_userpte(char *arg)
39{
40 if (!arg)
41 return -EINVAL;
42
43 /*
44 * "userpte=nohigh" disables allocation of user pagetables in
45 * high memory.
46 */
47 if (strcmp(arg, "nohigh") == 0)
48 __userpte_alloc_gfp &= ~__GFP_HIGHMEM;
49 else
50 return -EINVAL;
51 return 0;
52}
53early_param("userpte", setup_userpte);
54
Benjamin Herrenschmidt9e1b32c2009-07-22 15:44:28 +100055void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
Jeremy Fitzhardinge397f6872008-03-17 16:36:57 -070056{
57 pgtable_page_dtor(pte);
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -070058 paravirt_release_pte(page_to_pfn(pte));
Jeremy Fitzhardinge397f6872008-03-17 16:36:57 -070059 tlb_remove_page(tlb, pte);
60}
61
Kirill A. Shutemov98233362015-04-14 15:46:14 -070062#if CONFIG_PGTABLE_LEVELS > 2
Benjamin Herrenschmidt9e1b32c2009-07-22 15:44:28 +100063void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
Jeremy Fitzhardinge170fdff2008-03-17 16:36:58 -070064{
Kirill A. Shutemovc2836102013-11-21 14:32:09 -080065 struct page *page = virt_to_page(pmd);
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -070066 paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT);
Dave Hansen1de14c32013-04-12 16:23:54 -070067 /*
68 * NOTE! For PAE, any changes to the top page-directory-pointer-table
69 * entries need a full cr3 reload to flush.
70 */
71#ifdef CONFIG_X86_PAE
72 tlb->need_flush_all = 1;
73#endif
Kirill A. Shutemovc2836102013-11-21 14:32:09 -080074 pgtable_pmd_page_dtor(page);
75 tlb_remove_page(tlb, page);
Jeremy Fitzhardinge170fdff2008-03-17 16:36:58 -070076}
Jeremy Fitzhardinge5a5f8f42008-03-17 16:36:59 -070077
Kirill A. Shutemov98233362015-04-14 15:46:14 -070078#if CONFIG_PGTABLE_LEVELS > 3
Benjamin Herrenschmidt9e1b32c2009-07-22 15:44:28 +100079void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud)
Jeremy Fitzhardinge5a5f8f42008-03-17 16:36:59 -070080{
Jeremy Fitzhardinge2761fa02008-03-17 16:37:02 -070081 paravirt_release_pud(__pa(pud) >> PAGE_SHIFT);
Jeremy Fitzhardinge5a5f8f42008-03-17 16:36:59 -070082 tlb_remove_page(tlb, virt_to_page(pud));
83}
Kirill A. Shutemov98233362015-04-14 15:46:14 -070084#endif /* CONFIG_PGTABLE_LEVELS > 3 */
85#endif /* CONFIG_PGTABLE_LEVELS > 2 */
Jeremy Fitzhardinge170fdff2008-03-17 16:36:58 -070086
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -070087static inline void pgd_list_add(pgd_t *pgd)
88{
89 struct page *page = virt_to_page(pgd);
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -070090
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -070091 list_add(&page->lru, &pgd_list);
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -070092}
93
94static inline void pgd_list_del(pgd_t *pgd)
95{
96 struct page *page = virt_to_page(pgd);
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -070097
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -070098 list_del(&page->lru);
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -070099}
100
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700101#define UNSHARED_PTRS_PER_PGD \
Jeremy Fitzhardinge68db0652008-03-17 16:37:13 -0700102 (SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD)
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700103
Jeremy Fitzhardinge617d34d2010-09-21 12:01:51 -0700104
105static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm)
106{
107 BUILD_BUG_ON(sizeof(virt_to_page(pgd)->index) < sizeof(mm));
108 virt_to_page(pgd)->index = (pgoff_t)mm;
109}
110
111struct mm_struct *pgd_page_get_mm(struct page *page)
112{
113 return (struct mm_struct *)page->index;
114}
115
116static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd)
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700117{
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700118 /* If the pgd points to a shared pagetable level (either the
119 ptes in non-PAE, or shared PMD in PAE), then just copy the
120 references from swapper_pg_dir. */
Kirill A. Shutemov98233362015-04-14 15:46:14 -0700121 if (CONFIG_PGTABLE_LEVELS == 2 ||
122 (CONFIG_PGTABLE_LEVELS == 3 && SHARED_KERNEL_PMD) ||
123 CONFIG_PGTABLE_LEVELS == 4) {
Jeremy Fitzhardinge68db0652008-03-17 16:37:13 -0700124 clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY,
125 swapper_pg_dir + KERNEL_PGD_BOUNDARY,
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700126 KERNEL_PGD_PTRS);
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700127 }
128
129 /* list required to sync kernel mapping updates */
Jeremy Fitzhardinge617d34d2010-09-21 12:01:51 -0700130 if (!SHARED_KERNEL_PMD) {
131 pgd_set_mm(pgd, mm);
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700132 pgd_list_add(pgd);
Jeremy Fitzhardinge617d34d2010-09-21 12:01:51 -0700133 }
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700134}
135
Jan Beulich17b74622008-08-29 12:51:32 +0100136static void pgd_dtor(pgd_t *pgd)
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700137{
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700138 if (SHARED_KERNEL_PMD)
139 return;
140
Andrea Arcangelia79e53d2011-02-16 15:45:22 -0800141 spin_lock(&pgd_lock);
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700142 pgd_list_del(pgd);
Andrea Arcangelia79e53d2011-02-16 15:45:22 -0800143 spin_unlock(&pgd_lock);
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700144}
145
Jeremy Fitzhardinge85958b42008-03-17 16:37:14 -0700146/*
147 * List of all pgd's needed for non-PAE so it can invalidate entries
148 * in both cached and uncached pgd's; not needed for PAE since the
149 * kernel pmd is shared. If PAE were not to share the pmd a similar
150 * tactic would be needed. This is essentially codepath-based locking
151 * against pageattr.c; it is the unique case in which a valid change
152 * of kernel pagetables can't be lazily synchronized by vmalloc faults.
153 * vmalloc faults work because attached pagetables are never freed.
Nadia Yvette Chambers6d49e352012-12-06 10:39:54 +0100154 * -- nyc
Jeremy Fitzhardinge85958b42008-03-17 16:37:14 -0700155 */
156
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700157#ifdef CONFIG_X86_PAE
158/*
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700159 * In PAE mode, we need to do a cr3 reload (=tlb flush) when
160 * updating the top-level pagetable entries to guarantee the
161 * processor notices the update. Since this is expensive, and
162 * all 4 top-level entries are used almost immediately in a
163 * new process's life, we just pre-populate them here.
164 *
165 * Also, if we're in a paravirt environment where the kernel pmd is
166 * not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate
167 * and initialize the kernel pmds here.
168 */
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400169#define PREALLOCATED_PMDS UNSHARED_PTRS_PER_PGD
Ingo Molnar1ec1fe72008-03-19 20:30:40 +0100170
171void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd)
172{
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700173 paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
Ingo Molnar1ec1fe72008-03-19 20:30:40 +0100174
175 /* Note: almost everything apart from _PAGE_PRESENT is
176 reserved at the pmd (PDPT) level. */
177 set_pud(pudp, __pud(__pa(pmd) | _PAGE_PRESENT));
178
179 /*
180 * According to Intel App note "TLBs, Paging-Structure Caches,
181 * and Their Invalidation", April 2007, document 317080-001,
182 * section 8.1: in PAE mode we explicitly have to flush the
183 * TLB via cr3 if the top-level pgd is changed...
184 */
Shaohua Li4981d012011-03-16 11:37:29 +0800185 flush_tlb_mm(mm);
Ingo Molnar1ec1fe72008-03-19 20:30:40 +0100186}
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700187#else /* !CONFIG_X86_PAE */
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400188
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700189/* No need to prepopulate any pagetable entries in non-PAE modes. */
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400190#define PREALLOCATED_PMDS 0
191
192#endif /* CONFIG_X86_PAE */
193
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -0800194static void free_pmds(struct mm_struct *mm, pmd_t *pmds[])
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700195{
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400196 int i;
197
198 for(i = 0; i < PREALLOCATED_PMDS; i++)
Kirill A. Shutemov09ef4932013-11-14 14:31:13 -0800199 if (pmds[i]) {
200 pgtable_pmd_page_dtor(virt_to_page(pmds[i]));
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400201 free_page((unsigned long)pmds[i]);
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -0800202 mm_dec_nr_pmds(mm);
Kirill A. Shutemov09ef4932013-11-14 14:31:13 -0800203 }
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700204}
205
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -0800206static int preallocate_pmds(struct mm_struct *mm, pmd_t *pmds[])
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700207{
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400208 int i;
209 bool failed = false;
Vladimir Davydov3e79ec72016-07-26 15:24:30 -0700210 gfp_t gfp = PGALLOC_GFP;
211
212 if (mm == &init_mm)
213 gfp &= ~__GFP_ACCOUNT;
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400214
215 for(i = 0; i < PREALLOCATED_PMDS; i++) {
Vladimir Davydov3e79ec72016-07-26 15:24:30 -0700216 pmd_t *pmd = (pmd_t *)__get_free_page(gfp);
Kirill A. Shutemov09ef4932013-11-14 14:31:13 -0800217 if (!pmd)
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400218 failed = true;
Kirill A. Shutemov09ef4932013-11-14 14:31:13 -0800219 if (pmd && !pgtable_pmd_page_ctor(virt_to_page(pmd))) {
Al Viro2a46eed2013-11-20 22:16:36 +0000220 free_page((unsigned long)pmd);
Kirill A. Shutemov09ef4932013-11-14 14:31:13 -0800221 pmd = NULL;
222 failed = true;
223 }
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -0800224 if (pmd)
225 mm_inc_nr_pmds(mm);
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400226 pmds[i] = pmd;
227 }
228
229 if (failed) {
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -0800230 free_pmds(mm, pmds);
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400231 return -ENOMEM;
232 }
233
234 return 0;
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700235}
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400236
237/*
238 * Mop up any pmd pages which may still be attached to the pgd.
239 * Normally they will be freed by munmap/exit_mmap, but any pmd we
240 * preallocate which never got a corresponding vma will need to be
241 * freed manually.
242 */
243static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp)
244{
245 int i;
246
247 for(i = 0; i < PREALLOCATED_PMDS; i++) {
248 pgd_t pgd = pgdp[i];
249
250 if (pgd_val(pgd) != 0) {
251 pmd_t *pmd = (pmd_t *)pgd_page_vaddr(pgd);
252
253 pgdp[i] = native_make_pgd(0);
254
255 paravirt_release_pmd(pgd_val(pgd) >> PAGE_SHIFT);
256 pmd_free(mm, pmd);
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -0800257 mm_dec_nr_pmds(mm);
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400258 }
259 }
260}
261
262static void pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmds[])
263{
Kirill A. Shutemove0c4f672017-03-13 17:33:05 +0300264 p4d_t *p4d;
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400265 pud_t *pud;
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400266 int i;
267
Jeremy Fitzhardingecf3e5052008-08-08 13:46:07 -0700268 if (PREALLOCATED_PMDS == 0) /* Work around gcc-3.4.x bug */
269 return;
270
Kirill A. Shutemove0c4f672017-03-13 17:33:05 +0300271 p4d = p4d_offset(pgd, 0);
272 pud = pud_offset(p4d, 0);
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400273
Wanpeng Li73b44ff2013-07-08 16:00:17 -0700274 for (i = 0; i < PREALLOCATED_PMDS; i++, pud++) {
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400275 pmd_t *pmd = pmds[i];
276
277 if (i >= KERNEL_PGD_BOUNDARY)
278 memcpy(pmd, (pmd_t *)pgd_page_vaddr(swapper_pg_dir[i]),
279 sizeof(pmd_t) * PTRS_PER_PMD);
280
281 pud_populate(mm, pud, pmd);
282 }
283}
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700284
Fenghua Yu1db491f2015-01-15 20:30:01 -0800285/*
286 * Xen paravirt assumes pgd table should be in one page. 64 bit kernel also
287 * assumes that pgd should be in one page.
288 *
289 * But kernel with PAE paging that is not running as a Xen domain
290 * only needs to allocate 32 bytes for pgd instead of one page.
291 */
292#ifdef CONFIG_X86_PAE
293
294#include <linux/slab.h>
295
296#define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
297#define PGD_ALIGN 32
298
299static struct kmem_cache *pgd_cache;
300
301static int __init pgd_cache_init(void)
302{
303 /*
304 * When PAE kernel is running as a Xen domain, it does not use
305 * shared kernel pmd. And this requires a whole page for pgd.
306 */
307 if (!SHARED_KERNEL_PMD)
308 return 0;
309
310 /*
311 * when PAE kernel is not running as a Xen domain, it uses
312 * shared kernel pmd. Shared kernel pmd does not require a whole
313 * page for pgd. We are able to just allocate a 32-byte for pgd.
314 * During boot time, we create a 32-byte slab for pgd table allocation.
315 */
316 pgd_cache = kmem_cache_create("pgd_cache", PGD_SIZE, PGD_ALIGN,
317 SLAB_PANIC, NULL);
318 if (!pgd_cache)
319 return -ENOMEM;
320
321 return 0;
322}
323core_initcall(pgd_cache_init);
324
325static inline pgd_t *_pgd_alloc(void)
326{
327 /*
328 * If no SHARED_KERNEL_PMD, PAE kernel is running as a Xen domain.
329 * We allocate one page for pgd.
330 */
331 if (!SHARED_KERNEL_PMD)
332 return (pgd_t *)__get_free_page(PGALLOC_GFP);
333
334 /*
335 * Now PAE kernel is not running as a Xen domain. We can allocate
336 * a 32-byte slab for pgd to save memory space.
337 */
338 return kmem_cache_alloc(pgd_cache, PGALLOC_GFP);
339}
340
341static inline void _pgd_free(pgd_t *pgd)
342{
343 if (!SHARED_KERNEL_PMD)
344 free_page((unsigned long)pgd);
345 else
346 kmem_cache_free(pgd_cache, pgd);
347}
348#else
349static inline pgd_t *_pgd_alloc(void)
350{
351 return (pgd_t *)__get_free_page(PGALLOC_GFP);
352}
353
354static inline void _pgd_free(pgd_t *pgd)
355{
356 free_page((unsigned long)pgd);
357}
358#endif /* CONFIG_X86_PAE */
359
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700360pgd_t *pgd_alloc(struct mm_struct *mm)
361{
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400362 pgd_t *pgd;
363 pmd_t *pmds[PREALLOCATED_PMDS];
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700364
Fenghua Yu1db491f2015-01-15 20:30:01 -0800365 pgd = _pgd_alloc();
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400366
367 if (pgd == NULL)
368 goto out;
369
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700370 mm->pgd = pgd;
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700371
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -0800372 if (preallocate_pmds(mm, pmds) != 0)
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400373 goto out_free_pgd;
374
375 if (paravirt_pgd_alloc(mm) != 0)
376 goto out_free_pmds;
377
378 /*
379 * Make sure that pre-populating the pmds is atomic with
380 * respect to anything walking the pgd_list, so that they
381 * never see a partially populated pgd.
382 */
Andrea Arcangelia79e53d2011-02-16 15:45:22 -0800383 spin_lock(&pgd_lock);
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400384
Jeremy Fitzhardinge617d34d2010-09-21 12:01:51 -0700385 pgd_ctor(mm, pgd);
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400386 pgd_prepopulate_pmd(mm, pgd, pmds);
387
Andrea Arcangelia79e53d2011-02-16 15:45:22 -0800388 spin_unlock(&pgd_lock);
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700389
390 return pgd;
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400391
392out_free_pmds:
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -0800393 free_pmds(mm, pmds);
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400394out_free_pgd:
Fenghua Yu1db491f2015-01-15 20:30:01 -0800395 _pgd_free(pgd);
Jeremy Fitzhardinged8d59002008-06-25 00:19:13 -0400396out:
397 return NULL;
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700398}
399
400void pgd_free(struct mm_struct *mm, pgd_t *pgd)
401{
402 pgd_mop_up_pmds(mm, pgd);
403 pgd_dtor(pgd);
Jeremy Fitzhardingeeba00452008-06-25 00:19:12 -0400404 paravirt_pgd_free(mm, pgd);
Fenghua Yu1db491f2015-01-15 20:30:01 -0800405 _pgd_free(pgd);
Jeremy Fitzhardinge4f76cd32008-03-17 16:36:55 -0700406}
Jeremy Fitzhardingeee5aa8d2008-03-17 16:37:03 -0700407
Rik van Riel0f9a9212012-11-06 09:54:47 +0000408/*
409 * Used to set accessed or dirty bits in the page table entries
410 * on other architectures. On x86, the accessed and dirty bits
411 * are tracked by hardware. However, do_wp_page calls this function
412 * to also make the pte writeable at the same time the dirty bit is
413 * set. In that case we do actually need to write the PTE.
414 */
Jeremy Fitzhardingeee5aa8d2008-03-17 16:37:03 -0700415int ptep_set_access_flags(struct vm_area_struct *vma,
416 unsigned long address, pte_t *ptep,
417 pte_t entry, int dirty)
418{
419 int changed = !pte_same(*ptep, entry);
420
421 if (changed && dirty) {
422 *ptep = entry;
Juergen Grossd6ccc3e2015-11-17 15:51:19 +0100423 pte_update(vma->vm_mm, address, ptep);
Jeremy Fitzhardingeee5aa8d2008-03-17 16:37:03 -0700424 }
425
426 return changed;
427}
Jeremy Fitzhardingef9fbf1a2008-03-17 16:37:04 -0700428
Andrea Arcangelidb3eb96f2011-01-13 15:46:41 -0800429#ifdef CONFIG_TRANSPARENT_HUGEPAGE
430int pmdp_set_access_flags(struct vm_area_struct *vma,
431 unsigned long address, pmd_t *pmdp,
432 pmd_t entry, int dirty)
433{
434 int changed = !pmd_same(*pmdp, entry);
435
436 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
437
438 if (changed && dirty) {
439 *pmdp = entry;
Ingo Molnar5e4bf1a2012-11-20 13:02:51 +0100440 /*
441 * We had a write-protection fault here and changed the pmd
442 * to to more permissive. No need to flush the TLB for that,
443 * #PF is architecturally guaranteed to do that and in the
444 * worst-case we'll generate a spurious fault.
445 */
Andrea Arcangelidb3eb96f2011-01-13 15:46:41 -0800446 }
447
448 return changed;
449}
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -0800450
451int pudp_set_access_flags(struct vm_area_struct *vma, unsigned long address,
452 pud_t *pudp, pud_t entry, int dirty)
453{
454 int changed = !pud_same(*pudp, entry);
455
456 VM_BUG_ON(address & ~HPAGE_PUD_MASK);
457
458 if (changed && dirty) {
459 *pudp = entry;
460 /*
461 * We had a write-protection fault here and changed the pud
462 * to to more permissive. No need to flush the TLB for that,
463 * #PF is architecturally guaranteed to do that and in the
464 * worst-case we'll generate a spurious fault.
465 */
466 }
467
468 return changed;
469}
Andrea Arcangelidb3eb96f2011-01-13 15:46:41 -0800470#endif
471
Jeremy Fitzhardingef9fbf1a2008-03-17 16:37:04 -0700472int ptep_test_and_clear_young(struct vm_area_struct *vma,
473 unsigned long addr, pte_t *ptep)
474{
475 int ret = 0;
476
477 if (pte_young(*ptep))
478 ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
Thomas Gleixner48e23952008-05-24 17:24:34 +0200479 (unsigned long *) &ptep->pte);
Jeremy Fitzhardingef9fbf1a2008-03-17 16:37:04 -0700480
481 if (ret)
482 pte_update(vma->vm_mm, addr, ptep);
483
484 return ret;
485}
Jeremy Fitzhardingec20311e2008-03-17 16:37:05 -0700486
Andrea Arcangelidb3eb96f2011-01-13 15:46:41 -0800487#ifdef CONFIG_TRANSPARENT_HUGEPAGE
488int pmdp_test_and_clear_young(struct vm_area_struct *vma,
489 unsigned long addr, pmd_t *pmdp)
490{
491 int ret = 0;
492
493 if (pmd_young(*pmdp))
494 ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
Johannes Weinerf2d6bfe2011-01-13 15:47:01 -0800495 (unsigned long *)pmdp);
Andrea Arcangelidb3eb96f2011-01-13 15:46:41 -0800496
Andrea Arcangelidb3eb96f2011-01-13 15:46:41 -0800497 return ret;
498}
Matthew Wilcoxa00cc7d2017-02-24 14:57:02 -0800499int pudp_test_and_clear_young(struct vm_area_struct *vma,
500 unsigned long addr, pud_t *pudp)
501{
502 int ret = 0;
503
504 if (pud_young(*pudp))
505 ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
506 (unsigned long *)pudp);
507
508 return ret;
509}
Andrea Arcangelidb3eb96f2011-01-13 15:46:41 -0800510#endif
511
Jeremy Fitzhardingec20311e2008-03-17 16:37:05 -0700512int ptep_clear_flush_young(struct vm_area_struct *vma,
513 unsigned long address, pte_t *ptep)
514{
Shaohua Lib13b1d22014-04-08 15:58:09 +0800515 /*
516 * On x86 CPUs, clearing the accessed bit without a TLB flush
517 * doesn't cause data corruption. [ It could cause incorrect
518 * page aging and the (mistaken) reclaim of hot pages, but the
519 * chance of that should be relatively low. ]
520 *
521 * So as a performance optimization don't flush the TLB when
522 * clearing the accessed bit, it will eventually be flushed by
523 * a context switch or a VM operation anyway. [ In the rare
524 * event of it not getting flushed for a long time the delay
525 * shouldn't really matter because there's no real memory
526 * pressure for swapout to react to. ]
527 */
528 return ptep_test_and_clear_young(vma, address, ptep);
Jeremy Fitzhardingec20311e2008-03-17 16:37:05 -0700529}
Jeremy Fitzhardinge7c7e6e02008-06-17 11:41:54 -0700530
Andrea Arcangelidb3eb96f2011-01-13 15:46:41 -0800531#ifdef CONFIG_TRANSPARENT_HUGEPAGE
532int pmdp_clear_flush_young(struct vm_area_struct *vma,
533 unsigned long address, pmd_t *pmdp)
534{
535 int young;
536
537 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
538
539 young = pmdp_test_and_clear_young(vma, address, pmdp);
540 if (young)
541 flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
542
543 return young;
544}
Andrea Arcangelidb3eb96f2011-01-13 15:46:41 -0800545#endif
546
Gustavo F. Padovanfd862dd2009-02-15 21:48:54 -0300547/**
548 * reserve_top_address - reserves a hole in the top of kernel address space
549 * @reserve - size of hole to reserve
550 *
551 * Can be used to relocate the fixmap area and poke a hole in the top
552 * of kernel address space to make room for a hypervisor.
553 */
554void __init reserve_top_address(unsigned long reserve)
555{
556#ifdef CONFIG_X86_32
557 BUG_ON(fixmaps_set > 0);
Andy Lutomirski73159fd2014-05-05 12:19:31 -0700558 __FIXADDR_TOP = round_down(-reserve, 1 << PMD_SHIFT) - PAGE_SIZE;
559 printk(KERN_INFO "Reserving virtual address space above 0x%08lx (rounded to 0x%08lx)\n",
560 -reserve, __FIXADDR_TOP + PAGE_SIZE);
Gustavo F. Padovanfd862dd2009-02-15 21:48:54 -0300561#endif
562}
563
Jeremy Fitzhardinge7c7e6e02008-06-17 11:41:54 -0700564int fixmaps_set;
565
Jeremy Fitzhardingeaeaaa592008-06-17 11:42:01 -0700566void __native_set_fixmap(enum fixed_addresses idx, pte_t pte)
Jeremy Fitzhardinge7c7e6e02008-06-17 11:41:54 -0700567{
568 unsigned long address = __fix_to_virt(idx);
569
570 if (idx >= __end_of_fixed_addresses) {
571 BUG();
572 return;
573 }
Jeremy Fitzhardingeaeaaa592008-06-17 11:42:01 -0700574 set_pte_vaddr(address, pte);
Jeremy Fitzhardinge7c7e6e02008-06-17 11:41:54 -0700575 fixmaps_set++;
576}
Jeremy Fitzhardingeaeaaa592008-06-17 11:42:01 -0700577
Masami Hiramatsu3b3809a2009-04-09 10:55:33 -0700578void native_set_fixmap(enum fixed_addresses idx, phys_addr_t phys,
579 pgprot_t flags)
Jeremy Fitzhardingeaeaaa592008-06-17 11:42:01 -0700580{
581 __native_set_fixmap(idx, pfn_pte(phys >> PAGE_SHIFT, flags));
582}
Toshi Kani6b637832015-04-14 15:47:32 -0700583
584#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
Toshi Kani3d3ca412015-05-26 10:28:07 +0200585/**
586 * pud_set_huge - setup kernel PUD mapping
587 *
Toshi Kanib73522e2015-05-26 10:28:10 +0200588 * MTRRs can override PAT memory types with 4KiB granularity. Therefore, this
589 * function sets up a huge page only if any of the following conditions are met:
590 *
591 * - MTRRs are disabled, or
592 *
593 * - MTRRs are enabled and the range is completely covered by a single MTRR, or
594 *
595 * - MTRRs are enabled and the corresponding MTRR memory type is WB, which
596 * has no effect on the requested PAT memory type.
597 *
598 * Callers should try to decrease page size (1GB -> 2MB -> 4K) if the bigger
599 * page mapping attempt fails.
Toshi Kani3d3ca412015-05-26 10:28:07 +0200600 *
601 * Returns 1 on success and 0 on failure.
602 */
Toshi Kani6b637832015-04-14 15:47:32 -0700603int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
604{
Toshi Kanib73522e2015-05-26 10:28:10 +0200605 u8 mtrr, uniform;
Toshi Kani6b637832015-04-14 15:47:32 -0700606
Toshi Kanib73522e2015-05-26 10:28:10 +0200607 mtrr = mtrr_type_lookup(addr, addr + PUD_SIZE, &uniform);
608 if ((mtrr != MTRR_TYPE_INVALID) && (!uniform) &&
609 (mtrr != MTRR_TYPE_WRBACK))
Toshi Kani6b637832015-04-14 15:47:32 -0700610 return 0;
611
612 prot = pgprot_4k_2_large(prot);
613
614 set_pte((pte_t *)pud, pfn_pte(
615 (u64)addr >> PAGE_SHIFT,
616 __pgprot(pgprot_val(prot) | _PAGE_PSE)));
617
618 return 1;
619}
620
Toshi Kani3d3ca412015-05-26 10:28:07 +0200621/**
622 * pmd_set_huge - setup kernel PMD mapping
623 *
Toshi Kanib73522e2015-05-26 10:28:10 +0200624 * See text over pud_set_huge() above.
Toshi Kani3d3ca412015-05-26 10:28:07 +0200625 *
626 * Returns 1 on success and 0 on failure.
627 */
Toshi Kani6b637832015-04-14 15:47:32 -0700628int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
629{
Toshi Kanib73522e2015-05-26 10:28:10 +0200630 u8 mtrr, uniform;
Toshi Kani6b637832015-04-14 15:47:32 -0700631
Toshi Kanib73522e2015-05-26 10:28:10 +0200632 mtrr = mtrr_type_lookup(addr, addr + PMD_SIZE, &uniform);
633 if ((mtrr != MTRR_TYPE_INVALID) && (!uniform) &&
634 (mtrr != MTRR_TYPE_WRBACK)) {
635 pr_warn_once("%s: Cannot satisfy [mem %#010llx-%#010llx] with a huge-page mapping due to MTRR override.\n",
636 __func__, addr, addr + PMD_SIZE);
Toshi Kani6b637832015-04-14 15:47:32 -0700637 return 0;
Toshi Kanib73522e2015-05-26 10:28:10 +0200638 }
Toshi Kani6b637832015-04-14 15:47:32 -0700639
640 prot = pgprot_4k_2_large(prot);
641
642 set_pte((pte_t *)pmd, pfn_pte(
643 (u64)addr >> PAGE_SHIFT,
644 __pgprot(pgprot_val(prot) | _PAGE_PSE)));
645
646 return 1;
647}
648
Toshi Kani3d3ca412015-05-26 10:28:07 +0200649/**
650 * pud_clear_huge - clear kernel PUD mapping when it is set
651 *
652 * Returns 1 on success and 0 on failure (no PUD map is found).
653 */
Toshi Kani6b637832015-04-14 15:47:32 -0700654int pud_clear_huge(pud_t *pud)
655{
656 if (pud_large(*pud)) {
657 pud_clear(pud);
658 return 1;
659 }
660
661 return 0;
662}
663
Toshi Kani3d3ca412015-05-26 10:28:07 +0200664/**
665 * pmd_clear_huge - clear kernel PMD mapping when it is set
666 *
667 * Returns 1 on success and 0 on failure (no PMD map is found).
668 */
Toshi Kani6b637832015-04-14 15:47:32 -0700669int pmd_clear_huge(pmd_t *pmd)
670{
671 if (pmd_large(*pmd)) {
672 pmd_clear(pmd);
673 return 1;
674 }
675
676 return 0;
677}
678#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */