x86: cpa: fix the self-test
[linux-3.10.git] / arch / x86 / mm / pageattr.c
1 /*
2  * Copyright 2002 Andi Kleen, SuSE Labs.
3  * Thanks to Ben LaHaise for precious feedback.
4  */
5 #include <linux/highmem.h>
6 #include <linux/bootmem.h>
7 #include <linux/module.h>
8 #include <linux/sched.h>
9 #include <linux/slab.h>
10 #include <linux/mm.h>
11
12 #include <asm/e820.h>
13 #include <asm/processor.h>
14 #include <asm/tlbflush.h>
15 #include <asm/sections.h>
16 #include <asm/uaccess.h>
17 #include <asm/pgalloc.h>
18
19 static inline int
20 within(unsigned long addr, unsigned long start, unsigned long end)
21 {
22         return addr >= start && addr < end;
23 }
24
25 /*
26  * Flushing functions
27  */
28
29 /**
30  * clflush_cache_range - flush a cache range with clflush
31  * @addr:       virtual start address
32  * @size:       number of bytes to flush
33  *
34  * clflush is an unordered instruction which needs fencing with mfence
35  * to avoid ordering issues.
36  */
37 void clflush_cache_range(void *vaddr, unsigned int size)
38 {
39         void *vend = vaddr + size - 1;
40
41         mb();
42
43         for (; vaddr < vend; vaddr += boot_cpu_data.x86_clflush_size)
44                 clflush(vaddr);
45         /*
46          * Flush any possible final partial cacheline:
47          */
48         clflush(vend);
49
50         mb();
51 }
52
53 static void __cpa_flush_all(void *arg)
54 {
55         /*
56          * Flush all to work around Errata in early athlons regarding
57          * large page flushing.
58          */
59         __flush_tlb_all();
60
61         if (boot_cpu_data.x86_model >= 4)
62                 wbinvd();
63 }
64
65 static void cpa_flush_all(void)
66 {
67         BUG_ON(irqs_disabled());
68
69         on_each_cpu(__cpa_flush_all, NULL, 1, 1);
70 }
71
72 static void __cpa_flush_range(void *arg)
73 {
74         /*
75          * We could optimize that further and do individual per page
76          * tlb invalidates for a low number of pages. Caveat: we must
77          * flush the high aliases on 64bit as well.
78          */
79         __flush_tlb_all();
80 }
81
82 static void cpa_flush_range(unsigned long start, int numpages)
83 {
84         unsigned int i, level;
85         unsigned long addr;
86
87         BUG_ON(irqs_disabled());
88         WARN_ON(PAGE_ALIGN(start) != start);
89
90         on_each_cpu(__cpa_flush_range, NULL, 1, 1);
91
92         /*
93          * We only need to flush on one CPU,
94          * clflush is a MESI-coherent instruction that
95          * will cause all other CPUs to flush the same
96          * cachelines:
97          */
98         for (i = 0, addr = start; i < numpages; i++, addr += PAGE_SIZE) {
99                 pte_t *pte = lookup_address(addr, &level);
100
101                 /*
102                  * Only flush present addresses:
103                  */
104                 if (pte && pte_present(*pte))
105                         clflush_cache_range((void *) addr, PAGE_SIZE);
106         }
107 }
108
109 /*
110  * Certain areas of memory on x86 require very specific protection flags,
111  * for example the BIOS area or kernel text. Callers don't always get this
112  * right (again, ioremap() on BIOS memory is not uncommon) so this function
113  * checks and fixes these known static required protection bits.
114  */
115 static inline pgprot_t static_protections(pgprot_t prot, unsigned long address)
116 {
117         pgprot_t forbidden = __pgprot(0);
118
119         /*
120          * The BIOS area between 640k and 1Mb needs to be executable for
121          * PCI BIOS based config access (CONFIG_PCI_GOBIOS) support.
122          */
123         if (within(__pa(address), BIOS_BEGIN, BIOS_END))
124                 pgprot_val(forbidden) |= _PAGE_NX;
125
126         /*
127          * The kernel text needs to be executable for obvious reasons
128          * Does not cover __inittext since that is gone later on
129          */
130         if (within(address, (unsigned long)_text, (unsigned long)_etext))
131                 pgprot_val(forbidden) |= _PAGE_NX;
132
133 #ifdef CONFIG_DEBUG_RODATA
134         /* The .rodata section needs to be read-only */
135         if (within(address, (unsigned long)__start_rodata,
136                                 (unsigned long)__end_rodata))
137                 pgprot_val(forbidden) |= _PAGE_RW;
138 #endif
139
140         prot = __pgprot(pgprot_val(prot) & ~pgprot_val(forbidden));
141
142         return prot;
143 }
144
145 pte_t *lookup_address(unsigned long address, int *level)
146 {
147         pgd_t *pgd = pgd_offset_k(address);
148         pud_t *pud;
149         pmd_t *pmd;
150
151         *level = PG_LEVEL_NONE;
152
153         if (pgd_none(*pgd))
154                 return NULL;
155         pud = pud_offset(pgd, address);
156         if (pud_none(*pud))
157                 return NULL;
158         pmd = pmd_offset(pud, address);
159         if (pmd_none(*pmd))
160                 return NULL;
161
162         *level = PG_LEVEL_2M;
163         if (pmd_large(*pmd))
164                 return (pte_t *)pmd;
165
166         *level = PG_LEVEL_4K;
167         return pte_offset_kernel(pmd, address);
168 }
169
170 static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
171 {
172         /* change init_mm */
173         set_pte_atomic(kpte, pte);
174 #ifdef CONFIG_X86_32
175         if (!SHARED_KERNEL_PMD) {
176                 struct page *page;
177
178                 for (page = pgd_list; page; page = (struct page *)page->index) {
179                         pgd_t *pgd;
180                         pud_t *pud;
181                         pmd_t *pmd;
182
183                         pgd = (pgd_t *)page_address(page) + pgd_index(address);
184                         pud = pud_offset(pgd, address);
185                         pmd = pmd_offset(pud, address);
186                         set_pte_atomic((pte_t *)pmd, pte);
187                 }
188         }
189 #endif
190 }
191
192 static int split_large_page(pte_t *kpte, unsigned long address)
193 {
194         pgprot_t ref_prot = pte_pgprot(pte_clrhuge(*kpte));
195         gfp_t gfp_flags = GFP_KERNEL;
196         unsigned long flags;
197         unsigned long addr;
198         pte_t *pbase, *tmp;
199         struct page *base;
200         unsigned int i, level;
201
202 #ifdef CONFIG_DEBUG_PAGEALLOC
203         gfp_flags = __GFP_HIGH | __GFP_NOFAIL | __GFP_NOWARN;
204         gfp_flags = GFP_ATOMIC | __GFP_NOWARN;
205 #endif
206         base = alloc_pages(gfp_flags, 0);
207         if (!base)
208                 return -ENOMEM;
209
210         spin_lock_irqsave(&pgd_lock, flags);
211         /*
212          * Check for races, another CPU might have split this page
213          * up for us already:
214          */
215         tmp = lookup_address(address, &level);
216         if (tmp != kpte) {
217                 WARN_ON_ONCE(1);
218                 goto out_unlock;
219         }
220
221         address = __pa(address);
222         addr = address & LARGE_PAGE_MASK;
223         pbase = (pte_t *)page_address(base);
224 #ifdef CONFIG_X86_32
225         paravirt_alloc_pt(&init_mm, page_to_pfn(base));
226 #endif
227
228         pgprot_val(ref_prot) &= ~_PAGE_NX;
229         for (i = 0; i < PTRS_PER_PTE; i++, addr += PAGE_SIZE)
230                 set_pte(&pbase[i], pfn_pte(addr >> PAGE_SHIFT, ref_prot));
231
232         /*
233          * Install the new, split up pagetable. Important detail here:
234          *
235          * On Intel the NX bit of all levels must be cleared to make a
236          * page executable. See section 4.13.2 of Intel 64 and IA-32
237          * Architectures Software Developer's Manual).
238          */
239         ref_prot = pte_pgprot(pte_mkexec(pte_clrhuge(*kpte)));
240         __set_pmd_pte(kpte, address, mk_pte(base, ref_prot));
241         base = NULL;
242
243 out_unlock:
244         spin_unlock_irqrestore(&pgd_lock, flags);
245
246         if (base)
247                 __free_pages(base, 0);
248
249         return 0;
250 }
251
252 static int
253 __change_page_attr(unsigned long address, unsigned long pfn,
254                    pgprot_t mask_set, pgprot_t mask_clr)
255 {
256         struct page *kpte_page;
257         int level, err = 0;
258         pte_t *kpte;
259
260 #ifdef CONFIG_X86_32
261         BUG_ON(pfn > max_low_pfn);
262 #endif
263
264 repeat:
265         kpte = lookup_address(address, &level);
266         if (!kpte)
267                 return -EINVAL;
268
269         kpte_page = virt_to_page(kpte);
270         BUG_ON(PageLRU(kpte_page));
271         BUG_ON(PageCompound(kpte_page));
272
273         if (level == PG_LEVEL_4K) {
274                 pgprot_t new_prot = pte_pgprot(*kpte);
275                 pte_t new_pte, old_pte = *kpte;
276
277                 pgprot_val(new_prot) &= ~pgprot_val(mask_clr);
278                 pgprot_val(new_prot) |= pgprot_val(mask_set);
279
280                 new_prot = static_protections(new_prot, address);
281
282                 new_pte = pfn_pte(pfn, canon_pgprot(new_prot));
283                 BUG_ON(pte_pfn(new_pte) != pte_pfn(old_pte));
284
285                 set_pte_atomic(kpte, new_pte);
286         } else {
287                 err = split_large_page(kpte, address);
288                 if (!err)
289                         goto repeat;
290         }
291         return err;
292 }
293
294 /**
295  * change_page_attr_addr - Change page table attributes in linear mapping
296  * @address: Virtual address in linear mapping.
297  * @prot:    New page table attribute (PAGE_*)
298  *
299  * Change page attributes of a page in the direct mapping. This is a variant
300  * of change_page_attr() that also works on memory holes that do not have
301  * mem_map entry (pfn_valid() is false).
302  *
303  * See change_page_attr() documentation for more details.
304  *
305  * Modules and drivers should use the set_memory_* APIs instead.
306  */
307
308 static int
309 change_page_attr_addr(unsigned long address, pgprot_t mask_set,
310                                                         pgprot_t mask_clr)
311 {
312         int err = 0, kernel_map = 0;
313         unsigned long pfn;
314
315 #ifdef CONFIG_X86_64
316         if (address >= __START_KERNEL_map &&
317                         address < __START_KERNEL_map + KERNEL_TEXT_SIZE) {
318
319                 address = (unsigned long)__va(__pa((void *)address));
320                 kernel_map = 1;
321         }
322 #endif
323
324         pfn = __pa(address) >> PAGE_SHIFT;
325
326         if (!kernel_map || 1) {
327                 err = __change_page_attr(address, pfn, mask_set, mask_clr);
328                 if (err)
329                         return err;
330         }
331
332 #ifdef CONFIG_X86_64
333         /*
334          * Handle kernel mapping too which aliases part of
335          * lowmem:
336          */
337         if (__pa(address) < KERNEL_TEXT_SIZE) {
338                 unsigned long addr2;
339
340                 addr2 = __pa(address) + __START_KERNEL_map - phys_base;
341                 /* Make sure the kernel mappings stay executable */
342                 pgprot_val(mask_clr) |= _PAGE_NX;
343                 /*
344                  * Our high aliases are imprecise, so do not propagate
345                  * failures back to users:
346                  */
347                 __change_page_attr(addr2, pfn, mask_set, mask_clr);
348         }
349 #endif
350
351         return err;
352 }
353
354 static int __change_page_attr_set_clr(unsigned long addr, int numpages,
355                                       pgprot_t mask_set, pgprot_t mask_clr)
356 {
357         unsigned int i;
358         int ret;
359
360         for (i = 0; i < numpages ; i++, addr += PAGE_SIZE) {
361                 ret = change_page_attr_addr(addr, mask_set, mask_clr);
362                 if (ret)
363                         return ret;
364         }
365
366         return 0;
367 }
368
369 static int change_page_attr_set_clr(unsigned long addr, int numpages,
370                                     pgprot_t mask_set, pgprot_t mask_clr)
371 {
372         int ret = __change_page_attr_set_clr(addr, numpages, mask_set,
373                                              mask_clr);
374
375         /*
376          * On success we use clflush, when the CPU supports it to
377          * avoid the wbindv. If the CPU does not support it and in the
378          * error case we fall back to cpa_flush_all (which uses
379          * wbindv):
380          */
381         if (!ret && cpu_has_clflush)
382                 cpa_flush_range(addr, numpages);
383         else
384                 cpa_flush_all();
385
386         return ret;
387 }
388
389 static inline int change_page_attr_set(unsigned long addr, int numpages,
390                                        pgprot_t mask)
391 {
392         return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0));
393 }
394
395 static inline int change_page_attr_clear(unsigned long addr, int numpages,
396                                          pgprot_t mask)
397 {
398         return __change_page_attr_set_clr(addr, numpages, __pgprot(0), mask);
399
400 }
401
402 int set_memory_uc(unsigned long addr, int numpages)
403 {
404         return change_page_attr_set(addr, numpages,
405                                     __pgprot(_PAGE_PCD | _PAGE_PWT));
406 }
407 EXPORT_SYMBOL(set_memory_uc);
408
409 int set_memory_wb(unsigned long addr, int numpages)
410 {
411         return change_page_attr_clear(addr, numpages,
412                                       __pgprot(_PAGE_PCD | _PAGE_PWT));
413 }
414 EXPORT_SYMBOL(set_memory_wb);
415
416 int set_memory_x(unsigned long addr, int numpages)
417 {
418         return change_page_attr_clear(addr, numpages, __pgprot(_PAGE_NX));
419 }
420 EXPORT_SYMBOL(set_memory_x);
421
422 int set_memory_nx(unsigned long addr, int numpages)
423 {
424         return change_page_attr_set(addr, numpages, __pgprot(_PAGE_NX));
425 }
426 EXPORT_SYMBOL(set_memory_nx);
427
428 int set_memory_ro(unsigned long addr, int numpages)
429 {
430         return change_page_attr_clear(addr, numpages, __pgprot(_PAGE_RW));
431 }
432
433 int set_memory_rw(unsigned long addr, int numpages)
434 {
435         return change_page_attr_set(addr, numpages, __pgprot(_PAGE_RW));
436 }
437
438 int set_memory_np(unsigned long addr, int numpages)
439 {
440         return change_page_attr_clear(addr, numpages, __pgprot(_PAGE_PRESENT));
441 }
442
443 int set_pages_uc(struct page *page, int numpages)
444 {
445         unsigned long addr = (unsigned long)page_address(page);
446
447         return set_memory_uc(addr, numpages);
448 }
449 EXPORT_SYMBOL(set_pages_uc);
450
451 int set_pages_wb(struct page *page, int numpages)
452 {
453         unsigned long addr = (unsigned long)page_address(page);
454
455         return set_memory_wb(addr, numpages);
456 }
457 EXPORT_SYMBOL(set_pages_wb);
458
459 int set_pages_x(struct page *page, int numpages)
460 {
461         unsigned long addr = (unsigned long)page_address(page);
462
463         return set_memory_x(addr, numpages);
464 }
465 EXPORT_SYMBOL(set_pages_x);
466
467 int set_pages_nx(struct page *page, int numpages)
468 {
469         unsigned long addr = (unsigned long)page_address(page);
470
471         return set_memory_nx(addr, numpages);
472 }
473 EXPORT_SYMBOL(set_pages_nx);
474
475 int set_pages_ro(struct page *page, int numpages)
476 {
477         unsigned long addr = (unsigned long)page_address(page);
478
479         return set_memory_ro(addr, numpages);
480 }
481
482 int set_pages_rw(struct page *page, int numpages)
483 {
484         unsigned long addr = (unsigned long)page_address(page);
485
486         return set_memory_rw(addr, numpages);
487 }
488
489
490 #if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_CPA_DEBUG)
491 static inline int __change_page_attr_set(unsigned long addr, int numpages,
492                                          pgprot_t mask)
493 {
494         return __change_page_attr_set_clr(addr, numpages, mask, __pgprot(0));
495 }
496
497 static inline int __change_page_attr_clear(unsigned long addr, int numpages,
498                                            pgprot_t mask)
499 {
500         return __change_page_attr_set_clr(addr, numpages, __pgprot(0), mask);
501 }
502 #endif
503
504 #ifdef CONFIG_DEBUG_PAGEALLOC
505
506 static int __set_pages_p(struct page *page, int numpages)
507 {
508         unsigned long addr = (unsigned long)page_address(page);
509
510         return __change_page_attr_set(addr, numpages,
511                                       __pgprot(_PAGE_PRESENT | _PAGE_RW));
512 }
513
514 static int __set_pages_np(struct page *page, int numpages)
515 {
516         unsigned long addr = (unsigned long)page_address(page);
517
518         return __change_page_attr_clear(addr, numpages,
519                                         __pgprot(_PAGE_PRESENT));
520 }
521
522 void kernel_map_pages(struct page *page, int numpages, int enable)
523 {
524         if (PageHighMem(page))
525                 return;
526         if (!enable) {
527                 debug_check_no_locks_freed(page_address(page),
528                                            numpages * PAGE_SIZE);
529         }
530
531         /*
532          * If page allocator is not up yet then do not call c_p_a():
533          */
534         if (!debug_pagealloc_enabled)
535                 return;
536
537         /*
538          * The return value is ignored - the calls cannot fail,
539          * large pages are disabled at boot time:
540          */
541         if (enable)
542                 __set_pages_p(page, numpages);
543         else
544                 __set_pages_np(page, numpages);
545
546         /*
547          * We should perform an IPI and flush all tlbs,
548          * but that can deadlock->flush only current cpu:
549          */
550         __flush_tlb_all();
551 }
552 #endif
553
554 /*
555  * The testcases use internal knowledge of the implementation that shouldn't
556  * be exposed to the rest of the kernel. Include these directly here.
557  */
558 #ifdef CONFIG_CPA_DEBUG
559 #include "pageattr-test.c"
560 #endif