0ea6676996dccace4f6ba5e9c5ffdb1c9823af60
[linux-3.10.git] / arch / arm / mm / pageattr.c
1 /*
2  * Copyright 2002 Andi Kleen, SuSE Labs.
3  * Thanks to Ben LaHaise for precious feedback.
4  */
5 #include <linux/highmem.h>
6 #include <linux/bootmem.h>
7 #include <linux/module.h>
8 #include <linux/sched.h>
9 #include <linux/mm.h>
10 #include <linux/interrupt.h>
11 #include <linux/seq_file.h>
12 #include <linux/debugfs.h>
13 #include <linux/pfn.h>
14 #include <linux/percpu.h>
15 #include <linux/gfp.h>
16 #include <linux/vmalloc.h>
17 #include <linux/mutex.h>
18
19 #include <asm/cputype.h>
20 #include <asm/processor.h>
21 #include <asm/tlbflush.h>
22 #include <asm/sections.h>
23 #include <asm/setup.h>
24 #include <asm/uaccess.h>
25 #include <asm/pgalloc.h>
26
27 #include "mm.h"
28
29 #ifdef CPA_DEBUG
30 #define cpa_debug(x, ...)  printk(x, __VA_ARGS__)
31 #else
32 #define cpa_debug(x, ...)
33 #endif
34
35 static int inner_cache_maint_threshold = SZ_2M;
36 extern void v7_flush_kern_cache_all(void);
37 extern void __flush_dcache_page(struct address_space *, struct page *);
38
39 static inline void v7_flush_kern_cache_all_arg(void *arg)
40 {
41         v7_flush_kern_cache_all();
42 }
43
44 #if defined(CONFIG_NVMAP_CACHE_MAINT_BY_SET_WAYS)
45 static void inner_flush_cache_all(void)
46 {
47 #ifdef CONFIG_NVMAP_CACHE_MAINT_BY_SET_WAYS_ON_ONE_CPU
48         v7_flush_kern_cache_all();
49 #else
50         on_each_cpu(v7_flush_kern_cache_all_arg, NULL, 1);
51 #endif
52 }
53 #endif
54
55 #if defined(CONFIG_CPA)
56
57 /*
58  * The arm kernel uses different cache policies(CPOLICY_WRITEBACK,
59  * CPOLICY_WRITEALLOC, CPOLICY_WRITETHROUGH) based on architecture version
60  * and smp mode. Using L_PTE_MT_WRITEALLOC or L_PTE_MT_WRITEBACK or
61  * L_PTE_MT_WRITETHROUGH directly in CPA code can result in restoring incorrect
62  * PTE attributes.
63  * pgprot_kernel would always have PTE attributes based on the cache policy
64  * in use for kernel cache memory. Use this to set the correct PTE attributes
65  * for kernel cache memory.
66  * */
67 #define L_PTE_MT_KERNEL (pgprot_kernel & L_PTE_MT_MASK)
68
69 /*
70  * The current flushing context - we pass it instead of 5 arguments:
71  */
72 struct cpa_data {
73         unsigned long   *vaddr;
74         pgprot_t        mask_set;
75         pgprot_t        mask_clr;
76         int             numpages;
77         int             flags;
78         unsigned long   pfn;
79         unsigned        force_split:1;
80         int             curpage;
81         struct page     **pages;
82 };
83
84 /*
85  * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings)
86  * using cpa_lock. So that we don't allow any other cpu, with stale large tlb
87  * entries change the page attribute in parallel to some other cpu
88  * splitting a large page entry along with changing the attribute.
89  */
90 static DEFINE_MUTEX(cpa_lock);
91
92 #define CPA_FLUSHTLB 1
93 #define CPA_ARRAY 2
94 #define CPA_PAGES_ARRAY 4
95
96 #ifdef CONFIG_PROC_FS
97 static unsigned long direct_pages_count[PG_LEVEL_NUM];
98
99 void update_page_count(int level, unsigned long pages)
100 {
101         unsigned long flags;
102
103         /* Protect against CPA */
104         spin_lock_irqsave(&pgd_lock, flags);
105         direct_pages_count[level] += pages;
106         spin_unlock_irqrestore(&pgd_lock, flags);
107 }
108
109 static void split_page_count(int level)
110 {
111         direct_pages_count[level]--;
112         direct_pages_count[level - 1] += PTRS_PER_PTE;
113 }
114
115 void arch_report_meminfo(struct seq_file *m)
116 {
117         seq_printf(m, "DirectMap4k:    %8lu kB\n",
118                         direct_pages_count[PG_LEVEL_4K] << 2);
119         seq_printf(m, "DirectMap2M:    %8lu kB\n",
120                         direct_pages_count[PG_LEVEL_2M] << 11);
121 }
122 #else
123 static inline void split_page_count(int level) { }
124 #endif
125
126 #ifdef CONFIG_DEBUG_PAGEALLOC
127 # define debug_pagealloc 1
128 #else
129 # define debug_pagealloc 0
130 #endif
131
132 static inline int
133 within(unsigned long addr, unsigned long start, unsigned long end)
134 {
135         return addr >= start && addr < end;
136 }
137
138 static void cpa_flush_range(unsigned long start, int numpages, int cache)
139 {
140         unsigned int i, level;
141         unsigned long addr;
142
143         BUG_ON(irqs_disabled());
144         WARN_ON(PAGE_ALIGN(start) != start);
145
146         flush_tlb_kernel_range(start, start + (numpages << PAGE_SHIFT));
147
148         if (!cache)
149                 return;
150
151         for (i = 0, addr = start; i < numpages; i++, addr += PAGE_SIZE) {
152                 pte_t *pte = lookup_address(addr, &level);
153
154                 /*
155                  * Only flush present addresses:
156                  */
157                 if (pte && pte_present(*pte)) {
158                         __cpuc_flush_dcache_area((void *) addr, PAGE_SIZE);
159                         outer_flush_range(__pa((void *)addr),
160                                         __pa((void *)addr) + PAGE_SIZE);
161                 }
162         }
163 }
164
165 static void cpa_flush_array(unsigned long *start, int numpages, int cache,
166                             int in_flags, struct page **pages)
167 {
168         unsigned int i, level;
169         bool flush_inner = true;
170         unsigned long base;
171
172         BUG_ON(irqs_disabled());
173
174 #if defined(CONFIG_NVMAP_CACHE_MAINT_BY_SET_WAYS)
175         if (numpages >= (inner_cache_maint_threshold >> PAGE_SHIFT) &&
176                 cache && in_flags & CPA_PAGES_ARRAY) {
177                 inner_flush_cache_all();
178                 flush_inner = false;
179         }
180 #endif
181
182         for (i = 0; i < numpages; i++) {
183                 unsigned long addr;
184                 pte_t *pte;
185
186                 if (in_flags & CPA_PAGES_ARRAY)
187                         addr = (unsigned long)page_address(pages[i]);
188                 else
189                         addr = start[i];
190
191                 flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
192
193                 if (cache && in_flags & CPA_PAGES_ARRAY) {
194                         /* cache flush all pages including high mem pages. */
195                         if (flush_inner)
196                                 __flush_dcache_page(
197                                         page_mapping(pages[i]), pages[i]);
198                         base = page_to_phys(pages[i]);
199                         outer_flush_range(base, base + PAGE_SIZE);
200                 } else if (cache) {
201                         pte = lookup_address(addr, &level);
202
203                         /*
204                          * Only flush present addresses:
205                          */
206                         if (pte && pte_present(*pte)) {
207                                 __cpuc_flush_dcache_area((void *)addr,
208                                         PAGE_SIZE);
209                                 outer_flush_range(__pa((void *)addr),
210                                         __pa((void *)addr) + PAGE_SIZE);
211                         }
212                 }
213         }
214 }
215
216 /*
217  * Certain areas of memory require very specific protection flags,
218  * for example the kernel text. Callers don't always get this
219  * right so this function checks and fixes these known static
220  * required protection bits.
221  */
222 static inline pgprot_t static_protections(pgprot_t prot, unsigned long address,
223                                    unsigned long pfn)
224 {
225         pgprot_t forbidden = __pgprot(0);
226
227         /*
228          * The kernel text needs to be executable for obvious reasons
229          * Does not cover __inittext since that is gone later on.
230          */
231         if (within(address, (unsigned long)_text, (unsigned long)_etext))
232                 pgprot_val(forbidden) |= L_PTE_XN;
233
234         /*
235          * The .rodata section needs to be read-only. Using the pfn
236          * catches all aliases.
237          */
238         if (within(pfn, __pa((unsigned long)__start_rodata) >> PAGE_SHIFT,
239                    __pa((unsigned long)__end_rodata) >> PAGE_SHIFT))
240                 prot |= L_PTE_RDONLY;
241
242         /*
243          * Mask off the forbidden bits and set the bits that are needed
244         */
245         prot = __pgprot(pgprot_val(prot) & ~pgprot_val(forbidden));
246
247
248         return prot;
249 }
250
251 static inline pgprot_t pte_to_pmd_pgprot(unsigned long pte,
252                                 unsigned long ext_prot)
253 {
254         pgprot_t ref_prot;
255
256         ref_prot = PMD_TYPE_SECT | PMD_DOMAIN(DOMAIN_KERNEL) |
257                    PMD_SECT_AP_WRITE;
258
259         if (pte & L_PTE_MT_BUFFERABLE)
260                 ref_prot |= PMD_SECT_BUFFERABLE;
261
262         if (pte & L_PTE_MT_WRITETHROUGH)
263                 ref_prot |= PMD_SECT_CACHEABLE;
264
265         if (pte & L_PTE_XN)
266                 ref_prot |= PMD_SECT_XN;
267
268         if (pte & L_PTE_USER)
269                 ref_prot |= PMD_SECT_AP_READ;
270
271         if (pte & (1 << 4))
272                 ref_prot |= PMD_SECT_TEX(1);
273
274         if (pte & L_PTE_RDONLY)
275                 ref_prot |= PMD_SECT_APX;
276
277         if (pte & L_PTE_SHARED)
278                 ref_prot |= PMD_SECT_S;
279
280         if (pte & PTE_EXT_NG)
281                 ref_prot |= PMD_SECT_nG;
282
283         return ref_prot;
284 }
285
286 static inline pgprot_t pmd_to_pte_pgprot(unsigned long pmd,
287                                 unsigned long *ext_prot)
288 {
289         pgprot_t ref_prot;
290
291         *ext_prot = 0;
292         ref_prot = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY;
293
294         if (pmd & PMD_SECT_BUFFERABLE)
295                 ref_prot |= L_PTE_MT_BUFFERABLE;
296
297         if (pmd & PMD_SECT_CACHEABLE)
298                 ref_prot |= L_PTE_MT_WRITETHROUGH;
299
300         if (pmd & PMD_SECT_XN)
301                 ref_prot |= L_PTE_XN;
302
303         if (pmd & PMD_SECT_AP_READ)
304                 ref_prot |= L_PTE_USER;
305
306         if (pmd & PMD_SECT_TEX(1))
307                 ref_prot |= (1 << 4);
308
309         if (pmd & PMD_SECT_APX)
310                 ref_prot |= L_PTE_RDONLY;
311
312         if (pmd & PMD_SECT_S)
313                 ref_prot |= L_PTE_SHARED;
314
315         if (pmd & PMD_SECT_nG)
316                 ref_prot |= PTE_EXT_NG;
317
318         return ref_prot;
319 }
320
321 /*
322  * Lookup the page table entry for a virtual address. Return a pointer
323  * to the entry and the level of the mapping.
324  *
325  * Note: We return pud and pmd either when the entry is marked large
326  * or when the present bit is not set. Otherwise we would return a
327  * pointer to a nonexisting mapping.
328  */
329 pte_t *lookup_address(unsigned long address, unsigned int *level)
330 {
331         pgd_t *pgd = pgd_offset_k(address);
332         pte_t *pte;
333         pmd_t *pmd;
334
335         /* pmds are folded into pgds on ARM */
336         *level = PG_LEVEL_NONE;
337
338         if (pgd == NULL || pgd_none(*pgd))
339                 return NULL;
340
341         pmd = pmd_offset(pud_offset(pgd, address), address);
342
343         if (pmd == NULL || pmd_none(*pmd) || !pmd_present(*pmd))
344                 return NULL;
345
346         if (((pmd_val(*pmd) & (PMD_TYPE_SECT | PMD_SECT_SUPER))
347                 == (PMD_TYPE_SECT | PMD_SECT_SUPER)) || !pmd_present(*pmd)) {
348
349                 return NULL;
350         } else if (pmd_val(*pmd) & PMD_TYPE_SECT) {
351
352                 *level = PG_LEVEL_2M;
353                 return (pte_t *)pmd;
354         }
355
356         pte = pte_offset_kernel(pmd, address);
357
358         if ((pte == NULL) || pte_none(*pte))
359                 return NULL;
360
361         *level = PG_LEVEL_4K;
362
363         return pte;
364 }
365 EXPORT_SYMBOL_GPL(lookup_address);
366
367 /*
368  * Set the new pmd in all the pgds we know about:
369  */
370 static void __set_pmd_pte(pmd_t *pmd, unsigned long address, pte_t *pte)
371 {
372         struct page *page;
373         pud_t *pud;
374
375         cpa_debug("__set_pmd_pte %x %x %x\n", pmd, pte, *pte);
376
377         /* enforce pte entry stores ordering to avoid pmd writes
378          * bypassing pte stores.
379          */
380         dsb();
381         /* change init_mm */
382         pmd_populate_kernel(&init_mm, pmd, pte);
383
384         /* change entry in all the pgd's */
385         list_for_each_entry(page, &pgd_list, lru) {
386                 cpa_debug("list %x %x %x\n", (unsigned long)page,
387                         (unsigned long)pgd_index(address), address);
388                 pud = pud_offset(((pgd_t *)page_address(page)) +
389                         pgd_index(address), address);
390                 pmd = pmd_offset(pud, address);
391                 pmd_populate_kernel(NULL, pmd, pte);
392         }
393         /* enforce pmd entry stores ordering to avoid tlb flush bypassing
394          * pmd entry stores.
395          */
396         dsb();
397 }
398
399 static int
400 try_preserve_large_page(pte_t *kpte, unsigned long address,
401                         struct cpa_data *cpa)
402 {
403         unsigned long nextpage_addr, numpages, pmask, psize, flags, addr, pfn;
404         pte_t old_pte, *tmp;
405         pgprot_t old_prot, new_prot, req_prot;
406         unsigned long ext_prot;
407         int i, do_split = 1;
408         unsigned int level;
409
410         if (cpa->force_split)
411                 return 1;
412
413         spin_lock_irqsave(&pgd_lock, flags);
414         /*
415          * Check for races, another CPU might have split this page
416          * up already:
417          */
418         tmp = lookup_address(address, &level);
419         if (tmp != kpte)
420                 goto out_unlock;
421
422         switch (level) {
423
424         case PG_LEVEL_2M:
425                 psize = PMD_SIZE;
426                 pmask = PMD_MASK;
427                 break;
428
429         default:
430                 do_split = -EINVAL;
431                 goto out_unlock;
432         }
433
434         /*
435          * Calculate the number of pages, which fit into this large
436          * page starting at address:
437          */
438         nextpage_addr = (address + psize) & pmask;
439         numpages = (nextpage_addr - address) >> PAGE_SHIFT;
440         if (numpages < cpa->numpages)
441                 cpa->numpages = numpages;
442
443         old_pte = *kpte;
444         old_prot = new_prot = req_prot = pmd_to_pte_pgprot(pmd_val(*kpte),
445                                                 &ext_prot);
446
447         pgprot_val(req_prot) &= ~pgprot_val(cpa->mask_clr);
448         pgprot_val(req_prot) |= pgprot_val(cpa->mask_set);
449
450         /*
451          * old_pte points to the large page base address. So we need
452          * to add the offset of the virtual address:
453          */
454         pfn = pmd_pfn(*kpte) + ((address & (psize - 1)) >> PAGE_SHIFT);
455         cpa->pfn = pfn;
456
457         new_prot = static_protections(req_prot, address, pfn);
458
459         /*
460          * We need to check the full range, whether
461          * static_protection() requires a different pgprot for one of
462          * the pages in the range we try to preserve:
463          */
464         addr = address & pmask;
465         pfn = pmd_pfn(old_pte);
466         for (i = 0; i < (psize >> PAGE_SHIFT); i++, addr += PAGE_SIZE, pfn++) {
467                 pgprot_t chk_prot = static_protections(req_prot, addr, pfn);
468
469                 if (pgprot_val(chk_prot) != pgprot_val(new_prot))
470                         goto out_unlock;
471         }
472
473         /*
474          * If there are no changes, return. maxpages has been updated
475          * above:
476          */
477         if (pgprot_val(new_prot) == pgprot_val(old_prot)) {
478                 do_split = 0;
479                 goto out_unlock;
480         }
481
482         /*
483          * convert prot to pmd format
484          */
485         new_prot = pte_to_pmd_pgprot(new_prot, ext_prot);
486
487         /*
488          * We need to change the attributes. Check, whether we can
489          * change the large page in one go. We request a split, when
490          * the address is not aligned and the number of pages is
491          * smaller than the number of pages in the large page. Note
492          * that we limited the number of possible pages already to
493          * the number of pages in the large page.
494          */
495         if (address == (nextpage_addr - psize) && cpa->numpages == numpages) {
496                 /*
497                  * The address is aligned and the number of pages
498                  * covers the full page.
499                  */
500                 phys_addr_t phys = __pfn_to_phys(pmd_pfn(*kpte));
501                 pmd_t *p = (pmd_t *)kpte;
502
503                 *kpte++ = __pmd(phys | new_prot);
504                 *kpte   = __pmd((phys + SECTION_SIZE) | new_prot);
505                 flush_pmd_entry(p);
506                 cpa->flags |= CPA_FLUSHTLB;
507                 do_split = 0;
508                 cpa_debug("preserving page at phys %x pmd %x\n", phys, p);
509         }
510
511 out_unlock:
512         spin_unlock_irqrestore(&pgd_lock, flags);
513
514         return do_split;
515 }
516
517 static int split_large_page(pte_t *kpte, unsigned long address)
518 {
519         unsigned long flags, pfn, pfninc = 1;
520         unsigned int i, level;
521         pte_t *pbase, *tmp;
522         pgprot_t ref_prot = 0;
523         unsigned long ext_prot = 0;
524         int ret = 0;
525
526         BUG_ON((address & PMD_MASK) < __pa(_end));
527
528         if (!debug_pagealloc)
529                 mutex_unlock(&cpa_lock);
530         pbase = pte_alloc_one_kernel(&init_mm, address);
531         if (!debug_pagealloc)
532                 mutex_lock(&cpa_lock);
533         if (!pbase)
534                 return -ENOMEM;
535
536         cpa_debug("split_large_page %x PMD %x new pte @ %x\n", address,
537                         *kpte, pbase);
538
539         spin_lock_irqsave(&pgd_lock, flags);
540         /*
541          * Check for races, another CPU might have split this page
542          * up for us already:
543          */
544         tmp = lookup_address(address, &level);
545         if (tmp != kpte)
546                 goto out_unlock;
547
548         /*
549          * we only split 2MB entries for now
550         */
551         if (level != PG_LEVEL_2M) {
552                 ret = -EINVAL;
553                 goto out_unlock;
554         }
555
556         ref_prot = pmd_to_pte_pgprot(pmd_val(*kpte), &ext_prot);
557
558         BUG_ON(ref_prot != pgprot_kernel);
559         /*
560          * Get the target pfn from the original entry:
561          */
562         pfn = pmd_pfn(*kpte);
563         for (i = 0; i < PTRS_PER_PTE; i++, pfn += pfninc)
564                 set_pte_ext(&pbase[i], pfn_pte(pfn, ref_prot), ext_prot);
565
566         if (address >= (unsigned long)__va(0) &&
567                 address < (unsigned long)__va(arm_lowmem_limit))
568                 split_page_count(level);
569
570         /*
571          * Install the new, split up pagetable.
572          */
573         __set_pmd_pte((pmd_t *)kpte, address, pbase);
574
575         pbase = NULL;
576
577 out_unlock:
578         /*
579          * If we dropped out via the lookup_address check under
580          * pgd_lock then stick the page back into the pool:
581          */
582         if (pbase)
583                 pte_free_kernel(&init_mm, pbase);
584
585         spin_unlock_irqrestore(&pgd_lock, flags);
586
587         return ret;
588 }
589
590 static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
591                                int primary)
592 {
593         /*
594          * Ignore all non primary paths.
595          */
596         if (!primary)
597                 return 0;
598
599         /*
600          * Ignore the NULL PTE for kernel identity mapping, as it is expected
601          * to have holes.
602          * Also set numpages to '1' indicating that we processed cpa req for
603          * one virtual address page and its pfn. TBD: numpages can be set based
604          * on the initial value and the level returned by lookup_address().
605          */
606         if (within(vaddr, PAGE_OFFSET,
607                    PAGE_OFFSET + arm_lowmem_limit)) {
608                 cpa->numpages = 1;
609                 cpa->pfn = __pa(vaddr) >> PAGE_SHIFT;
610                 return 0;
611         } else {
612                 WARN(1, KERN_WARNING "CPA: called for zero pte. "
613                         "vaddr = %lx cpa->vaddr = %lx\n", vaddr,
614                         *cpa->vaddr);
615
616                 return -EFAULT;
617         }
618 }
619
620 static int __change_page_attr(struct cpa_data *cpa, int primary)
621 {
622         unsigned long address;
623         int do_split, err;
624         unsigned int level;
625         pte_t *kpte, old_pte;
626
627         if (cpa->flags & CPA_PAGES_ARRAY) {
628                 struct page *page = cpa->pages[cpa->curpage];
629
630                 if (unlikely(PageHighMem(page)))
631                         return 0;
632
633                 address = (unsigned long)page_address(page);
634
635         } else if (cpa->flags & CPA_ARRAY)
636                 address = cpa->vaddr[cpa->curpage];
637         else
638                 address = *cpa->vaddr;
639
640 repeat:
641         kpte = lookup_address(address, &level);
642         if (!kpte)
643                 return __cpa_process_fault(cpa, address, primary);
644
645         old_pte = *kpte;
646         if (!pte_val(old_pte))
647                 return __cpa_process_fault(cpa, address, primary);
648
649         if (level == PG_LEVEL_4K) {
650                 pte_t new_pte;
651                 pgprot_t new_prot = pte_pgprot(old_pte);
652                 unsigned long pfn = pte_pfn(old_pte);
653
654                 pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
655                 pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
656
657                 new_prot = static_protections(new_prot, address, pfn);
658
659                 /*
660                  * We need to keep the pfn from the existing PTE,
661                  * after all we're only going to change it's attributes
662                  * not the memory it points to
663                  */
664                 new_pte = pfn_pte(pfn, new_prot);
665                 cpa->pfn = pfn;
666
667                 /*
668                  * Do we really change anything ?
669                  */
670                 if (pte_val(old_pte) != pte_val(new_pte)) {
671                         set_pte_ext(kpte, new_pte, 0);
672                         /*
673                          * FIXME : is this needed on arm?
674                          * set_pte_ext already does a flush
675                          */
676                         cpa->flags |= CPA_FLUSHTLB;
677                 }
678                 cpa->numpages = 1;
679                 return 0;
680         }
681
682         /*
683          * Check, whether we can keep the large page intact
684          * and just change the pte:
685          */
686         do_split = try_preserve_large_page(kpte, address, cpa);
687
688         /*
689          * When the range fits into the existing large page,
690          * return. cp->numpages and cpa->tlbflush have been updated in
691          * try_large_page:
692          */
693         if (do_split <= 0)
694                 return do_split;
695
696         /*
697          * We have to split the large page:
698          */
699         err = split_large_page(kpte, address);
700
701         if (!err) {
702                 /*
703                  * Do a global flush tlb after splitting the large page
704                  * and before we do the actual change page attribute in the PTE.
705                  *
706                  * With out this, we violate the TLB application note, that says
707                  * "The TLBs may contain both ordinary and large-page
708                  *  translations for a 4-KByte range of linear addresses. This
709                  *  may occur if software modifies the paging structures so that
710                  *  the page size used for the address range changes. If the two
711                  *  translations differ with respect to page frame or attributes
712                  *  (e.g., permissions), processor behavior is undefined and may
713                  *  be implementation-specific."
714                  *
715                  * We do this global tlb flush inside the cpa_lock, so that we
716                  * don't allow any other cpu, with stale tlb entries change the
717                  * page attribute in parallel, that also falls into the
718                  * just split large page entry.
719                  */
720                 flush_tlb_all();
721                 goto repeat;
722         }
723
724         return err;
725 }
726
727 static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias);
728
729 static int cpa_process_alias(struct cpa_data *cpa)
730 {
731         struct cpa_data alias_cpa;
732         unsigned long laddr = (unsigned long)__va(cpa->pfn << PAGE_SHIFT);
733         unsigned long vaddr;
734         int ret;
735
736         if (cpa->pfn >= (arm_lowmem_limit >> PAGE_SHIFT))
737                 return 0;
738
739         /*
740          * No need to redo, when the primary call touched the direct
741          * mapping already:
742          */
743         if (cpa->flags & CPA_PAGES_ARRAY) {
744                 struct page *page = cpa->pages[cpa->curpage];
745                 if (unlikely(PageHighMem(page)))
746                         return 0;
747                 vaddr = (unsigned long)page_address(page);
748         } else if (cpa->flags & CPA_ARRAY)
749                 vaddr = cpa->vaddr[cpa->curpage];
750         else
751                 vaddr = *cpa->vaddr;
752
753         if (!(within(vaddr, PAGE_OFFSET,
754                     PAGE_OFFSET + arm_lowmem_limit))) {
755
756                 alias_cpa = *cpa;
757                 alias_cpa.vaddr = &laddr;
758                 alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
759
760                 ret = __change_page_attr_set_clr(&alias_cpa, 0);
761                 if (ret)
762                         return ret;
763         }
764
765         return 0;
766 }
767
768 static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias)
769 {
770         int ret, numpages = cpa->numpages;
771
772         while (numpages) {
773                 /*
774                  * Store the remaining nr of pages for the large page
775                  * preservation check.
776                  */
777                 cpa->numpages = numpages;
778                 /* for array changes, we can't use large page */
779                 if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY))
780                         cpa->numpages = 1;
781
782                 if (!debug_pagealloc)
783                         mutex_lock(&cpa_lock);
784                 ret = __change_page_attr(cpa, checkalias);
785                 if (!debug_pagealloc)
786                         mutex_unlock(&cpa_lock);
787                 if (ret)
788                         return ret;
789
790                 if (checkalias) {
791                         ret = cpa_process_alias(cpa);
792                         if (ret)
793                                 return ret;
794                 }
795
796                 /*
797                  * Adjust the number of pages with the result of the
798                  * CPA operation. Either a large page has been
799                  * preserved or a single page update happened.
800                  */
801                 BUG_ON(cpa->numpages > numpages);
802                 numpages -= cpa->numpages;
803                 if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY))
804                         cpa->curpage++;
805                 else
806                         *cpa->vaddr += cpa->numpages * PAGE_SIZE;
807         }
808         return 0;
809 }
810
811 static inline int cache_attr(pgprot_t attr)
812 {
813         /*
814          * We need to flush the cache for all memory type changes
815          * except when a page is being marked write back cacheable
816          */
817         return !((pgprot_val(attr) & L_PTE_MT_MASK) == L_PTE_MT_KERNEL);
818 }
819
820 static int change_page_attr_set_clr(unsigned long *addr, int numpages,
821                                     pgprot_t mask_set, pgprot_t mask_clr,
822                                     int force_split, int in_flag,
823                                     struct page **pages)
824 {
825         struct cpa_data cpa;
826         int ret, cache, checkalias;
827         unsigned long baddr = 0;
828
829         if (!pgprot_val(mask_set) && !pgprot_val(mask_clr) && !force_split)
830                 return 0;
831
832         /* Ensure we are PAGE_SIZE aligned */
833         if (in_flag & CPA_ARRAY) {
834                 int i;
835                 for (i = 0; i < numpages; i++) {
836                         if (addr[i] & ~PAGE_MASK) {
837                                 addr[i] &= PAGE_MASK;
838                                 WARN_ON_ONCE(1);
839                         }
840                 }
841         } else if (!(in_flag & CPA_PAGES_ARRAY)) {
842                 /*
843                  * in_flag of CPA_PAGES_ARRAY implies it is aligned.
844                  * No need to cehck in that case
845                  */
846                 if (*addr & ~PAGE_MASK) {
847                         *addr &= PAGE_MASK;
848                         /*
849                          * People should not be passing in unaligned addresses:
850                          */
851                         WARN_ON_ONCE(1);
852                 }
853                 /*
854                  * Save address for cache flush. *addr is modified in the call
855                  * to __change_page_attr_set_clr() below.
856                  */
857                 baddr = *addr;
858         }
859
860         /* Must avoid aliasing mappings in the highmem code */
861         kmap_flush_unused();
862
863         vm_unmap_aliases();
864
865         cpa.vaddr = addr;
866         cpa.pages = pages;
867         cpa.numpages = numpages;
868         cpa.mask_set = mask_set;
869         cpa.mask_clr = mask_clr;
870         cpa.flags = 0;
871         cpa.curpage = 0;
872         cpa.force_split = force_split;
873
874         if (in_flag & (CPA_ARRAY | CPA_PAGES_ARRAY))
875                 cpa.flags |= in_flag;
876
877         /* No alias checking for XN bit modifications */
878         checkalias = (pgprot_val(mask_set) |
879                                 pgprot_val(mask_clr)) != L_PTE_XN;
880
881         ret = __change_page_attr_set_clr(&cpa, checkalias);
882
883         cache = cache_attr(mask_set);
884         /*
885          * Check whether we really changed something or
886          * cache need to be flushed.
887          */
888         if (!(cpa.flags & CPA_FLUSHTLB) && !cache)
889                 goto out;
890
891         if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) {
892                 cpa_flush_array(addr, numpages, cache,
893                                 cpa.flags, pages);
894         } else
895                 cpa_flush_range(baddr, numpages, cache);
896
897 out:
898         return ret;
899 }
900
901 static inline int change_page_attr_set(unsigned long *addr, int numpages,
902                                        pgprot_t mask, int array)
903 {
904         return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,
905                 (array ? CPA_ARRAY : 0), NULL);
906 }
907
908 static inline int change_page_attr_clear(unsigned long *addr, int numpages,
909                                          pgprot_t mask, int array)
910 {
911         return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,
912                 (array ? CPA_ARRAY : 0), NULL);
913 }
914
915 static inline int cpa_set_pages_array(struct page **pages, int numpages,
916                                        pgprot_t mask)
917 {
918         return change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0,
919                 CPA_PAGES_ARRAY, pages);
920 }
921
922 static inline int cpa_clear_pages_array(struct page **pages, int numpages,
923                                          pgprot_t mask)
924 {
925         return change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0,
926                 CPA_PAGES_ARRAY, pages);
927 }
928
929 int set_memory_uc(unsigned long addr, int numpages)
930 {
931         return change_page_attr_set_clr(&addr, numpages,
932                 __pgprot(L_PTE_MT_UNCACHED),
933                         __pgprot(L_PTE_MT_MASK), 0, 0, NULL);
934 }
935 EXPORT_SYMBOL(set_memory_uc);
936
937 int _set_memory_array(unsigned long *addr, int addrinarray,
938                 unsigned long set, unsigned long clr)
939 {
940         return change_page_attr_set_clr(addr, addrinarray, __pgprot(set),
941                 __pgprot(clr), 0, CPA_ARRAY, NULL);
942 }
943
944 int set_memory_array_uc(unsigned long *addr, int addrinarray)
945 {
946         return _set_memory_array(addr, addrinarray,
947                 L_PTE_MT_UNCACHED, L_PTE_MT_MASK);
948 }
949 EXPORT_SYMBOL(set_memory_array_uc);
950
951 int set_memory_array_wc(unsigned long *addr, int addrinarray)
952 {
953         return _set_memory_array(addr, addrinarray,
954                 L_PTE_MT_BUFFERABLE, L_PTE_MT_MASK);
955 }
956 EXPORT_SYMBOL(set_memory_array_wc);
957
958 int set_memory_wc(unsigned long addr, int numpages)
959 {
960         int ret;
961
962         ret = change_page_attr_set_clr(&addr, numpages,
963                         __pgprot(L_PTE_MT_BUFFERABLE),
964                         __pgprot(L_PTE_MT_MASK),
965                         0, 0, NULL);
966         return ret;
967 }
968 EXPORT_SYMBOL(set_memory_wc);
969
970 int set_memory_wb(unsigned long addr, int numpages)
971 {
972         return change_page_attr_set_clr(&addr, numpages,
973                         __pgprot(L_PTE_MT_KERNEL),
974                         __pgprot(L_PTE_MT_MASK),
975                         0, 0, NULL);
976 }
977 EXPORT_SYMBOL(set_memory_wb);
978
979 int set_memory_iwb(unsigned long addr, int numpages)
980 {
981         return change_page_attr_set_clr(&addr, numpages,
982                         __pgprot(L_PTE_MT_INNER_WB),
983                         __pgprot(L_PTE_MT_MASK),
984                         0, 0, NULL);
985 }
986 EXPORT_SYMBOL(set_memory_iwb);
987
988 int set_memory_array_wb(unsigned long *addr, int addrinarray)
989 {
990         return change_page_attr_set_clr(addr, addrinarray,
991                         __pgprot(L_PTE_MT_KERNEL),
992                         __pgprot(L_PTE_MT_MASK),
993                         0, CPA_ARRAY, NULL);
994
995 }
996 EXPORT_SYMBOL(set_memory_array_wb);
997
998 int set_memory_array_iwb(unsigned long *addr, int addrinarray)
999 {
1000         return change_page_attr_set_clr(addr, addrinarray,
1001                         __pgprot(L_PTE_MT_INNER_WB),
1002                         __pgprot(L_PTE_MT_MASK),
1003                         0, CPA_ARRAY, NULL);
1004
1005 }
1006 EXPORT_SYMBOL(set_memory_array_iwb);
1007
1008 int set_memory_x(unsigned long addr, int numpages)
1009 {
1010         return change_page_attr_clear(&addr, numpages,
1011                 __pgprot(L_PTE_XN), 0);
1012 }
1013 EXPORT_SYMBOL(set_memory_x);
1014
1015 int set_memory_nx(unsigned long addr, int numpages)
1016 {
1017         return change_page_attr_set(&addr, numpages,
1018                 __pgprot(L_PTE_XN), 0);
1019 }
1020 EXPORT_SYMBOL(set_memory_nx);
1021
1022 int set_memory_ro(unsigned long addr, int numpages)
1023 {
1024         return change_page_attr_set(&addr, numpages,
1025                 __pgprot(L_PTE_RDONLY), 0);
1026 }
1027 EXPORT_SYMBOL_GPL(set_memory_ro);
1028
1029 int set_memory_rw(unsigned long addr, int numpages)
1030 {
1031         return change_page_attr_clear(&addr, numpages,
1032                 __pgprot(L_PTE_RDONLY), 0);
1033 }
1034 EXPORT_SYMBOL_GPL(set_memory_rw);
1035
1036 int set_memory_np(unsigned long addr, int numpages)
1037 {
1038         return change_page_attr_clear(&addr, numpages,
1039                  __pgprot(L_PTE_PRESENT), 0);
1040 }
1041
1042 int set_memory_4k(unsigned long addr, int numpages)
1043 {
1044         return change_page_attr_set_clr(&addr, numpages, __pgprot(0),
1045                                         __pgprot(0), 1, 0, NULL);
1046 }
1047
1048 static int _set_pages_array(struct page **pages, int addrinarray,
1049                 unsigned long set, unsigned long clr)
1050 {
1051         return change_page_attr_set_clr(NULL, addrinarray,
1052                         __pgprot(set),
1053                         __pgprot(clr),
1054                         0, CPA_PAGES_ARRAY, pages);
1055 }
1056
1057 int set_pages_array_uc(struct page **pages, int addrinarray)
1058 {
1059         return _set_pages_array(pages, addrinarray,
1060                         L_PTE_MT_UNCACHED, L_PTE_MT_MASK);
1061 }
1062 EXPORT_SYMBOL(set_pages_array_uc);
1063
1064 int set_pages_array_wc(struct page **pages, int addrinarray)
1065 {
1066         return _set_pages_array(pages, addrinarray, L_PTE_MT_BUFFERABLE,
1067                         L_PTE_MT_MASK);
1068 }
1069 EXPORT_SYMBOL(set_pages_array_wc);
1070
1071 int set_pages_array_wb(struct page **pages, int addrinarray)
1072 {
1073         return _set_pages_array(pages, addrinarray,
1074                         L_PTE_MT_KERNEL, L_PTE_MT_MASK);
1075 }
1076 EXPORT_SYMBOL(set_pages_array_wb);
1077
1078 int set_pages_array_iwb(struct page **pages, int addrinarray)
1079 {
1080         return _set_pages_array(pages, addrinarray,
1081                         L_PTE_MT_INNER_WB, L_PTE_MT_MASK);
1082 }
1083 EXPORT_SYMBOL(set_pages_array_iwb);
1084
1085 #else /* CONFIG_CPA */
1086
1087 void update_page_count(int level, unsigned long pages)
1088 {
1089 }
1090
1091 static void flush_cache(struct page **pages, int numpages)
1092 {
1093         unsigned int i;
1094         bool flush_inner = true;
1095         unsigned long base;
1096
1097 #if defined(CONFIG_NVMAP_CACHE_MAINT_BY_SET_WAYS)
1098         if (numpages >= (inner_cache_maint_threshold >> PAGE_SHIFT)) {
1099                 inner_flush_cache_all();
1100                 flush_inner = false;
1101         }
1102 #endif
1103
1104         for (i = 0; i < numpages; i++) {
1105                 if (flush_inner)
1106                         __flush_dcache_page(page_mapping(pages[i]), pages[i]);
1107                 base = page_to_phys(pages[i]);
1108                 outer_flush_range(base, base + PAGE_SIZE);
1109         }
1110 }
1111
1112 int set_pages_array_uc(struct page **pages, int addrinarray)
1113 {
1114         flush_cache(pages, addrinarray);
1115         return 0;
1116 }
1117 EXPORT_SYMBOL(set_pages_array_uc);
1118
1119 int set_pages_array_wc(struct page **pages, int addrinarray)
1120 {
1121         flush_cache(pages, addrinarray);
1122         return 0;
1123 }
1124 EXPORT_SYMBOL(set_pages_array_wc);
1125
1126 int set_pages_array_wb(struct page **pages, int addrinarray)
1127 {
1128         return 0;
1129 }
1130 EXPORT_SYMBOL(set_pages_array_wb);
1131
1132 int set_pages_array_iwb(struct page **pages, int addrinarray)
1133 {
1134         flush_cache(pages, addrinarray);
1135         return 0;
1136 }
1137 EXPORT_SYMBOL(set_pages_array_iwb);
1138
1139 #endif
1140
1141 static __init int init_cache_size(void)
1142 {
1143         unsigned int id;
1144
1145         id = read_cpuid_id();
1146         /* cortex-a9 */
1147         if ((id >> 4 & 0xfff) == 0xc09)
1148                 inner_cache_maint_threshold = SZ_32K;
1149         pr_info("CPA: inner cache maint threshold=%d",
1150                 inner_cache_maint_threshold);
1151         return 0;
1152 }
1153
1154 core_initcall(init_cache_size);