uml: fold mmu_context_skas into mm_context
[linux-2.6.git] / arch / um / kernel / tlb.c
1 /*
2  * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com)
3  * Licensed under the GPL
4  */
5
6 #include "linux/mm.h"
7 #include "asm/pgtable.h"
8 #include "asm/tlbflush.h"
9 #include "as-layout.h"
10 #include "mem_user.h"
11 #include "os.h"
12 #include "skas.h"
13 #include "tlb.h"
14
15 static int add_mmap(unsigned long virt, unsigned long phys, unsigned long len,
16                     unsigned int prot, struct host_vm_op *ops, int *index,
17                     int last_filled, struct mm_context *mmu, void **flush,
18                     int (*do_ops)(struct mm_context *, struct host_vm_op *,
19                                   int, int, void **))
20 {
21         __u64 offset;
22         struct host_vm_op *last;
23         int fd, ret = 0;
24
25         fd = phys_mapping(phys, &offset);
26         if (*index != -1) {
27                 last = &ops[*index];
28                 if ((last->type == MMAP) &&
29                    (last->u.mmap.addr + last->u.mmap.len == virt) &&
30                    (last->u.mmap.prot == prot) && (last->u.mmap.fd == fd) &&
31                    (last->u.mmap.offset + last->u.mmap.len == offset)) {
32                         last->u.mmap.len += len;
33                         return 0;
34                 }
35         }
36
37         if (*index == last_filled) {
38                 ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
39                 *index = -1;
40         }
41
42         ops[++*index] = ((struct host_vm_op) { .type    = MMAP,
43                                                 .u = { .mmap = {
44                                                        .addr    = virt,
45                                                        .len     = len,
46                                                        .prot    = prot,
47                                                        .fd      = fd,
48                                                        .offset  = offset }
49                            } });
50         return ret;
51 }
52
53 static int add_munmap(unsigned long addr, unsigned long len,
54                       struct host_vm_op *ops, int *index, int last_filled,
55                       struct mm_context *mmu, void **flush,
56                       int (*do_ops)(struct mm_context *, struct host_vm_op *,
57                                     int, int, void **))
58 {
59         struct host_vm_op *last;
60         int ret = 0;
61
62         if (*index != -1) {
63                 last = &ops[*index];
64                 if ((last->type == MUNMAP) &&
65                    (last->u.munmap.addr + last->u.mmap.len == addr)) {
66                         last->u.munmap.len += len;
67                         return 0;
68                 }
69         }
70
71         if (*index == last_filled) {
72                 ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
73                 *index = -1;
74         }
75
76         ops[++*index] = ((struct host_vm_op) { .type    = MUNMAP,
77                                                .u = { .munmap = {
78                                                         .addr   = addr,
79                                                         .len    = len } } });
80         return ret;
81 }
82
83 static int add_mprotect(unsigned long addr, unsigned long len,
84                         unsigned int prot, struct host_vm_op *ops, int *index,
85                         int last_filled, struct mm_context *mmu, void **flush,
86                         int (*do_ops)(struct mm_context *, struct host_vm_op *,
87                                       int, int, void **))
88 {
89         struct host_vm_op *last;
90         int ret = 0;
91
92         if (*index != -1) {
93                 last = &ops[*index];
94                 if ((last->type == MPROTECT) &&
95                    (last->u.mprotect.addr + last->u.mprotect.len == addr) &&
96                    (last->u.mprotect.prot == prot)) {
97                         last->u.mprotect.len += len;
98                         return 0;
99                 }
100         }
101
102         if (*index == last_filled) {
103                 ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
104                 *index = -1;
105         }
106
107         ops[++*index] = ((struct host_vm_op) { .type    = MPROTECT,
108                                                .u = { .mprotect = {
109                                                        .addr    = addr,
110                                                        .len     = len,
111                                                        .prot    = prot } } });
112         return ret;
113 }
114
115 #define ADD_ROUND(n, inc) (((n) + (inc)) & ~((inc) - 1))
116
117 static inline int update_pte_range(pmd_t *pmd, unsigned long addr,
118                                    unsigned long end, struct host_vm_op *ops,
119                                    int last_op, int *op_index, int force,
120                                    struct mm_context *mmu, void **flush,
121                                    int (*do_ops)(struct mm_context *,
122                                                  struct host_vm_op *, int, int,
123                                                  void **))
124 {
125         pte_t *pte;
126         int r, w, x, prot, ret = 0;
127
128         pte = pte_offset_kernel(pmd, addr);
129         do {
130                 r = pte_read(*pte);
131                 w = pte_write(*pte);
132                 x = pte_exec(*pte);
133                 if (!pte_young(*pte)) {
134                         r = 0;
135                         w = 0;
136                 } else if (!pte_dirty(*pte)) {
137                         w = 0;
138                 }
139                 prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) |
140                         (x ? UM_PROT_EXEC : 0));
141                 if (force || pte_newpage(*pte)) {
142                         if (pte_present(*pte))
143                                 ret = add_mmap(addr, pte_val(*pte) & PAGE_MASK,
144                                                PAGE_SIZE, prot, ops, op_index,
145                                                last_op, mmu, flush, do_ops);
146                         else ret = add_munmap(addr, PAGE_SIZE, ops, op_index,
147                                               last_op, mmu, flush, do_ops);
148                 }
149                 else if (pte_newprot(*pte))
150                         ret = add_mprotect(addr, PAGE_SIZE, prot, ops, op_index,
151                                            last_op, mmu, flush, do_ops);
152                 *pte = pte_mkuptodate(*pte);
153         } while (pte++, addr += PAGE_SIZE, ((addr != end) && !ret));
154         return ret;
155 }
156
157 static inline int update_pmd_range(pud_t *pud, unsigned long addr,
158                                    unsigned long end, struct host_vm_op *ops,
159                                    int last_op, int *op_index, int force,
160                                    struct mm_context *mmu, void **flush,
161                                    int (*do_ops)(struct mm_context *,
162                                                  struct host_vm_op *, int, int,
163                                                  void **))
164 {
165         pmd_t *pmd;
166         unsigned long next;
167         int ret = 0;
168
169         pmd = pmd_offset(pud, addr);
170         do {
171                 next = pmd_addr_end(addr, end);
172                 if (!pmd_present(*pmd)) {
173                         if (force || pmd_newpage(*pmd)) {
174                                 ret = add_munmap(addr, next - addr, ops,
175                                                  op_index, last_op, mmu,
176                                                  flush, do_ops);
177                                 pmd_mkuptodate(*pmd);
178                         }
179                 }
180                 else ret = update_pte_range(pmd, addr, next, ops, last_op,
181                                             op_index, force, mmu, flush,
182                                             do_ops);
183         } while (pmd++, addr = next, ((addr != end) && !ret));
184         return ret;
185 }
186
187 static inline int update_pud_range(pgd_t *pgd, unsigned long addr,
188                                    unsigned long end, struct host_vm_op *ops,
189                                    int last_op, int *op_index, int force,
190                                    struct mm_context *mmu, void **flush,
191                                    int (*do_ops)(struct mm_context *,
192                                                  struct host_vm_op *, int, int,
193                                                  void **))
194 {
195         pud_t *pud;
196         unsigned long next;
197         int ret = 0;
198
199         pud = pud_offset(pgd, addr);
200         do {
201                 next = pud_addr_end(addr, end);
202                 if (!pud_present(*pud)) {
203                         if (force || pud_newpage(*pud)) {
204                                 ret = add_munmap(addr, next - addr, ops,
205                                                  op_index, last_op, mmu,
206                                                  flush, do_ops);
207                                 pud_mkuptodate(*pud);
208                         }
209                 }
210                 else ret = update_pmd_range(pud, addr, next, ops, last_op,
211                                             op_index, force, mmu, flush,
212                                             do_ops);
213         } while (pud++, addr = next, ((addr != end) && !ret));
214         return ret;
215 }
216
217 void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
218                       unsigned long end_addr, int force,
219                       int (*do_ops)(struct mm_context *, struct host_vm_op *,
220                                     int, int, void **))
221 {
222         pgd_t *pgd;
223         struct mm_context *mmu = &mm->context;
224         struct host_vm_op ops[1];
225         unsigned long addr = start_addr, next;
226         int ret = 0, last_op = ARRAY_SIZE(ops) - 1, op_index = -1;
227         void *flush = NULL;
228
229         ops[0].type = NONE;
230         pgd = pgd_offset(mm, addr);
231         do {
232                 next = pgd_addr_end(addr, end_addr);
233                 if (!pgd_present(*pgd)) {
234                         if (force || pgd_newpage(*pgd)) {
235                                 ret = add_munmap(addr, next - addr, ops,
236                                                  &op_index, last_op, mmu,
237                                                  &flush, do_ops);
238                                 pgd_mkuptodate(*pgd);
239                         }
240                 }
241                 else ret = update_pud_range(pgd, addr, next, ops, last_op,
242                                             &op_index, force, mmu, &flush,
243                                             do_ops);
244         } while (pgd++, addr = next, ((addr != end_addr) && !ret));
245
246         if (!ret)
247                 ret = (*do_ops)(mmu, ops, op_index, 1, &flush);
248
249         /* This is not an else because ret is modified above */
250         if (ret) {
251                 printk(KERN_ERR "fix_range_common: failed, killing current "
252                        "process\n");
253                 force_sig(SIGKILL, current);
254         }
255 }
256
257 int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
258 {
259         struct mm_struct *mm;
260         pgd_t *pgd;
261         pud_t *pud;
262         pmd_t *pmd;
263         pte_t *pte;
264         unsigned long addr, last;
265         int updated = 0, err;
266
267         mm = &init_mm;
268         for (addr = start; addr < end;) {
269                 pgd = pgd_offset(mm, addr);
270                 if (!pgd_present(*pgd)) {
271                         last = ADD_ROUND(addr, PGDIR_SIZE);
272                         if (last > end)
273                                 last = end;
274                         if (pgd_newpage(*pgd)) {
275                                 updated = 1;
276                                 err = os_unmap_memory((void *) addr,
277                                                       last - addr);
278                                 if (err < 0)
279                                         panic("munmap failed, errno = %d\n",
280                                               -err);
281                         }
282                         addr = last;
283                         continue;
284                 }
285
286                 pud = pud_offset(pgd, addr);
287                 if (!pud_present(*pud)) {
288                         last = ADD_ROUND(addr, PUD_SIZE);
289                         if (last > end)
290                                 last = end;
291                         if (pud_newpage(*pud)) {
292                                 updated = 1;
293                                 err = os_unmap_memory((void *) addr,
294                                                       last - addr);
295                                 if (err < 0)
296                                         panic("munmap failed, errno = %d\n",
297                                               -err);
298                         }
299                         addr = last;
300                         continue;
301                 }
302
303                 pmd = pmd_offset(pud, addr);
304                 if (!pmd_present(*pmd)) {
305                         last = ADD_ROUND(addr, PMD_SIZE);
306                         if (last > end)
307                                 last = end;
308                         if (pmd_newpage(*pmd)) {
309                                 updated = 1;
310                                 err = os_unmap_memory((void *) addr,
311                                                       last - addr);
312                                 if (err < 0)
313                                         panic("munmap failed, errno = %d\n",
314                                               -err);
315                         }
316                         addr = last;
317                         continue;
318                 }
319
320                 pte = pte_offset_kernel(pmd, addr);
321                 if (!pte_present(*pte) || pte_newpage(*pte)) {
322                         updated = 1;
323                         err = os_unmap_memory((void *) addr,
324                                               PAGE_SIZE);
325                         if (err < 0)
326                                 panic("munmap failed, errno = %d\n",
327                                       -err);
328                         if (pte_present(*pte))
329                                 map_memory(addr,
330                                            pte_val(*pte) & PAGE_MASK,
331                                            PAGE_SIZE, 1, 1, 1);
332                 }
333                 else if (pte_newprot(*pte)) {
334                         updated = 1;
335                         os_protect_memory((void *) addr, PAGE_SIZE, 1, 1, 1);
336                 }
337                 addr += PAGE_SIZE;
338         }
339         return updated;
340 }
341
342 void flush_tlb_page(struct vm_area_struct *vma, unsigned long address)
343 {
344         pgd_t *pgd;
345         pud_t *pud;
346         pmd_t *pmd;
347         pte_t *pte;
348         struct mm_struct *mm = vma->vm_mm;
349         void *flush = NULL;
350         int r, w, x, prot, err = 0;
351         struct mm_id *mm_id;
352
353         address &= PAGE_MASK;
354         pgd = pgd_offset(mm, address);
355         if (!pgd_present(*pgd))
356                 goto kill;
357
358         pud = pud_offset(pgd, address);
359         if (!pud_present(*pud))
360                 goto kill;
361
362         pmd = pmd_offset(pud, address);
363         if (!pmd_present(*pmd))
364                 goto kill;
365
366         pte = pte_offset_kernel(pmd, address);
367
368         r = pte_read(*pte);
369         w = pte_write(*pte);
370         x = pte_exec(*pte);
371         if (!pte_young(*pte)) {
372                 r = 0;
373                 w = 0;
374         } else if (!pte_dirty(*pte)) {
375                 w = 0;
376         }
377
378         mm_id = &mm->context.id;
379         prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) |
380                 (x ? UM_PROT_EXEC : 0));
381         if (pte_newpage(*pte)) {
382                 if (pte_present(*pte)) {
383                         unsigned long long offset;
384                         int fd;
385
386                         fd = phys_mapping(pte_val(*pte) & PAGE_MASK, &offset);
387                         err = map(mm_id, address, PAGE_SIZE, prot, fd, offset,
388                                   1, &flush);
389                 }
390                 else err = unmap(mm_id, address, PAGE_SIZE, 1, &flush);
391         }
392         else if (pte_newprot(*pte))
393                 err = protect(mm_id, address, PAGE_SIZE, prot, 1, &flush);
394
395         if (err)
396                 goto kill;
397
398         *pte = pte_mkuptodate(*pte);
399
400         return;
401
402 kill:
403         printk(KERN_ERR "Failed to flush page for address 0x%lx\n", address);
404         force_sig(SIGKILL, current);
405 }
406
407 pgd_t *pgd_offset_proc(struct mm_struct *mm, unsigned long address)
408 {
409         return pgd_offset(mm, address);
410 }
411
412 pud_t *pud_offset_proc(pgd_t *pgd, unsigned long address)
413 {
414         return pud_offset(pgd, address);
415 }
416
417 pmd_t *pmd_offset_proc(pud_t *pud, unsigned long address)
418 {
419         return pmd_offset(pud, address);
420 }
421
422 pte_t *pte_offset_proc(pmd_t *pmd, unsigned long address)
423 {
424         return pte_offset_kernel(pmd, address);
425 }
426
427 pte_t *addr_pte(struct task_struct *task, unsigned long addr)
428 {
429         pgd_t *pgd = pgd_offset(task->mm, addr);
430         pud_t *pud = pud_offset(pgd, addr);
431         pmd_t *pmd = pmd_offset(pud, addr);
432
433         return pte_offset_map(pmd, addr);
434 }
435
436 void flush_tlb_all(void)
437 {
438         flush_tlb_mm(current->mm);
439 }
440
441 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
442 {
443         flush_tlb_kernel_range_common(start, end);
444 }
445
446 void flush_tlb_kernel_vm(void)
447 {
448         flush_tlb_kernel_range_common(start_vm, end_vm);
449 }
450
451 void __flush_tlb_one(unsigned long addr)
452 {
453         flush_tlb_kernel_range_common(addr, addr + PAGE_SIZE);
454 }
455
456 static int do_ops(struct mm_context *mmu, struct host_vm_op *ops, int last,
457                   int finished, void **flush)
458 {
459         struct host_vm_op *op;
460         int i, ret = 0;
461
462         for (i = 0; i <= last && !ret; i++) {
463         op = &ops[i];
464                 switch(op->type) {
465                 case MMAP:
466                         ret = map(&mmu->id, op->u.mmap.addr, op->u.mmap.len,
467                                   op->u.mmap.prot, op->u.mmap.fd,
468                                   op->u.mmap.offset, finished, flush);
469                         break;
470                 case MUNMAP:
471                         ret = unmap(&mmu->id, op->u.munmap.addr,
472                                     op->u.munmap.len, finished, flush);
473                         break;
474                 case MPROTECT:
475                         ret = protect(&mmu->id, op->u.mprotect.addr,
476                                       op->u.mprotect.len, op->u.mprotect.prot,
477                                       finished, flush);
478                         break;
479                 default:
480                         printk(KERN_ERR "Unknown op type %d in do_ops\n",
481                                op->type);
482                         break;
483                 }
484         }
485
486         return ret;
487 }
488
489 static void fix_range(struct mm_struct *mm, unsigned long start_addr,
490                       unsigned long end_addr, int force)
491 {
492         if (!proc_mm && (end_addr > CONFIG_STUB_START))
493                 end_addr = CONFIG_STUB_START;
494
495         fix_range_common(mm, start_addr, end_addr, force, do_ops);
496 }
497
498 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
499                      unsigned long end)
500 {
501         if (vma->vm_mm == NULL)
502                 flush_tlb_kernel_range_common(start, end);
503         else fix_range(vma->vm_mm, start, end, 0);
504 }
505
506 void flush_tlb_mm(struct mm_struct *mm)
507 {
508         unsigned long end;
509
510         /*
511          * Don't bother flushing if this address space is about to be
512          * destroyed.
513          */
514         if (atomic_read(&mm->mm_users) == 0)
515                 return;
516
517         end = proc_mm ? task_size : CONFIG_STUB_START;
518         fix_range(mm, 0, end, 0);
519 }
520
521 void force_flush_all(void)
522 {
523         struct mm_struct *mm = current->mm;
524         struct vm_area_struct *vma = mm->mmap;
525
526         while (vma != NULL) {
527                 fix_range(mm, vma->vm_start, vma->vm_end, 1);
528                 vma = vma->vm_next;
529         }
530 }