uml: remove debugging remnants
[linux-2.6.git] / arch / um / kernel / tlb.c
1 /*
2  * Copyright (C) 2000, 2001, 2002 Jeff Dike (jdike@karaya.com)
3  * Licensed under the GPL
4  */
5
6 #include "linux/mm.h"
7 #include "asm/page.h"
8 #include "asm/pgalloc.h"
9 #include "asm/pgtable.h"
10 #include "asm/tlbflush.h"
11 #include "choose-mode.h"
12 #include "mode_kern.h"
13 #include "as-layout.h"
14 #include "tlb.h"
15 #include "mem.h"
16 #include "mem_user.h"
17 #include "os.h"
18
19 static int add_mmap(unsigned long virt, unsigned long phys, unsigned long len,
20                     int r, int w, int x, struct host_vm_op *ops, int *index,
21                     int last_filled, union mm_context *mmu, void **flush,
22                     int (*do_ops)(union mm_context *, struct host_vm_op *,
23                                   int, int, void **))
24 {
25         __u64 offset;
26         struct host_vm_op *last;
27         int fd, ret = 0;
28
29         fd = phys_mapping(phys, &offset);
30         if(*index != -1){
31                 last = &ops[*index];
32                 if((last->type == MMAP) &&
33                    (last->u.mmap.addr + last->u.mmap.len == virt) &&
34                    (last->u.mmap.r == r) && (last->u.mmap.w == w) &&
35                    (last->u.mmap.x == x) && (last->u.mmap.fd == fd) &&
36                    (last->u.mmap.offset + last->u.mmap.len == offset)){
37                         last->u.mmap.len += len;
38                         return 0;
39                 }
40         }
41
42         if(*index == last_filled){
43                 ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
44                 *index = -1;
45         }
46
47         ops[++*index] = ((struct host_vm_op) { .type    = MMAP,
48                                                 .u = { .mmap = {
49                                                        .addr    = virt,
50                                                        .len     = len,
51                                                        .r       = r,
52                                                        .w       = w,
53                                                        .x       = x,
54                                                        .fd      = fd,
55                                                        .offset  = offset }
56                            } });
57         return ret;
58 }
59
60 static int add_munmap(unsigned long addr, unsigned long len,
61                       struct host_vm_op *ops, int *index, int last_filled,
62                       union mm_context *mmu, void **flush,
63                       int (*do_ops)(union mm_context *, struct host_vm_op *,
64                                     int, int, void **))
65 {
66         struct host_vm_op *last;
67         int ret = 0;
68
69         if(*index != -1){
70                 last = &ops[*index];
71                 if((last->type == MUNMAP) &&
72                    (last->u.munmap.addr + last->u.mmap.len == addr)){
73                         last->u.munmap.len += len;
74                         return 0;
75                 }
76         }
77
78         if(*index == last_filled){
79                 ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
80                 *index = -1;
81         }
82
83         ops[++*index] = ((struct host_vm_op) { .type    = MUNMAP,
84                                                .u = { .munmap = {
85                                                         .addr   = addr,
86                                                         .len    = len } } });
87         return ret;
88 }
89
90 static int add_mprotect(unsigned long addr, unsigned long len, int r, int w,
91                         int x, struct host_vm_op *ops, int *index,
92                         int last_filled, union mm_context *mmu, void **flush,
93                         int (*do_ops)(union mm_context *, struct host_vm_op *,
94                                       int, int, void **))
95 {
96         struct host_vm_op *last;
97         int ret = 0;
98
99         if(*index != -1){
100                 last = &ops[*index];
101                 if((last->type == MPROTECT) &&
102                    (last->u.mprotect.addr + last->u.mprotect.len == addr) &&
103                    (last->u.mprotect.r == r) && (last->u.mprotect.w == w) &&
104                    (last->u.mprotect.x == x)){
105                         last->u.mprotect.len += len;
106                         return 0;
107                 }
108         }
109
110         if(*index == last_filled){
111                 ret = (*do_ops)(mmu, ops, last_filled, 0, flush);
112                 *index = -1;
113         }
114
115         ops[++*index] = ((struct host_vm_op) { .type    = MPROTECT,
116                                                .u = { .mprotect = {
117                                                        .addr    = addr,
118                                                        .len     = len,
119                                                        .r       = r,
120                                                        .w       = w,
121                                                        .x       = x } } });
122         return ret;
123 }
124
125 #define ADD_ROUND(n, inc) (((n) + (inc)) & ~((inc) - 1))
126
127 static inline int update_pte_range(pmd_t *pmd, unsigned long addr,
128                                    unsigned long end, struct host_vm_op *ops,
129                                    int last_op, int *op_index, int force,
130                                    union mm_context *mmu, void **flush,
131                                    int (*do_ops)(union mm_context *,
132                                                  struct host_vm_op *, int, int,
133                                                  void **))
134 {
135         pte_t *pte;
136         int r, w, x, ret = 0;
137
138         pte = pte_offset_kernel(pmd, addr);
139         do {
140                 r = pte_read(*pte);
141                 w = pte_write(*pte);
142                 x = pte_exec(*pte);
143                 if (!pte_young(*pte)) {
144                         r = 0;
145                         w = 0;
146                 } else if (!pte_dirty(*pte)) {
147                         w = 0;
148                 }
149                 if(force || pte_newpage(*pte)){
150                         if(pte_present(*pte))
151                                 ret = add_mmap(addr, pte_val(*pte) & PAGE_MASK,
152                                                PAGE_SIZE, r, w, x, ops,
153                                                op_index, last_op, mmu, flush,
154                                                do_ops);
155                         else ret = add_munmap(addr, PAGE_SIZE, ops, op_index,
156                                               last_op, mmu, flush, do_ops);
157                 }
158                 else if(pte_newprot(*pte))
159                         ret = add_mprotect(addr, PAGE_SIZE, r, w, x, ops,
160                                            op_index, last_op, mmu, flush,
161                                            do_ops);
162                 *pte = pte_mkuptodate(*pte);
163         } while (pte++, addr += PAGE_SIZE, ((addr != end) && !ret));
164         return ret;
165 }
166
167 static inline int update_pmd_range(pud_t *pud, unsigned long addr,
168                                    unsigned long end, struct host_vm_op *ops,
169                                    int last_op, int *op_index, int force,
170                                    union mm_context *mmu, void **flush,
171                                    int (*do_ops)(union mm_context *,
172                                                  struct host_vm_op *, int, int,
173                                                  void **))
174 {
175         pmd_t *pmd;
176         unsigned long next;
177         int ret = 0;
178
179         pmd = pmd_offset(pud, addr);
180         do {
181                 next = pmd_addr_end(addr, end);
182                 if(!pmd_present(*pmd)){
183                         if(force || pmd_newpage(*pmd)){
184                                 ret = add_munmap(addr, next - addr, ops,
185                                                  op_index, last_op, mmu,
186                                                  flush, do_ops);
187                                 pmd_mkuptodate(*pmd);
188                         }
189                 }
190                 else ret = update_pte_range(pmd, addr, next, ops, last_op,
191                                             op_index, force, mmu, flush,
192                                             do_ops);
193         } while (pmd++, addr = next, ((addr != end) && !ret));
194         return ret;
195 }
196
197 static inline int update_pud_range(pgd_t *pgd, unsigned long addr,
198                                    unsigned long end, struct host_vm_op *ops,
199                                    int last_op, int *op_index, int force,
200                                    union mm_context *mmu, void **flush,
201                                    int (*do_ops)(union mm_context *,
202                                                  struct host_vm_op *, int, int,
203                                                  void **))
204 {
205         pud_t *pud;
206         unsigned long next;
207         int ret = 0;
208
209         pud = pud_offset(pgd, addr);
210         do {
211                 next = pud_addr_end(addr, end);
212                 if(!pud_present(*pud)){
213                         if(force || pud_newpage(*pud)){
214                                 ret = add_munmap(addr, next - addr, ops,
215                                                  op_index, last_op, mmu,
216                                                  flush, do_ops);
217                                 pud_mkuptodate(*pud);
218                         }
219                 }
220                 else ret = update_pmd_range(pud, addr, next, ops, last_op,
221                                             op_index, force, mmu, flush,
222                                             do_ops);
223         } while (pud++, addr = next, ((addr != end) && !ret));
224         return ret;
225 }
226
227 void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
228                       unsigned long end_addr, int force,
229                       int (*do_ops)(union mm_context *, struct host_vm_op *,
230                                     int, int, void **))
231 {
232         pgd_t *pgd;
233         union mm_context *mmu = &mm->context;
234         struct host_vm_op ops[1];
235         unsigned long addr = start_addr, next;
236         int ret = 0, last_op = ARRAY_SIZE(ops) - 1, op_index = -1;
237         void *flush = NULL;
238
239         ops[0].type = NONE;
240         pgd = pgd_offset(mm, addr);
241         do {
242                 next = pgd_addr_end(addr, end_addr);
243                 if(!pgd_present(*pgd)){
244                         if (force || pgd_newpage(*pgd)){
245                                 ret = add_munmap(addr, next - addr, ops,
246                                                  &op_index, last_op, mmu,
247                                                  &flush, do_ops);
248                                 pgd_mkuptodate(*pgd);
249                         }
250                 }
251                 else ret = update_pud_range(pgd, addr, next, ops, last_op,
252                                             &op_index, force, mmu, &flush,
253                                             do_ops);
254         } while (pgd++, addr = next, ((addr != end_addr) && !ret));
255         log_info("total flush time - %Ld nsecs\n", end_time - start_time);
256
257         if(!ret)
258                 ret = (*do_ops)(mmu, ops, op_index, 1, &flush);
259
260         /* This is not an else because ret is modified above */
261         if(ret) {
262                 printk("fix_range_common: failed, killing current process\n");
263                 force_sig(SIGKILL, current);
264         }
265 }
266
267 int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
268 {
269         struct mm_struct *mm;
270         pgd_t *pgd;
271         pud_t *pud;
272         pmd_t *pmd;
273         pte_t *pte;
274         unsigned long addr, last;
275         int updated = 0, err;
276
277         mm = &init_mm;
278         for(addr = start; addr < end;){
279                 pgd = pgd_offset(mm, addr);
280                 if(!pgd_present(*pgd)){
281                         last = ADD_ROUND(addr, PGDIR_SIZE);
282                         if(last > end)
283                                 last = end;
284                         if(pgd_newpage(*pgd)){
285                                 updated = 1;
286                                 err = os_unmap_memory((void *) addr,
287                                                       last - addr);
288                                 if(err < 0)
289                                         panic("munmap failed, errno = %d\n",
290                                               -err);
291                         }
292                         addr = last;
293                         continue;
294                 }
295
296                 pud = pud_offset(pgd, addr);
297                 if(!pud_present(*pud)){
298                         last = ADD_ROUND(addr, PUD_SIZE);
299                         if(last > end)
300                                 last = end;
301                         if(pud_newpage(*pud)){
302                                 updated = 1;
303                                 err = os_unmap_memory((void *) addr,
304                                                       last - addr);
305                                 if(err < 0)
306                                         panic("munmap failed, errno = %d\n",
307                                               -err);
308                         }
309                         addr = last;
310                         continue;
311                 }
312
313                 pmd = pmd_offset(pud, addr);
314                 if(!pmd_present(*pmd)){
315                         last = ADD_ROUND(addr, PMD_SIZE);
316                         if(last > end)
317                                 last = end;
318                         if(pmd_newpage(*pmd)){
319                                 updated = 1;
320                                 err = os_unmap_memory((void *) addr,
321                                                       last - addr);
322                                 if(err < 0)
323                                         panic("munmap failed, errno = %d\n",
324                                               -err);
325                         }
326                         addr = last;
327                         continue;
328                 }
329
330                 pte = pte_offset_kernel(pmd, addr);
331                 if(!pte_present(*pte) || pte_newpage(*pte)){
332                         updated = 1;
333                         err = os_unmap_memory((void *) addr,
334                                               PAGE_SIZE);
335                         if(err < 0)
336                                 panic("munmap failed, errno = %d\n",
337                                       -err);
338                         if(pte_present(*pte))
339                                 map_memory(addr,
340                                            pte_val(*pte) & PAGE_MASK,
341                                            PAGE_SIZE, 1, 1, 1);
342                 }
343                 else if(pte_newprot(*pte)){
344                         updated = 1;
345                         os_protect_memory((void *) addr, PAGE_SIZE, 1, 1, 1);
346                 }
347                 addr += PAGE_SIZE;
348         }
349         return(updated);
350 }
351
352 pgd_t *pgd_offset_proc(struct mm_struct *mm, unsigned long address)
353 {
354         return(pgd_offset(mm, address));
355 }
356
357 pud_t *pud_offset_proc(pgd_t *pgd, unsigned long address)
358 {
359         return(pud_offset(pgd, address));
360 }
361
362 pmd_t *pmd_offset_proc(pud_t *pud, unsigned long address)
363 {
364         return(pmd_offset(pud, address));
365 }
366
367 pte_t *pte_offset_proc(pmd_t *pmd, unsigned long address)
368 {
369         return(pte_offset_kernel(pmd, address));
370 }
371
372 pte_t *addr_pte(struct task_struct *task, unsigned long addr)
373 {
374         pgd_t *pgd = pgd_offset(task->mm, addr);
375         pud_t *pud = pud_offset(pgd, addr);
376         pmd_t *pmd = pmd_offset(pud, addr);
377
378         return(pte_offset_map(pmd, addr));
379 }
380
381 void flush_tlb_page(struct vm_area_struct *vma, unsigned long address)
382 {
383         address &= PAGE_MASK;
384         flush_tlb_range(vma, address, address + PAGE_SIZE);
385 }
386
387 void flush_tlb_all(void)
388 {
389         flush_tlb_mm(current->mm);
390 }
391
392 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
393 {
394         CHOOSE_MODE_PROC(flush_tlb_kernel_range_tt,
395                          flush_tlb_kernel_range_common, start, end);
396 }
397
398 void flush_tlb_kernel_vm(void)
399 {
400         CHOOSE_MODE(flush_tlb_kernel_vm_tt(),
401                     flush_tlb_kernel_range_common(start_vm, end_vm));
402 }
403
404 void __flush_tlb_one(unsigned long addr)
405 {
406         CHOOSE_MODE_PROC(__flush_tlb_one_tt, __flush_tlb_one_skas, addr);
407 }
408
409 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
410                      unsigned long end)
411 {
412         CHOOSE_MODE_PROC(flush_tlb_range_tt, flush_tlb_range_skas, vma, start,
413                          end);
414 }
415
416 void flush_tlb_mm(struct mm_struct *mm)
417 {
418         CHOOSE_MODE_PROC(flush_tlb_mm_tt, flush_tlb_mm_skas, mm);
419 }
420
421 void force_flush_all(void)
422 {
423         CHOOSE_MODE(force_flush_all_tt(), force_flush_all_skas());
424 }
425