Merge branch 'drm-next' of git://people.freedesktop.org/~airlied/linux
[linux-2.6.git] / drivers / gpu / drm / gma500 / mmu.c
1 /**************************************************************************
2  * Copyright (c) 2007, Intel Corporation.
3  *
4  * This program is free software; you can redistribute it and/or modify it
5  * under the terms and conditions of the GNU General Public License,
6  * version 2, as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope it will be useful, but WITHOUT
9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
11  * more details.
12  *
13  * You should have received a copy of the GNU General Public License along with
14  * this program; if not, write to the Free Software Foundation, Inc.,
15  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
16  *
17  **************************************************************************/
18 #include <drm/drmP.h>
19 #include "psb_drv.h"
20 #include "psb_reg.h"
21
22 /*
23  * Code for the SGX MMU:
24  */
25
26 /*
27  * clflush on one processor only:
28  * clflush should apparently flush the cache line on all processors in an
29  * SMP system.
30  */
31
32 /*
33  * kmap atomic:
34  * The usage of the slots must be completely encapsulated within a spinlock, and
35  * no other functions that may be using the locks for other purposed may be
36  * called from within the locked region.
37  * Since the slots are per processor, this will guarantee that we are the only
38  * user.
39  */
40
41 /*
42  * TODO: Inserting ptes from an interrupt handler:
43  * This may be desirable for some SGX functionality where the GPU can fault in
44  * needed pages. For that, we need to make an atomic insert_pages function, that
45  * may fail.
46  * If it fails, the caller need to insert the page using a workqueue function,
47  * but on average it should be fast.
48  */
49
50 struct psb_mmu_driver {
51         /* protects driver- and pd structures. Always take in read mode
52          * before taking the page table spinlock.
53          */
54         struct rw_semaphore sem;
55
56         /* protects page tables, directory tables and pt tables.
57          * and pt structures.
58          */
59         spinlock_t lock;
60
61         atomic_t needs_tlbflush;
62
63         uint8_t __iomem *register_map;
64         struct psb_mmu_pd *default_pd;
65         /*uint32_t bif_ctrl;*/
66         int has_clflush;
67         int clflush_add;
68         unsigned long clflush_mask;
69
70         struct drm_psb_private *dev_priv;
71 };
72
73 struct psb_mmu_pd;
74
75 struct psb_mmu_pt {
76         struct psb_mmu_pd *pd;
77         uint32_t index;
78         uint32_t count;
79         struct page *p;
80         uint32_t *v;
81 };
82
83 struct psb_mmu_pd {
84         struct psb_mmu_driver *driver;
85         int hw_context;
86         struct psb_mmu_pt **tables;
87         struct page *p;
88         struct page *dummy_pt;
89         struct page *dummy_page;
90         uint32_t pd_mask;
91         uint32_t invalid_pde;
92         uint32_t invalid_pte;
93 };
94
95 static inline uint32_t psb_mmu_pt_index(uint32_t offset)
96 {
97         return (offset >> PSB_PTE_SHIFT) & 0x3FF;
98 }
99
100 static inline uint32_t psb_mmu_pd_index(uint32_t offset)
101 {
102         return offset >> PSB_PDE_SHIFT;
103 }
104
105 static inline void psb_clflush(void *addr)
106 {
107         __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory");
108 }
109
110 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver,
111                                    void *addr)
112 {
113         if (!driver->has_clflush)
114                 return;
115
116         mb();
117         psb_clflush(addr);
118         mb();
119 }
120
121 static void psb_page_clflush(struct psb_mmu_driver *driver, struct page* page)
122 {
123         uint32_t clflush_add = driver->clflush_add >> PAGE_SHIFT;
124         uint32_t clflush_count = PAGE_SIZE / clflush_add;
125         int i;
126         uint8_t *clf;
127
128         clf = kmap_atomic(page);
129         mb();
130         for (i = 0; i < clflush_count; ++i) {
131                 psb_clflush(clf);
132                 clf += clflush_add;
133         }
134         mb();
135         kunmap_atomic(clf);
136 }
137
138 static void psb_pages_clflush(struct psb_mmu_driver *driver,
139                                 struct page *page[], unsigned long num_pages)
140 {
141         int i;
142
143         if (!driver->has_clflush)
144                 return ;
145
146         for (i = 0; i < num_pages; i++)
147                 psb_page_clflush(driver, *page++);
148 }
149
150 static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver,
151                                     int force)
152 {
153         atomic_set(&driver->needs_tlbflush, 0);
154 }
155
156 static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force)
157 {
158         down_write(&driver->sem);
159         psb_mmu_flush_pd_locked(driver, force);
160         up_write(&driver->sem);
161 }
162
163 void psb_mmu_flush(struct psb_mmu_driver *driver, int rc_prot)
164 {
165         if (rc_prot)
166                 down_write(&driver->sem);
167         if (rc_prot)
168                 up_write(&driver->sem);
169 }
170
171 void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context)
172 {
173         /*ttm_tt_cache_flush(&pd->p, 1);*/
174         psb_pages_clflush(pd->driver, &pd->p, 1);
175         down_write(&pd->driver->sem);
176         wmb();
177         psb_mmu_flush_pd_locked(pd->driver, 1);
178         pd->hw_context = hw_context;
179         up_write(&pd->driver->sem);
180
181 }
182
183 static inline unsigned long psb_pd_addr_end(unsigned long addr,
184                                             unsigned long end)
185 {
186
187         addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK;
188         return (addr < end) ? addr : end;
189 }
190
191 static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type)
192 {
193         uint32_t mask = PSB_PTE_VALID;
194
195         if (type & PSB_MMU_CACHED_MEMORY)
196                 mask |= PSB_PTE_CACHED;
197         if (type & PSB_MMU_RO_MEMORY)
198                 mask |= PSB_PTE_RO;
199         if (type & PSB_MMU_WO_MEMORY)
200                 mask |= PSB_PTE_WO;
201
202         return (pfn << PAGE_SHIFT) | mask;
203 }
204
205 struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver,
206                                     int trap_pagefaults, int invalid_type)
207 {
208         struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL);
209         uint32_t *v;
210         int i;
211
212         if (!pd)
213                 return NULL;
214
215         pd->p = alloc_page(GFP_DMA32);
216         if (!pd->p)
217                 goto out_err1;
218         pd->dummy_pt = alloc_page(GFP_DMA32);
219         if (!pd->dummy_pt)
220                 goto out_err2;
221         pd->dummy_page = alloc_page(GFP_DMA32);
222         if (!pd->dummy_page)
223                 goto out_err3;
224
225         if (!trap_pagefaults) {
226                 pd->invalid_pde =
227                     psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt),
228                                      invalid_type);
229                 pd->invalid_pte =
230                     psb_mmu_mask_pte(page_to_pfn(pd->dummy_page),
231                                      invalid_type);
232         } else {
233                 pd->invalid_pde = 0;
234                 pd->invalid_pte = 0;
235         }
236
237         v = kmap(pd->dummy_pt);
238         for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
239                 v[i] = pd->invalid_pte;
240
241         kunmap(pd->dummy_pt);
242
243         v = kmap(pd->p);
244         for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
245                 v[i] = pd->invalid_pde;
246
247         kunmap(pd->p);
248
249         clear_page(kmap(pd->dummy_page));
250         kunmap(pd->dummy_page);
251
252         pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024);
253         if (!pd->tables)
254                 goto out_err4;
255
256         pd->hw_context = -1;
257         pd->pd_mask = PSB_PTE_VALID;
258         pd->driver = driver;
259
260         return pd;
261
262 out_err4:
263         __free_page(pd->dummy_page);
264 out_err3:
265         __free_page(pd->dummy_pt);
266 out_err2:
267         __free_page(pd->p);
268 out_err1:
269         kfree(pd);
270         return NULL;
271 }
272
273 static void psb_mmu_free_pt(struct psb_mmu_pt *pt)
274 {
275         __free_page(pt->p);
276         kfree(pt);
277 }
278
279 void psb_mmu_free_pagedir(struct psb_mmu_pd *pd)
280 {
281         struct psb_mmu_driver *driver = pd->driver;
282         struct psb_mmu_pt *pt;
283         int i;
284
285         down_write(&driver->sem);
286         if (pd->hw_context != -1)
287                 psb_mmu_flush_pd_locked(driver, 1);
288
289         /* Should take the spinlock here, but we don't need to do that
290            since we have the semaphore in write mode. */
291
292         for (i = 0; i < 1024; ++i) {
293                 pt = pd->tables[i];
294                 if (pt)
295                         psb_mmu_free_pt(pt);
296         }
297
298         vfree(pd->tables);
299         __free_page(pd->dummy_page);
300         __free_page(pd->dummy_pt);
301         __free_page(pd->p);
302         kfree(pd);
303         up_write(&driver->sem);
304 }
305
306 static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd)
307 {
308         struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL);
309         void *v;
310         uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT;
311         uint32_t clflush_count = PAGE_SIZE / clflush_add;
312         spinlock_t *lock = &pd->driver->lock;
313         uint8_t *clf;
314         uint32_t *ptes;
315         int i;
316
317         if (!pt)
318                 return NULL;
319
320         pt->p = alloc_page(GFP_DMA32);
321         if (!pt->p) {
322                 kfree(pt);
323                 return NULL;
324         }
325
326         spin_lock(lock);
327
328         v = kmap_atomic(pt->p);
329         clf = (uint8_t *) v;
330         ptes = (uint32_t *) v;
331         for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
332                 *ptes++ = pd->invalid_pte;
333
334
335         if (pd->driver->has_clflush && pd->hw_context != -1) {
336                 mb();
337                 for (i = 0; i < clflush_count; ++i) {
338                         psb_clflush(clf);
339                         clf += clflush_add;
340                 }
341                 mb();
342         }
343
344         kunmap_atomic(v);
345         spin_unlock(lock);
346
347         pt->count = 0;
348         pt->pd = pd;
349         pt->index = 0;
350
351         return pt;
352 }
353
354 static struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd,
355                                              unsigned long addr)
356 {
357         uint32_t index = psb_mmu_pd_index(addr);
358         struct psb_mmu_pt *pt;
359         uint32_t *v;
360         spinlock_t *lock = &pd->driver->lock;
361
362         spin_lock(lock);
363         pt = pd->tables[index];
364         while (!pt) {
365                 spin_unlock(lock);
366                 pt = psb_mmu_alloc_pt(pd);
367                 if (!pt)
368                         return NULL;
369                 spin_lock(lock);
370
371                 if (pd->tables[index]) {
372                         spin_unlock(lock);
373                         psb_mmu_free_pt(pt);
374                         spin_lock(lock);
375                         pt = pd->tables[index];
376                         continue;
377                 }
378
379                 v = kmap_atomic(pd->p);
380                 pd->tables[index] = pt;
381                 v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask;
382                 pt->index = index;
383                 kunmap_atomic((void *) v);
384
385                 if (pd->hw_context != -1) {
386                         psb_mmu_clflush(pd->driver, (void *) &v[index]);
387                         atomic_set(&pd->driver->needs_tlbflush, 1);
388                 }
389         }
390         pt->v = kmap_atomic(pt->p);
391         return pt;
392 }
393
394 static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd,
395                                               unsigned long addr)
396 {
397         uint32_t index = psb_mmu_pd_index(addr);
398         struct psb_mmu_pt *pt;
399         spinlock_t *lock = &pd->driver->lock;
400
401         spin_lock(lock);
402         pt = pd->tables[index];
403         if (!pt) {
404                 spin_unlock(lock);
405                 return NULL;
406         }
407         pt->v = kmap_atomic(pt->p);
408         return pt;
409 }
410
411 static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt)
412 {
413         struct psb_mmu_pd *pd = pt->pd;
414         uint32_t *v;
415
416         kunmap_atomic(pt->v);
417         if (pt->count == 0) {
418                 v = kmap_atomic(pd->p);
419                 v[pt->index] = pd->invalid_pde;
420                 pd->tables[pt->index] = NULL;
421
422                 if (pd->hw_context != -1) {
423                         psb_mmu_clflush(pd->driver,
424                                         (void *) &v[pt->index]);
425                         atomic_set(&pd->driver->needs_tlbflush, 1);
426                 }
427                 kunmap_atomic(pt->v);
428                 spin_unlock(&pd->driver->lock);
429                 psb_mmu_free_pt(pt);
430                 return;
431         }
432         spin_unlock(&pd->driver->lock);
433 }
434
435 static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt,
436                                    unsigned long addr, uint32_t pte)
437 {
438         pt->v[psb_mmu_pt_index(addr)] = pte;
439 }
440
441 static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt,
442                                           unsigned long addr)
443 {
444         pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte;
445 }
446
447
448 void psb_mmu_mirror_gtt(struct psb_mmu_pd *pd,
449                         uint32_t mmu_offset, uint32_t gtt_start,
450                         uint32_t gtt_pages)
451 {
452         uint32_t *v;
453         uint32_t start = psb_mmu_pd_index(mmu_offset);
454         struct psb_mmu_driver *driver = pd->driver;
455         int num_pages = gtt_pages;
456
457         down_read(&driver->sem);
458         spin_lock(&driver->lock);
459
460         v = kmap_atomic(pd->p);
461         v += start;
462
463         while (gtt_pages--) {
464                 *v++ = gtt_start | pd->pd_mask;
465                 gtt_start += PAGE_SIZE;
466         }
467
468         /*ttm_tt_cache_flush(&pd->p, num_pages);*/
469         psb_pages_clflush(pd->driver, &pd->p, num_pages);
470         kunmap_atomic(v);
471         spin_unlock(&driver->lock);
472
473         if (pd->hw_context != -1)
474                 atomic_set(&pd->driver->needs_tlbflush, 1);
475
476         up_read(&pd->driver->sem);
477         psb_mmu_flush_pd(pd->driver, 0);
478 }
479
480 struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)
481 {
482         struct psb_mmu_pd *pd;
483
484         /* down_read(&driver->sem); */
485         pd = driver->default_pd;
486         /* up_read(&driver->sem); */
487
488         return pd;
489 }
490
491 void psb_mmu_driver_takedown(struct psb_mmu_driver *driver)
492 {
493         psb_mmu_free_pagedir(driver->default_pd);
494         kfree(driver);
495 }
496
497 struct psb_mmu_driver *psb_mmu_driver_init(uint8_t __iomem * registers,
498                                         int trap_pagefaults,
499                                         int invalid_type,
500                                         struct drm_psb_private *dev_priv)
501 {
502         struct psb_mmu_driver *driver;
503
504         driver = kmalloc(sizeof(*driver), GFP_KERNEL);
505
506         if (!driver)
507                 return NULL;
508         driver->dev_priv = dev_priv;
509
510         driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults,
511                                               invalid_type);
512         if (!driver->default_pd)
513                 goto out_err1;
514
515         spin_lock_init(&driver->lock);
516         init_rwsem(&driver->sem);
517         down_write(&driver->sem);
518         driver->register_map = registers;
519         atomic_set(&driver->needs_tlbflush, 1);
520
521         driver->has_clflush = 0;
522
523         if (boot_cpu_has(X86_FEATURE_CLFLSH)) {
524                 uint32_t tfms, misc, cap0, cap4, clflush_size;
525
526                 /*
527                  * clflush size is determined at kernel setup for x86_64
528                  *  but not for i386. We have to do it here.
529                  */
530
531                 cpuid(0x00000001, &tfms, &misc, &cap0, &cap4);
532                 clflush_size = ((misc >> 8) & 0xff) * 8;
533                 driver->has_clflush = 1;
534                 driver->clflush_add =
535                     PAGE_SIZE * clflush_size / sizeof(uint32_t);
536                 driver->clflush_mask = driver->clflush_add - 1;
537                 driver->clflush_mask = ~driver->clflush_mask;
538         }
539
540         up_write(&driver->sem);
541         return driver;
542
543 out_err1:
544         kfree(driver);
545         return NULL;
546 }
547
548 static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd,
549                                unsigned long address, uint32_t num_pages,
550                                uint32_t desired_tile_stride,
551                                uint32_t hw_tile_stride)
552 {
553         struct psb_mmu_pt *pt;
554         uint32_t rows = 1;
555         uint32_t i;
556         unsigned long addr;
557         unsigned long end;
558         unsigned long next;
559         unsigned long add;
560         unsigned long row_add;
561         unsigned long clflush_add = pd->driver->clflush_add;
562         unsigned long clflush_mask = pd->driver->clflush_mask;
563
564         if (!pd->driver->has_clflush) {
565                 /*ttm_tt_cache_flush(&pd->p, num_pages);*/
566                 psb_pages_clflush(pd->driver, &pd->p, num_pages);
567                 return;
568         }
569
570         if (hw_tile_stride)
571                 rows = num_pages / desired_tile_stride;
572         else
573                 desired_tile_stride = num_pages;
574
575         add = desired_tile_stride << PAGE_SHIFT;
576         row_add = hw_tile_stride << PAGE_SHIFT;
577         mb();
578         for (i = 0; i < rows; ++i) {
579
580                 addr = address;
581                 end = addr + add;
582
583                 do {
584                         next = psb_pd_addr_end(addr, end);
585                         pt = psb_mmu_pt_map_lock(pd, addr);
586                         if (!pt)
587                                 continue;
588                         do {
589                                 psb_clflush(&pt->v
590                                             [psb_mmu_pt_index(addr)]);
591                         } while (addr +=
592                                  clflush_add,
593                                  (addr & clflush_mask) < next);
594
595                         psb_mmu_pt_unmap_unlock(pt);
596                 } while (addr = next, next != end);
597                 address += row_add;
598         }
599         mb();
600 }
601
602 void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd,
603                                  unsigned long address, uint32_t num_pages)
604 {
605         struct psb_mmu_pt *pt;
606         unsigned long addr;
607         unsigned long end;
608         unsigned long next;
609         unsigned long f_address = address;
610
611         down_read(&pd->driver->sem);
612
613         addr = address;
614         end = addr + (num_pages << PAGE_SHIFT);
615
616         do {
617                 next = psb_pd_addr_end(addr, end);
618                 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
619                 if (!pt)
620                         goto out;
621                 do {
622                         psb_mmu_invalidate_pte(pt, addr);
623                         --pt->count;
624                 } while (addr += PAGE_SIZE, addr < next);
625                 psb_mmu_pt_unmap_unlock(pt);
626
627         } while (addr = next, next != end);
628
629 out:
630         if (pd->hw_context != -1)
631                 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
632
633         up_read(&pd->driver->sem);
634
635         if (pd->hw_context != -1)
636                 psb_mmu_flush(pd->driver, 0);
637
638         return;
639 }
640
641 void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address,
642                           uint32_t num_pages, uint32_t desired_tile_stride,
643                           uint32_t hw_tile_stride)
644 {
645         struct psb_mmu_pt *pt;
646         uint32_t rows = 1;
647         uint32_t i;
648         unsigned long addr;
649         unsigned long end;
650         unsigned long next;
651         unsigned long add;
652         unsigned long row_add;
653         unsigned long f_address = address;
654
655         if (hw_tile_stride)
656                 rows = num_pages / desired_tile_stride;
657         else
658                 desired_tile_stride = num_pages;
659
660         add = desired_tile_stride << PAGE_SHIFT;
661         row_add = hw_tile_stride << PAGE_SHIFT;
662
663         /* down_read(&pd->driver->sem); */
664
665         /* Make sure we only need to flush this processor's cache */
666
667         for (i = 0; i < rows; ++i) {
668
669                 addr = address;
670                 end = addr + add;
671
672                 do {
673                         next = psb_pd_addr_end(addr, end);
674                         pt = psb_mmu_pt_map_lock(pd, addr);
675                         if (!pt)
676                                 continue;
677                         do {
678                                 psb_mmu_invalidate_pte(pt, addr);
679                                 --pt->count;
680
681                         } while (addr += PAGE_SIZE, addr < next);
682                         psb_mmu_pt_unmap_unlock(pt);
683
684                 } while (addr = next, next != end);
685                 address += row_add;
686         }
687         if (pd->hw_context != -1)
688                 psb_mmu_flush_ptes(pd, f_address, num_pages,
689                                    desired_tile_stride, hw_tile_stride);
690
691         /* up_read(&pd->driver->sem); */
692
693         if (pd->hw_context != -1)
694                 psb_mmu_flush(pd->driver, 0);
695 }
696
697 int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn,
698                                 unsigned long address, uint32_t num_pages,
699                                 int type)
700 {
701         struct psb_mmu_pt *pt;
702         uint32_t pte;
703         unsigned long addr;
704         unsigned long end;
705         unsigned long next;
706         unsigned long f_address = address;
707         int ret = 0;
708
709         down_read(&pd->driver->sem);
710
711         addr = address;
712         end = addr + (num_pages << PAGE_SHIFT);
713
714         do {
715                 next = psb_pd_addr_end(addr, end);
716                 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
717                 if (!pt) {
718                         ret = -ENOMEM;
719                         goto out;
720                 }
721                 do {
722                         pte = psb_mmu_mask_pte(start_pfn++, type);
723                         psb_mmu_set_pte(pt, addr, pte);
724                         pt->count++;
725                 } while (addr += PAGE_SIZE, addr < next);
726                 psb_mmu_pt_unmap_unlock(pt);
727
728         } while (addr = next, next != end);
729
730 out:
731         if (pd->hw_context != -1)
732                 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
733
734         up_read(&pd->driver->sem);
735
736         if (pd->hw_context != -1)
737                 psb_mmu_flush(pd->driver, 1);
738
739         return ret;
740 }
741
742 int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,
743                          unsigned long address, uint32_t num_pages,
744                          uint32_t desired_tile_stride,
745                          uint32_t hw_tile_stride, int type)
746 {
747         struct psb_mmu_pt *pt;
748         uint32_t rows = 1;
749         uint32_t i;
750         uint32_t pte;
751         unsigned long addr;
752         unsigned long end;
753         unsigned long next;
754         unsigned long add;
755         unsigned long row_add;
756         unsigned long f_address = address;
757         int ret = 0;
758
759         if (hw_tile_stride) {
760                 if (num_pages % desired_tile_stride != 0)
761                         return -EINVAL;
762                 rows = num_pages / desired_tile_stride;
763         } else {
764                 desired_tile_stride = num_pages;
765         }
766
767         add = desired_tile_stride << PAGE_SHIFT;
768         row_add = hw_tile_stride << PAGE_SHIFT;
769
770         down_read(&pd->driver->sem);
771
772         for (i = 0; i < rows; ++i) {
773
774                 addr = address;
775                 end = addr + add;
776
777                 do {
778                         next = psb_pd_addr_end(addr, end);
779                         pt = psb_mmu_pt_alloc_map_lock(pd, addr);
780                         if (!pt) {
781                                 ret = -ENOMEM;
782                                 goto out;
783                         }
784                         do {
785                                 pte =
786                                     psb_mmu_mask_pte(page_to_pfn(*pages++),
787                                                      type);
788                                 psb_mmu_set_pte(pt, addr, pte);
789                                 pt->count++;
790                         } while (addr += PAGE_SIZE, addr < next);
791                         psb_mmu_pt_unmap_unlock(pt);
792
793                 } while (addr = next, next != end);
794
795                 address += row_add;
796         }
797 out:
798         if (pd->hw_context != -1)
799                 psb_mmu_flush_ptes(pd, f_address, num_pages,
800                                    desired_tile_stride, hw_tile_stride);
801
802         up_read(&pd->driver->sem);
803
804         if (pd->hw_context != -1)
805                 psb_mmu_flush(pd->driver, 1);
806
807         return ret;
808 }
809
810 int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual,
811                            unsigned long *pfn)
812 {
813         int ret;
814         struct psb_mmu_pt *pt;
815         uint32_t tmp;
816         spinlock_t *lock = &pd->driver->lock;
817
818         down_read(&pd->driver->sem);
819         pt = psb_mmu_pt_map_lock(pd, virtual);
820         if (!pt) {
821                 uint32_t *v;
822
823                 spin_lock(lock);
824                 v = kmap_atomic(pd->p);
825                 tmp = v[psb_mmu_pd_index(virtual)];
826                 kunmap_atomic(v);
827                 spin_unlock(lock);
828
829                 if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) ||
830                     !(pd->invalid_pte & PSB_PTE_VALID)) {
831                         ret = -EINVAL;
832                         goto out;
833                 }
834                 ret = 0;
835                 *pfn = pd->invalid_pte >> PAGE_SHIFT;
836                 goto out;
837         }
838         tmp = pt->v[psb_mmu_pt_index(virtual)];
839         if (!(tmp & PSB_PTE_VALID)) {
840                 ret = -EINVAL;
841         } else {
842                 ret = 0;
843                 *pfn = tmp >> PAGE_SHIFT;
844         }
845         psb_mmu_pt_unmap_unlock(pt);
846 out:
847         up_read(&pd->driver->sem);
848         return ret;
849 }