Linux-2.6.12-rc2
[linux-3.10.git] / arch / ia64 / lib / copy_page.S
1 /*
2  *
3  * Optimized version of the standard copy_page() function
4  *
5  * Inputs:
6  *      in0:    address of target page
7  *      in1:    address of source page
8  * Output:
9  *      no return value
10  *
11  * Copyright (C) 1999, 2001 Hewlett-Packard Co
12  *      Stephane Eranian <eranian@hpl.hp.com>
13  *      David Mosberger <davidm@hpl.hp.com>
14  *
15  * 4/06/01 davidm       Tuned to make it perform well both for cached and uncached copies.
16  */
17 #include <asm/asmmacro.h>
18 #include <asm/page.h>
19
20 #define PIPE_DEPTH      3
21 #define EPI             p[PIPE_DEPTH-1]
22
23 #define lcount          r16
24 #define saved_pr        r17
25 #define saved_lc        r18
26 #define saved_pfs       r19
27 #define src1            r20
28 #define src2            r21
29 #define tgt1            r22
30 #define tgt2            r23
31 #define srcf            r24
32 #define tgtf            r25
33 #define tgt_last        r26
34
35 #define Nrot            ((8*PIPE_DEPTH+7)&~7)
36
37 GLOBAL_ENTRY(copy_page)
38         .prologue
39         .save ar.pfs, saved_pfs
40         alloc saved_pfs=ar.pfs,3,Nrot-3,0,Nrot
41
42         .rotr t1[PIPE_DEPTH], t2[PIPE_DEPTH], t3[PIPE_DEPTH], t4[PIPE_DEPTH], \
43               t5[PIPE_DEPTH], t6[PIPE_DEPTH], t7[PIPE_DEPTH], t8[PIPE_DEPTH]
44         .rotp p[PIPE_DEPTH]
45
46         .save ar.lc, saved_lc
47         mov saved_lc=ar.lc
48         mov ar.ec=PIPE_DEPTH
49
50         mov lcount=PAGE_SIZE/64-1
51         .save pr, saved_pr
52         mov saved_pr=pr
53         mov pr.rot=1<<16
54
55         .body
56
57         mov src1=in1
58         adds src2=8,in1
59         mov tgt_last = PAGE_SIZE
60         ;;
61         adds tgt2=8,in0
62         add srcf=512,in1
63         mov ar.lc=lcount
64         mov tgt1=in0
65         add tgtf=512,in0
66         add tgt_last = tgt_last, in0
67         ;;
68 1:
69 (p[0])  ld8 t1[0]=[src1],16
70 (EPI)   st8 [tgt1]=t1[PIPE_DEPTH-1],16
71 (p[0])  ld8 t2[0]=[src2],16
72 (EPI)   st8 [tgt2]=t2[PIPE_DEPTH-1],16
73         cmp.ltu p6,p0 = tgtf, tgt_last
74         ;;
75 (p[0])  ld8 t3[0]=[src1],16
76 (EPI)   st8 [tgt1]=t3[PIPE_DEPTH-1],16
77 (p[0])  ld8 t4[0]=[src2],16
78 (EPI)   st8 [tgt2]=t4[PIPE_DEPTH-1],16
79         ;;
80 (p[0])  ld8 t5[0]=[src1],16
81 (EPI)   st8 [tgt1]=t5[PIPE_DEPTH-1],16
82 (p[0])  ld8 t6[0]=[src2],16
83 (EPI)   st8 [tgt2]=t6[PIPE_DEPTH-1],16
84         ;;
85 (p[0])  ld8 t7[0]=[src1],16
86 (EPI)   st8 [tgt1]=t7[PIPE_DEPTH-1],16
87 (p[0])  ld8 t8[0]=[src2],16
88 (EPI)   st8 [tgt2]=t8[PIPE_DEPTH-1],16
89
90 (p6)    lfetch [srcf], 64
91 (p6)    lfetch [tgtf], 64
92         br.ctop.sptk.few 1b
93         ;;
94         mov pr=saved_pr,0xffffffffffff0000      // restore predicates
95         mov ar.pfs=saved_pfs
96         mov ar.lc=saved_lc
97         br.ret.sptk.many rp
98 END(copy_page)