ARM: 6384/1: Remove the domain switching on ARMv6k/v7 CPUs
[linux-3.10.git] / arch / arm / lib / uaccess.S
1 /*
2  *  linux/arch/arm/lib/uaccess.S
3  *
4  *  Copyright (C) 1995, 1996,1997,1998 Russell King
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 as
8  * published by the Free Software Foundation.
9  *
10  *  Routines to block copy data to/from user memory
11  *   These are highly optimised both for the 4k page size
12  *   and for various alignments.
13  */
14 #include <linux/linkage.h>
15 #include <asm/assembler.h>
16 #include <asm/errno.h>
17 #include <asm/domain.h>
18
19                 .text
20
21 #define PAGE_SHIFT 12
22
23 /* Prototype: int __copy_to_user(void *to, const char *from, size_t n)
24  * Purpose  : copy a block to user memory from kernel memory
25  * Params   : to   - user memory
26  *          : from - kernel memory
27  *          : n    - number of bytes to copy
28  * Returns  : Number of bytes NOT copied.
29  */
30
31 .Lc2u_dest_not_aligned:
32                 rsb     ip, ip, #4
33                 cmp     ip, #2
34                 ldrb    r3, [r1], #1
35 USER(           T(strb) r3, [r0], #1)                   @ May fault
36                 ldrgeb  r3, [r1], #1
37 USER(           T(strgeb) r3, [r0], #1)                 @ May fault
38                 ldrgtb  r3, [r1], #1
39 USER(           T(strgtb) r3, [r0], #1)                 @ May fault
40                 sub     r2, r2, ip
41                 b       .Lc2u_dest_aligned
42
43 ENTRY(__copy_to_user)
44                 stmfd   sp!, {r2, r4 - r7, lr}
45                 cmp     r2, #4
46                 blt     .Lc2u_not_enough
47                 ands    ip, r0, #3
48                 bne     .Lc2u_dest_not_aligned
49 .Lc2u_dest_aligned:
50
51                 ands    ip, r1, #3
52                 bne     .Lc2u_src_not_aligned
53 /*
54  * Seeing as there has to be at least 8 bytes to copy, we can
55  * copy one word, and force a user-mode page fault...
56  */
57
58 .Lc2u_0fupi:    subs    r2, r2, #4
59                 addmi   ip, r2, #4
60                 bmi     .Lc2u_0nowords
61                 ldr     r3, [r1], #4
62 USER(           T(str)  r3, [r0], #4)                   @ May fault
63                 mov     ip, r0, lsl #32 - PAGE_SHIFT    @ On each page, use a ld/st??t instruction
64                 rsb     ip, ip, #0
65                 movs    ip, ip, lsr #32 - PAGE_SHIFT
66                 beq     .Lc2u_0fupi
67 /*
68  * ip = max no. of bytes to copy before needing another "strt" insn
69  */
70                 cmp     r2, ip
71                 movlt   ip, r2
72                 sub     r2, r2, ip
73                 subs    ip, ip, #32
74                 blt     .Lc2u_0rem8lp
75
76 .Lc2u_0cpy8lp:  ldmia   r1!, {r3 - r6}
77                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
78                 ldmia   r1!, {r3 - r6}
79                 subs    ip, ip, #32
80                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
81                 bpl     .Lc2u_0cpy8lp
82
83 .Lc2u_0rem8lp:  cmn     ip, #16
84                 ldmgeia r1!, {r3 - r6}
85                 stmgeia r0!, {r3 - r6}                  @ Shouldnt fault
86                 tst     ip, #8
87                 ldmneia r1!, {r3 - r4}
88                 stmneia r0!, {r3 - r4}                  @ Shouldnt fault
89                 tst     ip, #4
90                 ldrne   r3, [r1], #4
91                 T(strne) r3, [r0], #4                   @ Shouldnt fault
92                 ands    ip, ip, #3
93                 beq     .Lc2u_0fupi
94 .Lc2u_0nowords: teq     ip, #0
95                 beq     .Lc2u_finished
96 .Lc2u_nowords:  cmp     ip, #2
97                 ldrb    r3, [r1], #1
98 USER(           T(strb) r3, [r0], #1)                   @ May fault
99                 ldrgeb  r3, [r1], #1
100 USER(           T(strgeb) r3, [r0], #1)                 @ May fault
101                 ldrgtb  r3, [r1], #1
102 USER(           T(strgtb) r3, [r0], #1)                 @ May fault
103                 b       .Lc2u_finished
104
105 .Lc2u_not_enough:
106                 movs    ip, r2
107                 bne     .Lc2u_nowords
108 .Lc2u_finished: mov     r0, #0
109                 ldmfd   sp!, {r2, r4 - r7, pc}
110
111 .Lc2u_src_not_aligned:
112                 bic     r1, r1, #3
113                 ldr     r7, [r1], #4
114                 cmp     ip, #2
115                 bgt     .Lc2u_3fupi
116                 beq     .Lc2u_2fupi
117 .Lc2u_1fupi:    subs    r2, r2, #4
118                 addmi   ip, r2, #4
119                 bmi     .Lc2u_1nowords
120                 mov     r3, r7, pull #8
121                 ldr     r7, [r1], #4
122                 orr     r3, r3, r7, push #24
123 USER(           T(str)  r3, [r0], #4)                   @ May fault
124                 mov     ip, r0, lsl #32 - PAGE_SHIFT
125                 rsb     ip, ip, #0
126                 movs    ip, ip, lsr #32 - PAGE_SHIFT
127                 beq     .Lc2u_1fupi
128                 cmp     r2, ip
129                 movlt   ip, r2
130                 sub     r2, r2, ip
131                 subs    ip, ip, #16
132                 blt     .Lc2u_1rem8lp
133
134 .Lc2u_1cpy8lp:  mov     r3, r7, pull #8
135                 ldmia   r1!, {r4 - r7}
136                 subs    ip, ip, #16
137                 orr     r3, r3, r4, push #24
138                 mov     r4, r4, pull #8
139                 orr     r4, r4, r5, push #24
140                 mov     r5, r5, pull #8
141                 orr     r5, r5, r6, push #24
142                 mov     r6, r6, pull #8
143                 orr     r6, r6, r7, push #24
144                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
145                 bpl     .Lc2u_1cpy8lp
146
147 .Lc2u_1rem8lp:  tst     ip, #8
148                 movne   r3, r7, pull #8
149                 ldmneia r1!, {r4, r7}
150                 orrne   r3, r3, r4, push #24
151                 movne   r4, r4, pull #8
152                 orrne   r4, r4, r7, push #24
153                 stmneia r0!, {r3 - r4}                  @ Shouldnt fault
154                 tst     ip, #4
155                 movne   r3, r7, pull #8
156                 ldrne   r7, [r1], #4
157                 orrne   r3, r3, r7, push #24
158                 T(strne) r3, [r0], #4                   @ Shouldnt fault
159                 ands    ip, ip, #3
160                 beq     .Lc2u_1fupi
161 .Lc2u_1nowords: mov     r3, r7, get_byte_1
162                 teq     ip, #0
163                 beq     .Lc2u_finished
164                 cmp     ip, #2
165 USER(           T(strb) r3, [r0], #1)                   @ May fault
166                 movge   r3, r7, get_byte_2
167 USER(           T(strgeb) r3, [r0], #1)                 @ May fault
168                 movgt   r3, r7, get_byte_3
169 USER(           T(strgtb) r3, [r0], #1)                 @ May fault
170                 b       .Lc2u_finished
171
172 .Lc2u_2fupi:    subs    r2, r2, #4
173                 addmi   ip, r2, #4
174                 bmi     .Lc2u_2nowords
175                 mov     r3, r7, pull #16
176                 ldr     r7, [r1], #4
177                 orr     r3, r3, r7, push #16
178 USER(           T(str)  r3, [r0], #4)                   @ May fault
179                 mov     ip, r0, lsl #32 - PAGE_SHIFT
180                 rsb     ip, ip, #0
181                 movs    ip, ip, lsr #32 - PAGE_SHIFT
182                 beq     .Lc2u_2fupi
183                 cmp     r2, ip
184                 movlt   ip, r2
185                 sub     r2, r2, ip
186                 subs    ip, ip, #16
187                 blt     .Lc2u_2rem8lp
188
189 .Lc2u_2cpy8lp:  mov     r3, r7, pull #16
190                 ldmia   r1!, {r4 - r7}
191                 subs    ip, ip, #16
192                 orr     r3, r3, r4, push #16
193                 mov     r4, r4, pull #16
194                 orr     r4, r4, r5, push #16
195                 mov     r5, r5, pull #16
196                 orr     r5, r5, r6, push #16
197                 mov     r6, r6, pull #16
198                 orr     r6, r6, r7, push #16
199                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
200                 bpl     .Lc2u_2cpy8lp
201
202 .Lc2u_2rem8lp:  tst     ip, #8
203                 movne   r3, r7, pull #16
204                 ldmneia r1!, {r4, r7}
205                 orrne   r3, r3, r4, push #16
206                 movne   r4, r4, pull #16
207                 orrne   r4, r4, r7, push #16
208                 stmneia r0!, {r3 - r4}                  @ Shouldnt fault
209                 tst     ip, #4
210                 movne   r3, r7, pull #16
211                 ldrne   r7, [r1], #4
212                 orrne   r3, r3, r7, push #16
213                 T(strne) r3, [r0], #4                   @ Shouldnt fault
214                 ands    ip, ip, #3
215                 beq     .Lc2u_2fupi
216 .Lc2u_2nowords: mov     r3, r7, get_byte_2
217                 teq     ip, #0
218                 beq     .Lc2u_finished
219                 cmp     ip, #2
220 USER(           T(strb) r3, [r0], #1)                   @ May fault
221                 movge   r3, r7, get_byte_3
222 USER(           T(strgeb) r3, [r0], #1)                 @ May fault
223                 ldrgtb  r3, [r1], #0
224 USER(           T(strgtb) r3, [r0], #1)                 @ May fault
225                 b       .Lc2u_finished
226
227 .Lc2u_3fupi:    subs    r2, r2, #4
228                 addmi   ip, r2, #4
229                 bmi     .Lc2u_3nowords
230                 mov     r3, r7, pull #24
231                 ldr     r7, [r1], #4
232                 orr     r3, r3, r7, push #8
233 USER(           T(str)  r3, [r0], #4)                   @ May fault
234                 mov     ip, r0, lsl #32 - PAGE_SHIFT
235                 rsb     ip, ip, #0
236                 movs    ip, ip, lsr #32 - PAGE_SHIFT
237                 beq     .Lc2u_3fupi
238                 cmp     r2, ip
239                 movlt   ip, r2
240                 sub     r2, r2, ip
241                 subs    ip, ip, #16
242                 blt     .Lc2u_3rem8lp
243
244 .Lc2u_3cpy8lp:  mov     r3, r7, pull #24
245                 ldmia   r1!, {r4 - r7}
246                 subs    ip, ip, #16
247                 orr     r3, r3, r4, push #8
248                 mov     r4, r4, pull #24
249                 orr     r4, r4, r5, push #8
250                 mov     r5, r5, pull #24
251                 orr     r5, r5, r6, push #8
252                 mov     r6, r6, pull #24
253                 orr     r6, r6, r7, push #8
254                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
255                 bpl     .Lc2u_3cpy8lp
256
257 .Lc2u_3rem8lp:  tst     ip, #8
258                 movne   r3, r7, pull #24
259                 ldmneia r1!, {r4, r7}
260                 orrne   r3, r3, r4, push #8
261                 movne   r4, r4, pull #24
262                 orrne   r4, r4, r7, push #8
263                 stmneia r0!, {r3 - r4}                  @ Shouldnt fault
264                 tst     ip, #4
265                 movne   r3, r7, pull #24
266                 ldrne   r7, [r1], #4
267                 orrne   r3, r3, r7, push #8
268                 T(strne) r3, [r0], #4                   @ Shouldnt fault
269                 ands    ip, ip, #3
270                 beq     .Lc2u_3fupi
271 .Lc2u_3nowords: mov     r3, r7, get_byte_3
272                 teq     ip, #0
273                 beq     .Lc2u_finished
274                 cmp     ip, #2
275 USER(           T(strb) r3, [r0], #1)                   @ May fault
276                 ldrgeb  r3, [r1], #1
277 USER(           T(strgeb) r3, [r0], #1)                 @ May fault
278                 ldrgtb  r3, [r1], #0
279 USER(           T(strgtb) r3, [r0], #1)                 @ May fault
280                 b       .Lc2u_finished
281 ENDPROC(__copy_to_user)
282
283                 .pushsection .fixup,"ax"
284                 .align  0
285 9001:           ldmfd   sp!, {r0, r4 - r7, pc}
286                 .popsection
287
288 /* Prototype: unsigned long __copy_from_user(void *to,const void *from,unsigned long n);
289  * Purpose  : copy a block from user memory to kernel memory
290  * Params   : to   - kernel memory
291  *          : from - user memory
292  *          : n    - number of bytes to copy
293  * Returns  : Number of bytes NOT copied.
294  */
295 .Lcfu_dest_not_aligned:
296                 rsb     ip, ip, #4
297                 cmp     ip, #2
298 USER(           T(ldrb) r3, [r1], #1)                   @ May fault
299                 strb    r3, [r0], #1
300 USER(           T(ldrgeb) r3, [r1], #1)                 @ May fault
301                 strgeb  r3, [r0], #1
302 USER(           T(ldrgtb) r3, [r1], #1)                 @ May fault
303                 strgtb  r3, [r0], #1
304                 sub     r2, r2, ip
305                 b       .Lcfu_dest_aligned
306
307 ENTRY(__copy_from_user)
308                 stmfd   sp!, {r0, r2, r4 - r7, lr}
309                 cmp     r2, #4
310                 blt     .Lcfu_not_enough
311                 ands    ip, r0, #3
312                 bne     .Lcfu_dest_not_aligned
313 .Lcfu_dest_aligned:
314                 ands    ip, r1, #3
315                 bne     .Lcfu_src_not_aligned
316
317 /*
318  * Seeing as there has to be at least 8 bytes to copy, we can
319  * copy one word, and force a user-mode page fault...
320  */
321
322 .Lcfu_0fupi:    subs    r2, r2, #4
323                 addmi   ip, r2, #4
324                 bmi     .Lcfu_0nowords
325 USER(           T(ldr)  r3, [r1], #4)
326                 str     r3, [r0], #4
327                 mov     ip, r1, lsl #32 - PAGE_SHIFT    @ On each page, use a ld/st??t instruction
328                 rsb     ip, ip, #0
329                 movs    ip, ip, lsr #32 - PAGE_SHIFT
330                 beq     .Lcfu_0fupi
331 /*
332  * ip = max no. of bytes to copy before needing another "strt" insn
333  */
334                 cmp     r2, ip
335                 movlt   ip, r2
336                 sub     r2, r2, ip
337                 subs    ip, ip, #32
338                 blt     .Lcfu_0rem8lp
339
340 .Lcfu_0cpy8lp:  ldmia   r1!, {r3 - r6}                  @ Shouldnt fault
341                 stmia   r0!, {r3 - r6}
342                 ldmia   r1!, {r3 - r6}                  @ Shouldnt fault
343                 subs    ip, ip, #32
344                 stmia   r0!, {r3 - r6}
345                 bpl     .Lcfu_0cpy8lp
346
347 .Lcfu_0rem8lp:  cmn     ip, #16
348                 ldmgeia r1!, {r3 - r6}                  @ Shouldnt fault
349                 stmgeia r0!, {r3 - r6}
350                 tst     ip, #8
351                 ldmneia r1!, {r3 - r4}                  @ Shouldnt fault
352                 stmneia r0!, {r3 - r4}
353                 tst     ip, #4
354                 T(ldrne) r3, [r1], #4                   @ Shouldnt fault
355                 strne   r3, [r0], #4
356                 ands    ip, ip, #3
357                 beq     .Lcfu_0fupi
358 .Lcfu_0nowords: teq     ip, #0
359                 beq     .Lcfu_finished
360 .Lcfu_nowords:  cmp     ip, #2
361 USER(           T(ldrb) r3, [r1], #1)                   @ May fault
362                 strb    r3, [r0], #1
363 USER(           T(ldrgeb) r3, [r1], #1)                 @ May fault
364                 strgeb  r3, [r0], #1
365 USER(           T(ldrgtb) r3, [r1], #1)                 @ May fault
366                 strgtb  r3, [r0], #1
367                 b       .Lcfu_finished
368
369 .Lcfu_not_enough:
370                 movs    ip, r2
371                 bne     .Lcfu_nowords
372 .Lcfu_finished: mov     r0, #0
373                 add     sp, sp, #8
374                 ldmfd   sp!, {r4 - r7, pc}
375
376 .Lcfu_src_not_aligned:
377                 bic     r1, r1, #3
378 USER(           T(ldr)  r7, [r1], #4)                   @ May fault
379                 cmp     ip, #2
380                 bgt     .Lcfu_3fupi
381                 beq     .Lcfu_2fupi
382 .Lcfu_1fupi:    subs    r2, r2, #4
383                 addmi   ip, r2, #4
384                 bmi     .Lcfu_1nowords
385                 mov     r3, r7, pull #8
386 USER(           T(ldr)  r7, [r1], #4)                   @ May fault
387                 orr     r3, r3, r7, push #24
388                 str     r3, [r0], #4
389                 mov     ip, r1, lsl #32 - PAGE_SHIFT
390                 rsb     ip, ip, #0
391                 movs    ip, ip, lsr #32 - PAGE_SHIFT
392                 beq     .Lcfu_1fupi
393                 cmp     r2, ip
394                 movlt   ip, r2
395                 sub     r2, r2, ip
396                 subs    ip, ip, #16
397                 blt     .Lcfu_1rem8lp
398
399 .Lcfu_1cpy8lp:  mov     r3, r7, pull #8
400                 ldmia   r1!, {r4 - r7}                  @ Shouldnt fault
401                 subs    ip, ip, #16
402                 orr     r3, r3, r4, push #24
403                 mov     r4, r4, pull #8
404                 orr     r4, r4, r5, push #24
405                 mov     r5, r5, pull #8
406                 orr     r5, r5, r6, push #24
407                 mov     r6, r6, pull #8
408                 orr     r6, r6, r7, push #24
409                 stmia   r0!, {r3 - r6}
410                 bpl     .Lcfu_1cpy8lp
411
412 .Lcfu_1rem8lp:  tst     ip, #8
413                 movne   r3, r7, pull #8
414                 ldmneia r1!, {r4, r7}                   @ Shouldnt fault
415                 orrne   r3, r3, r4, push #24
416                 movne   r4, r4, pull #8
417                 orrne   r4, r4, r7, push #24
418                 stmneia r0!, {r3 - r4}
419                 tst     ip, #4
420                 movne   r3, r7, pull #8
421 USER(           T(ldrne) r7, [r1], #4)                  @ May fault
422                 orrne   r3, r3, r7, push #24
423                 strne   r3, [r0], #4
424                 ands    ip, ip, #3
425                 beq     .Lcfu_1fupi
426 .Lcfu_1nowords: mov     r3, r7, get_byte_1
427                 teq     ip, #0
428                 beq     .Lcfu_finished
429                 cmp     ip, #2
430                 strb    r3, [r0], #1
431                 movge   r3, r7, get_byte_2
432                 strgeb  r3, [r0], #1
433                 movgt   r3, r7, get_byte_3
434                 strgtb  r3, [r0], #1
435                 b       .Lcfu_finished
436
437 .Lcfu_2fupi:    subs    r2, r2, #4
438                 addmi   ip, r2, #4
439                 bmi     .Lcfu_2nowords
440                 mov     r3, r7, pull #16
441 USER(           T(ldr)  r7, [r1], #4)                   @ May fault
442                 orr     r3, r3, r7, push #16
443                 str     r3, [r0], #4
444                 mov     ip, r1, lsl #32 - PAGE_SHIFT
445                 rsb     ip, ip, #0
446                 movs    ip, ip, lsr #32 - PAGE_SHIFT
447                 beq     .Lcfu_2fupi
448                 cmp     r2, ip
449                 movlt   ip, r2
450                 sub     r2, r2, ip
451                 subs    ip, ip, #16
452                 blt     .Lcfu_2rem8lp
453
454
455 .Lcfu_2cpy8lp:  mov     r3, r7, pull #16
456                 ldmia   r1!, {r4 - r7}                  @ Shouldnt fault
457                 subs    ip, ip, #16
458                 orr     r3, r3, r4, push #16
459                 mov     r4, r4, pull #16
460                 orr     r4, r4, r5, push #16
461                 mov     r5, r5, pull #16
462                 orr     r5, r5, r6, push #16
463                 mov     r6, r6, pull #16
464                 orr     r6, r6, r7, push #16
465                 stmia   r0!, {r3 - r6}
466                 bpl     .Lcfu_2cpy8lp
467
468 .Lcfu_2rem8lp:  tst     ip, #8
469                 movne   r3, r7, pull #16
470                 ldmneia r1!, {r4, r7}                   @ Shouldnt fault
471                 orrne   r3, r3, r4, push #16
472                 movne   r4, r4, pull #16
473                 orrne   r4, r4, r7, push #16
474                 stmneia r0!, {r3 - r4}
475                 tst     ip, #4
476                 movne   r3, r7, pull #16
477 USER(           T(ldrne) r7, [r1], #4)                  @ May fault
478                 orrne   r3, r3, r7, push #16
479                 strne   r3, [r0], #4
480                 ands    ip, ip, #3
481                 beq     .Lcfu_2fupi
482 .Lcfu_2nowords: mov     r3, r7, get_byte_2
483                 teq     ip, #0
484                 beq     .Lcfu_finished
485                 cmp     ip, #2
486                 strb    r3, [r0], #1
487                 movge   r3, r7, get_byte_3
488                 strgeb  r3, [r0], #1
489 USER(           T(ldrgtb) r3, [r1], #0)                 @ May fault
490                 strgtb  r3, [r0], #1
491                 b       .Lcfu_finished
492
493 .Lcfu_3fupi:    subs    r2, r2, #4
494                 addmi   ip, r2, #4
495                 bmi     .Lcfu_3nowords
496                 mov     r3, r7, pull #24
497 USER(           T(ldr)  r7, [r1], #4)                   @ May fault
498                 orr     r3, r3, r7, push #8
499                 str     r3, [r0], #4
500                 mov     ip, r1, lsl #32 - PAGE_SHIFT
501                 rsb     ip, ip, #0
502                 movs    ip, ip, lsr #32 - PAGE_SHIFT
503                 beq     .Lcfu_3fupi
504                 cmp     r2, ip
505                 movlt   ip, r2
506                 sub     r2, r2, ip
507                 subs    ip, ip, #16
508                 blt     .Lcfu_3rem8lp
509
510 .Lcfu_3cpy8lp:  mov     r3, r7, pull #24
511                 ldmia   r1!, {r4 - r7}                  @ Shouldnt fault
512                 orr     r3, r3, r4, push #8
513                 mov     r4, r4, pull #24
514                 orr     r4, r4, r5, push #8
515                 mov     r5, r5, pull #24
516                 orr     r5, r5, r6, push #8
517                 mov     r6, r6, pull #24
518                 orr     r6, r6, r7, push #8
519                 stmia   r0!, {r3 - r6}
520                 subs    ip, ip, #16
521                 bpl     .Lcfu_3cpy8lp
522
523 .Lcfu_3rem8lp:  tst     ip, #8
524                 movne   r3, r7, pull #24
525                 ldmneia r1!, {r4, r7}                   @ Shouldnt fault
526                 orrne   r3, r3, r4, push #8
527                 movne   r4, r4, pull #24
528                 orrne   r4, r4, r7, push #8
529                 stmneia r0!, {r3 - r4}
530                 tst     ip, #4
531                 movne   r3, r7, pull #24
532 USER(           T(ldrne) r7, [r1], #4)                  @ May fault
533                 orrne   r3, r3, r7, push #8
534                 strne   r3, [r0], #4
535                 ands    ip, ip, #3
536                 beq     .Lcfu_3fupi
537 .Lcfu_3nowords: mov     r3, r7, get_byte_3
538                 teq     ip, #0
539                 beq     .Lcfu_finished
540                 cmp     ip, #2
541                 strb    r3, [r0], #1
542 USER(           T(ldrgeb) r3, [r1], #1)                 @ May fault
543                 strgeb  r3, [r0], #1
544 USER(           T(ldrgtb) r3, [r1], #1)                 @ May fault
545                 strgtb  r3, [r0], #1
546                 b       .Lcfu_finished
547 ENDPROC(__copy_from_user)
548
549                 .pushsection .fixup,"ax"
550                 .align  0
551                 /*
552                  * We took an exception.  r0 contains a pointer to
553                  * the byte not copied.
554                  */
555 9001:           ldr     r2, [sp], #4                    @ void *to
556                 sub     r2, r0, r2                      @ bytes copied
557                 ldr     r1, [sp], #4                    @ unsigned long count
558                 subs    r4, r1, r2                      @ bytes left to copy
559                 movne   r1, r4
560                 blne    __memzero
561                 mov     r0, r4
562                 ldmfd   sp!, {r4 - r7, pc}
563                 .popsection
564