| 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
|---|
| 2 | /* |
|---|
| 3 | * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com> |
|---|
| 4 | * Copyright 2002 Andi Kleen, SuSE Labs. |
|---|
| 5 | * |
|---|
| 6 | * Functions to copy from and to user space. |
|---|
| 7 | */ |
|---|
| 8 | |
|---|
| 9 | #include <linux/export.h> |
|---|
| 10 | #include <linux/linkage.h> |
|---|
| 11 | #include <asm/cpufeatures.h> |
|---|
| 12 | #include <asm/alternative.h> |
|---|
| 13 | #include <asm/asm.h> |
|---|
| 14 | |
|---|
| 15 | /* |
|---|
| 16 | * rep_movs_alternative - memory copy with exception handling. |
|---|
| 17 | * This version is for CPUs that don't have FSRM (Fast Short Rep Movs) |
|---|
| 18 | * |
|---|
| 19 | * Input: |
|---|
| 20 | * rdi destination |
|---|
| 21 | * rsi source |
|---|
| 22 | * rcx count |
|---|
| 23 | * |
|---|
| 24 | * Output: |
|---|
| 25 | * rcx uncopied bytes or 0 if successful. |
|---|
| 26 | * |
|---|
| 27 | * NOTE! The calling convention is very intentionally the same as |
|---|
| 28 | * for 'rep movs', so that we can rewrite the function call with |
|---|
| 29 | * just a plain 'rep movs' on machines that have FSRM. But to make |
|---|
| 30 | * it simpler for us, we can clobber rsi/rdi and rax freely. |
|---|
| 31 | */ |
|---|
| 32 | SYM_FUNC_START(rep_movs_alternative) |
|---|
| 33 | cmpq $64,%rcx |
|---|
| 34 | jae .Llarge |
|---|
| 35 | |
|---|
| 36 | cmp $8,%ecx |
|---|
| 37 | jae .Lword |
|---|
| 38 | |
|---|
| 39 | testl %ecx,%ecx |
|---|
| 40 | je .Lexit |
|---|
| 41 | |
|---|
| 42 | .Lcopy_user_tail: |
|---|
| 43 | 0: movb (%rsi),%al |
|---|
| 44 | 1: movb %al,(%rdi) |
|---|
| 45 | inc %rdi |
|---|
| 46 | inc %rsi |
|---|
| 47 | dec %rcx |
|---|
| 48 | jne .Lcopy_user_tail |
|---|
| 49 | .Lexit: |
|---|
| 50 | RET |
|---|
| 51 | |
|---|
| 52 | _ASM_EXTABLE_UA( 0b, .Lexit) |
|---|
| 53 | _ASM_EXTABLE_UA( 1b, .Lexit) |
|---|
| 54 | |
|---|
| 55 | .p2align 4 |
|---|
| 56 | .Lword: |
|---|
| 57 | 2: movq (%rsi),%rax |
|---|
| 58 | 3: movq %rax,(%rdi) |
|---|
| 59 | addq $8,%rsi |
|---|
| 60 | addq $8,%rdi |
|---|
| 61 | sub $8,%ecx |
|---|
| 62 | je .Lexit |
|---|
| 63 | cmp $8,%ecx |
|---|
| 64 | jae .Lword |
|---|
| 65 | jmp .Lcopy_user_tail |
|---|
| 66 | |
|---|
| 67 | _ASM_EXTABLE_UA( 2b, .Lcopy_user_tail) |
|---|
| 68 | _ASM_EXTABLE_UA( 3b, .Lcopy_user_tail) |
|---|
| 69 | |
|---|
| 70 | .Llarge: |
|---|
| 71 | 0: ALTERNATIVE "jmp .Llarge_movsq" , "rep movsb" , X86_FEATURE_ERMS |
|---|
| 72 | 1: RET |
|---|
| 73 | |
|---|
| 74 | _ASM_EXTABLE_UA( 0b, 1b) |
|---|
| 75 | |
|---|
| 76 | .Llarge_movsq: |
|---|
| 77 | movq %rcx,%rax |
|---|
| 78 | shrq $3,%rcx |
|---|
| 79 | andl $7,%eax |
|---|
| 80 | 0: rep movsq |
|---|
| 81 | movl %eax,%ecx |
|---|
| 82 | testl %ecx,%ecx |
|---|
| 83 | jne .Lcopy_user_tail |
|---|
| 84 | RET |
|---|
| 85 | |
|---|
| 86 | 1: leaq (%rax,%rcx,8),%rcx |
|---|
| 87 | jmp .Lcopy_user_tail |
|---|
| 88 | |
|---|
| 89 | _ASM_EXTABLE_UA( 0b, 1b) |
|---|
| 90 | SYM_FUNC_END(rep_movs_alternative) |
|---|
| 91 | EXPORT_SYMBOL(rep_movs_alternative) |
|---|
| 92 | |
|---|
