1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * __get_user functions.
4 *
5 * (C) Copyright 1998 Linus Torvalds
6 * (C) Copyright 2005 Andi Kleen
7 * (C) Copyright 2008 Glauber Costa
8 *
9 * These functions have a non-standard call interface
10 * to make them more efficient, especially as they
11 * return an error value in addition to the "real"
12 * return value.
13 */
14
15/*
16 * __get_user_X
17 *
18 * Inputs: %[r|e]ax contains the address.
19 *
20 * Outputs: %[r|e]ax is error code (0 or -EFAULT)
21 * %[r|e]dx contains zero-extended value
22 * %ecx contains the high half for 32-bit __get_user_8
23 *
24 *
25 * These functions should not modify any other registers,
26 * as they get called from within inline assembly.
27 */
28
29#include <linux/export.h>
30#include <linux/linkage.h>
31#include <linux/objtool.h>
32#include <asm/page_types.h>
33#include <asm/errno.h>
34#include <asm/asm-offsets.h>
35#include <asm/thread_info.h>
36#include <asm/asm.h>
37#include <asm/smap.h>
38#include <asm/runtime-const.h>
39
40#define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC
41
42.macro check_range size:req
43.if IS_ENABLED(CONFIG_X86_64)
44 RUNTIME_CONST_PTR USER_PTR_MAX, rdx
45 cmp %rdx, %rax
46 cmova %rdx, %rax
47.else
48 cmp $TASK_SIZE_MAX-\size+1, %eax
49 jae .Lbad_get_user
50 sbb %edx, %edx /* array_index_mask_nospec() */
51 and %edx, %eax
52.endif
53.endm
54
55.macro UACCESS op src dst
561: \op \src,\dst
57 _ASM_EXTABLE_UA(1b, __get_user_handle_exception)
58.endm
59
60
61 .text
62SYM_FUNC_START(__get_user_1)
63 ANNOTATE_NOENDBR
64 check_range size=1
65 ASM_STAC
66 UACCESS movzbl (%_ASM_AX),%edx
67 xor %eax,%eax
68 ASM_CLAC
69 RET
70SYM_FUNC_END(__get_user_1)
71EXPORT_SYMBOL(__get_user_1)
72
73SYM_FUNC_START(__get_user_2)
74 ANNOTATE_NOENDBR
75 check_range size=2
76 ASM_STAC
77 UACCESS movzwl (%_ASM_AX),%edx
78 xor %eax,%eax
79 ASM_CLAC
80 RET
81SYM_FUNC_END(__get_user_2)
82EXPORT_SYMBOL(__get_user_2)
83
84SYM_FUNC_START(__get_user_4)
85 ANNOTATE_NOENDBR
86 check_range size=4
87 ASM_STAC
88 UACCESS movl (%_ASM_AX),%edx
89 xor %eax,%eax
90 ASM_CLAC
91 RET
92SYM_FUNC_END(__get_user_4)
93EXPORT_SYMBOL(__get_user_4)
94
95SYM_FUNC_START(__get_user_8)
96 ANNOTATE_NOENDBR
97#ifndef CONFIG_X86_64
98 xor %ecx,%ecx
99#endif
100 check_range size=8
101 ASM_STAC
102#ifdef CONFIG_X86_64
103 UACCESS movq (%_ASM_AX),%rdx
104#else
105 UACCESS movl (%_ASM_AX),%edx
106 UACCESS movl 4(%_ASM_AX),%ecx
107#endif
108 xor %eax,%eax
109 ASM_CLAC
110 RET
111SYM_FUNC_END(__get_user_8)
112EXPORT_SYMBOL(__get_user_8)
113
114/* .. and the same for __get_user, just without the range checks */
115SYM_FUNC_START(__get_user_nocheck_1)
116 ANNOTATE_NOENDBR
117 ASM_STAC
118 ASM_BARRIER_NOSPEC
119 UACCESS movzbl (%_ASM_AX),%edx
120 xor %eax,%eax
121 ASM_CLAC
122 RET
123SYM_FUNC_END(__get_user_nocheck_1)
124EXPORT_SYMBOL(__get_user_nocheck_1)
125
126SYM_FUNC_START(__get_user_nocheck_2)
127 ANNOTATE_NOENDBR
128 ASM_STAC
129 ASM_BARRIER_NOSPEC
130 UACCESS movzwl (%_ASM_AX),%edx
131 xor %eax,%eax
132 ASM_CLAC
133 RET
134SYM_FUNC_END(__get_user_nocheck_2)
135EXPORT_SYMBOL(__get_user_nocheck_2)
136
137SYM_FUNC_START(__get_user_nocheck_4)
138 ANNOTATE_NOENDBR
139 ASM_STAC
140 ASM_BARRIER_NOSPEC
141 UACCESS movl (%_ASM_AX),%edx
142 xor %eax,%eax
143 ASM_CLAC
144 RET
145SYM_FUNC_END(__get_user_nocheck_4)
146EXPORT_SYMBOL(__get_user_nocheck_4)
147
148SYM_FUNC_START(__get_user_nocheck_8)
149 ANNOTATE_NOENDBR
150 ASM_STAC
151 ASM_BARRIER_NOSPEC
152#ifdef CONFIG_X86_64
153 UACCESS movq (%_ASM_AX),%rdx
154#else
155 xor %ecx,%ecx
156 UACCESS movl (%_ASM_AX),%edx
157 UACCESS movl 4(%_ASM_AX),%ecx
158#endif
159 xor %eax,%eax
160 ASM_CLAC
161 RET
162SYM_FUNC_END(__get_user_nocheck_8)
163EXPORT_SYMBOL(__get_user_nocheck_8)
164
165
166SYM_CODE_START_LOCAL(__get_user_handle_exception)
167 ASM_CLAC
168.Lbad_get_user:
169 xor %edx,%edx
170 mov $(-EFAULT),%_ASM_AX
171 RET
172SYM_CODE_END(__get_user_handle_exception)
173

source code of linux/arch/x86/lib/getuser.S