1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | /* |
3 | * AMD Memory Encryption Support |
4 | * |
5 | * Copyright (C) 2017 Advanced Micro Devices, Inc. |
6 | * |
7 | * Author: Tom Lendacky <thomas.lendacky@amd.com> |
8 | */ |
9 | |
10 | #include <linux/linkage.h> |
11 | |
12 | #include <asm/processor-flags.h> |
13 | #include <asm/msr.h> |
14 | #include <asm/asm-offsets.h> |
15 | #include <asm/segment.h> |
16 | #include <asm/trapnr.h> |
17 | |
18 | .text |
19 | .code32 |
20 | SYM_FUNC_START(get_sev_encryption_bit) |
21 | push %ebx |
22 | |
23 | movl $0x80000000, %eax /* CPUID to check the highest leaf */ |
24 | cpuid |
25 | cmpl $0x8000001f, %eax /* See if 0x8000001f is available */ |
26 | jb .Lno_sev |
27 | |
28 | /* |
29 | * Check for the SEV feature: |
30 | * CPUID Fn8000_001F[EAX] - Bit 1 |
31 | * CPUID Fn8000_001F[EBX] - Bits 5:0 |
32 | * Pagetable bit position used to indicate encryption |
33 | */ |
34 | movl $0x8000001f, %eax |
35 | cpuid |
36 | bt $1, %eax /* Check if SEV is available */ |
37 | jnc .Lno_sev |
38 | |
39 | movl $MSR_AMD64_SEV, %ecx /* Read the SEV MSR */ |
40 | rdmsr |
41 | bt $MSR_AMD64_SEV_ENABLED_BIT, %eax /* Check if SEV is active */ |
42 | jnc .Lno_sev |
43 | |
44 | movl %ebx, %eax |
45 | andl $0x3f, %eax /* Return the encryption bit location */ |
46 | jmp .Lsev_exit |
47 | |
48 | .Lno_sev: |
49 | xor %eax, %eax |
50 | |
51 | .Lsev_exit: |
52 | pop %ebx |
53 | RET |
54 | SYM_FUNC_END(get_sev_encryption_bit) |
55 | |
56 | /** |
57 | * sev_es_req_cpuid - Request a CPUID value from the Hypervisor using |
58 | * the GHCB MSR protocol |
59 | * |
60 | * @%eax: Register to request (0=EAX, 1=EBX, 2=ECX, 3=EDX) |
61 | * @%edx: CPUID Function |
62 | * |
63 | * Returns 0 in %eax on success, non-zero on failure |
64 | * %edx returns CPUID value on success |
65 | */ |
66 | SYM_CODE_START_LOCAL(sev_es_req_cpuid) |
67 | shll $30, %eax |
68 | orl $0x00000004, %eax |
69 | movl $MSR_AMD64_SEV_ES_GHCB, %ecx |
70 | wrmsr |
71 | rep; vmmcall # VMGEXIT |
72 | rdmsr |
73 | |
74 | /* Check response */ |
75 | movl %eax, %ecx |
76 | andl $0x3ffff000, %ecx # Bits [12-29] MBZ |
77 | jnz 2f |
78 | |
79 | /* Check return code */ |
80 | andl $0xfff, %eax |
81 | cmpl $5, %eax |
82 | jne 2f |
83 | |
84 | /* All good - return success */ |
85 | xorl %eax, %eax |
86 | 1: |
87 | RET |
88 | 2: |
89 | movl $-1, %eax |
90 | jmp 1b |
91 | SYM_CODE_END(sev_es_req_cpuid) |
92 | |
93 | SYM_CODE_START_LOCAL(startup32_vc_handler) |
94 | pushl %eax |
95 | pushl %ebx |
96 | pushl %ecx |
97 | pushl %edx |
98 | |
99 | /* Keep CPUID function in %ebx */ |
100 | movl %eax, %ebx |
101 | |
102 | /* Check if error-code == SVM_EXIT_CPUID */ |
103 | cmpl $0x72, 16(%esp) |
104 | jne .Lfail |
105 | |
106 | movl $0, %eax # Request CPUID[fn].EAX |
107 | movl %ebx, %edx # CPUID fn |
108 | call sev_es_req_cpuid # Call helper |
109 | testl %eax, %eax # Check return code |
110 | jnz .Lfail |
111 | movl %edx, 12(%esp) # Store result |
112 | |
113 | movl $1, %eax # Request CPUID[fn].EBX |
114 | movl %ebx, %edx # CPUID fn |
115 | call sev_es_req_cpuid # Call helper |
116 | testl %eax, %eax # Check return code |
117 | jnz .Lfail |
118 | movl %edx, 8(%esp) # Store result |
119 | |
120 | movl $2, %eax # Request CPUID[fn].ECX |
121 | movl %ebx, %edx # CPUID fn |
122 | call sev_es_req_cpuid # Call helper |
123 | testl %eax, %eax # Check return code |
124 | jnz .Lfail |
125 | movl %edx, 4(%esp) # Store result |
126 | |
127 | movl $3, %eax # Request CPUID[fn].EDX |
128 | movl %ebx, %edx # CPUID fn |
129 | call sev_es_req_cpuid # Call helper |
130 | testl %eax, %eax # Check return code |
131 | jnz .Lfail |
132 | movl %edx, 0(%esp) # Store result |
133 | |
134 | /* |
135 | * Sanity check CPUID results from the Hypervisor. See comment in |
136 | * do_vc_no_ghcb() for more details on why this is necessary. |
137 | */ |
138 | |
139 | /* Fail if SEV leaf not available in CPUID[0x80000000].EAX */ |
140 | cmpl $0x80000000, %ebx |
141 | jne .Lcheck_sev |
142 | cmpl $0x8000001f, 12(%esp) |
143 | jb .Lfail |
144 | jmp .Ldone |
145 | |
146 | .Lcheck_sev: |
147 | /* Fail if SEV bit not set in CPUID[0x8000001f].EAX[1] */ |
148 | cmpl $0x8000001f, %ebx |
149 | jne .Ldone |
150 | btl $1, 12(%esp) |
151 | jnc .Lfail |
152 | |
153 | .Ldone: |
154 | popl %edx |
155 | popl %ecx |
156 | popl %ebx |
157 | popl %eax |
158 | |
159 | /* Remove error code */ |
160 | addl $4, %esp |
161 | |
162 | /* Jump over CPUID instruction */ |
163 | addl $2, (%esp) |
164 | |
165 | iret |
166 | .Lfail: |
167 | /* Send terminate request to Hypervisor */ |
168 | movl $0x100, %eax |
169 | xorl %edx, %edx |
170 | movl $MSR_AMD64_SEV_ES_GHCB, %ecx |
171 | wrmsr |
172 | rep; vmmcall |
173 | |
174 | /* If request fails, go to hlt loop */ |
175 | hlt |
176 | jmp .Lfail |
177 | SYM_CODE_END(startup32_vc_handler) |
178 | |
179 | /* |
180 | * Write an IDT entry into boot32_idt |
181 | * |
182 | * Parameters: |
183 | * |
184 | * %eax: Handler address |
185 | * %edx: Vector number |
186 | * %ecx: IDT address |
187 | */ |
188 | SYM_FUNC_START_LOCAL(startup32_set_idt_entry) |
189 | /* IDT entry address to %ecx */ |
190 | leal (%ecx, %edx, 8), %ecx |
191 | |
192 | /* Build IDT entry, lower 4 bytes */ |
193 | movl %eax, %edx |
194 | andl $0x0000ffff, %edx # Target code segment offset [15:0] |
195 | orl $(__KERNEL32_CS << 16), %edx # Target code segment selector |
196 | |
197 | /* Store lower 4 bytes to IDT */ |
198 | movl %edx, (%ecx) |
199 | |
200 | /* Build IDT entry, upper 4 bytes */ |
201 | movl %eax, %edx |
202 | andl $0xffff0000, %edx # Target code segment offset [31:16] |
203 | orl $0x00008e00, %edx # Present, Type 32-bit Interrupt Gate |
204 | |
205 | /* Store upper 4 bytes to IDT */ |
206 | movl %edx, 4(%ecx) |
207 | |
208 | RET |
209 | SYM_FUNC_END(startup32_set_idt_entry) |
210 | |
211 | SYM_FUNC_START(startup32_load_idt) |
212 | push %ebp |
213 | push %ebx |
214 | |
215 | call 1f |
216 | 1: pop %ebp |
217 | |
218 | leal (boot32_idt - 1b)(%ebp), %ebx |
219 | |
220 | /* #VC handler */ |
221 | leal (startup32_vc_handler - 1b)(%ebp), %eax |
222 | movl $X86_TRAP_VC, %edx |
223 | movl %ebx, %ecx |
224 | call startup32_set_idt_entry |
225 | |
226 | /* Load IDT */ |
227 | leal (boot32_idt_desc - 1b)(%ebp), %ecx |
228 | movl %ebx, 2(%ecx) |
229 | lidt (%ecx) |
230 | |
231 | pop %ebx |
232 | pop %ebp |
233 | RET |
234 | SYM_FUNC_END(startup32_load_idt) |
235 | |
236 | /* |
237 | * Check for the correct C-bit position when the startup_32 boot-path is used. |
238 | * |
239 | * The check makes use of the fact that all memory is encrypted when paging is |
240 | * disabled. The function creates 64 bits of random data using the RDRAND |
241 | * instruction. RDRAND is mandatory for SEV guests, so always available. If the |
242 | * hypervisor violates that the kernel will crash right here. |
243 | * |
244 | * The 64 bits of random data are stored to a memory location and at the same |
245 | * time kept in the %eax and %ebx registers. Since encryption is always active |
246 | * when paging is off the random data will be stored encrypted in main memory. |
247 | * |
248 | * Then paging is enabled. When the C-bit position is correct all memory is |
249 | * still mapped encrypted and comparing the register values with memory will |
250 | * succeed. An incorrect C-bit position will map all memory unencrypted, so that |
251 | * the compare will use the encrypted random data and fail. |
252 | */ |
253 | SYM_FUNC_START(startup32_check_sev_cbit) |
254 | pushl %ebx |
255 | pushl %ebp |
256 | |
257 | call 0f |
258 | 0: popl %ebp |
259 | |
260 | /* Check for non-zero sev_status */ |
261 | movl (sev_status - 0b)(%ebp), %eax |
262 | testl %eax, %eax |
263 | jz 4f |
264 | |
265 | /* |
266 | * Get two 32-bit random values - Don't bail out if RDRAND fails |
267 | * because it is better to prevent forward progress if no random value |
268 | * can be gathered. |
269 | */ |
270 | 1: rdrand %eax |
271 | jnc 1b |
272 | 2: rdrand %ebx |
273 | jnc 2b |
274 | |
275 | /* Store to memory and keep it in the registers */ |
276 | leal (sev_check_data - 0b)(%ebp), %ebp |
277 | movl %eax, 0(%ebp) |
278 | movl %ebx, 4(%ebp) |
279 | |
280 | /* Enable paging to see if encryption is active */ |
281 | movl %cr0, %edx /* Backup %cr0 in %edx */ |
282 | movl $(X86_CR0_PG | X86_CR0_PE), %ecx /* Enable Paging and Protected mode */ |
283 | movl %ecx, %cr0 |
284 | |
285 | cmpl %eax, 0(%ebp) |
286 | jne 3f |
287 | cmpl %ebx, 4(%ebp) |
288 | jne 3f |
289 | |
290 | movl %edx, %cr0 /* Restore previous %cr0 */ |
291 | |
292 | jmp 4f |
293 | |
294 | 3: /* Check failed - hlt the machine */ |
295 | hlt |
296 | jmp 3b |
297 | |
298 | 4: |
299 | popl %ebp |
300 | popl %ebx |
301 | RET |
302 | SYM_FUNC_END(startup32_check_sev_cbit) |
303 | |
304 | .code64 |
305 | |
306 | #include "../../kernel/sev_verify_cbit.S" |
307 | |
308 | .data |
309 | |
310 | .balign 8 |
311 | SYM_DATA(sme_me_mask, .quad 0) |
312 | SYM_DATA(sev_status, .quad 0) |
313 | SYM_DATA(sev_check_data, .quad 0) |
314 | |
315 | SYM_DATA_START_LOCAL(boot32_idt) |
316 | .rept 32 |
317 | .quad 0 |
318 | .endr |
319 | SYM_DATA_END(boot32_idt) |
320 | |
321 | SYM_DATA_START_LOCAL(boot32_idt_desc) |
322 | .word . - boot32_idt - 1 |
323 | .long 0 |
324 | SYM_DATA_END(boot32_idt_desc) |
325 | |