1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * AMD Secure Encrypted Virtualization (SEV) guest driver interface |
4 | * |
5 | * Copyright (C) 2021 Advanced Micro Devices, Inc. |
6 | * |
7 | * Author: Brijesh Singh <brijesh.singh@amd.com> |
8 | */ |
9 | |
10 | #include <linux/module.h> |
11 | #include <linux/kernel.h> |
12 | #include <linux/types.h> |
13 | #include <linux/mutex.h> |
14 | #include <linux/io.h> |
15 | #include <linux/platform_device.h> |
16 | #include <linux/miscdevice.h> |
17 | #include <linux/set_memory.h> |
18 | #include <linux/fs.h> |
19 | #include <linux/tsm.h> |
20 | #include <crypto/aead.h> |
21 | #include <linux/scatterlist.h> |
22 | #include <linux/psp-sev.h> |
23 | #include <linux/sockptr.h> |
24 | #include <linux/cleanup.h> |
25 | #include <linux/uuid.h> |
26 | #include <uapi/linux/sev-guest.h> |
27 | #include <uapi/linux/psp-sev.h> |
28 | |
29 | #include <asm/svm.h> |
30 | #include <asm/sev.h> |
31 | |
32 | #include "sev-guest.h" |
33 | |
34 | #define DEVICE_NAME "sev-guest" |
35 | #define AAD_LEN 48 |
36 | #define MSG_HDR_VER 1 |
37 | |
38 | #define SNP_REQ_MAX_RETRY_DURATION (60*HZ) |
39 | #define SNP_REQ_RETRY_DELAY (2*HZ) |
40 | |
41 | struct snp_guest_crypto { |
42 | struct crypto_aead *tfm; |
43 | u8 *iv, *authtag; |
44 | int iv_len, a_len; |
45 | }; |
46 | |
47 | struct snp_guest_dev { |
48 | struct device *dev; |
49 | struct miscdevice misc; |
50 | |
51 | void *certs_data; |
52 | struct snp_guest_crypto *crypto; |
53 | /* request and response are in unencrypted memory */ |
54 | struct snp_guest_msg *request, *response; |
55 | |
56 | /* |
57 | * Avoid information leakage by double-buffering shared messages |
58 | * in fields that are in regular encrypted memory. |
59 | */ |
60 | struct snp_guest_msg secret_request, secret_response; |
61 | |
62 | struct snp_secrets_page_layout *layout; |
63 | struct snp_req_data input; |
64 | union { |
65 | struct snp_report_req report; |
66 | struct snp_derived_key_req derived_key; |
67 | struct snp_ext_report_req ext_report; |
68 | } req; |
69 | u32 *os_area_msg_seqno; |
70 | u8 *vmpck; |
71 | }; |
72 | |
73 | static u32 vmpck_id; |
74 | module_param(vmpck_id, uint, 0444); |
75 | MODULE_PARM_DESC(vmpck_id, "The VMPCK ID to use when communicating with the PSP." ); |
76 | |
77 | /* Mutex to serialize the shared buffer access and command handling. */ |
78 | static DEFINE_MUTEX(snp_cmd_mutex); |
79 | |
80 | static bool is_vmpck_empty(struct snp_guest_dev *snp_dev) |
81 | { |
82 | char zero_key[VMPCK_KEY_LEN] = {0}; |
83 | |
84 | if (snp_dev->vmpck) |
85 | return !memcmp(p: snp_dev->vmpck, q: zero_key, VMPCK_KEY_LEN); |
86 | |
87 | return true; |
88 | } |
89 | |
90 | /* |
91 | * If an error is received from the host or AMD Secure Processor (ASP) there |
92 | * are two options. Either retry the exact same encrypted request or discontinue |
93 | * using the VMPCK. |
94 | * |
95 | * This is because in the current encryption scheme GHCB v2 uses AES-GCM to |
96 | * encrypt the requests. The IV for this scheme is the sequence number. GCM |
97 | * cannot tolerate IV reuse. |
98 | * |
99 | * The ASP FW v1.51 only increments the sequence numbers on a successful |
100 | * guest<->ASP back and forth and only accepts messages at its exact sequence |
101 | * number. |
102 | * |
103 | * So if the sequence number were to be reused the encryption scheme is |
104 | * vulnerable. If the sequence number were incremented for a fresh IV the ASP |
105 | * will reject the request. |
106 | */ |
107 | static void snp_disable_vmpck(struct snp_guest_dev *snp_dev) |
108 | { |
109 | dev_alert(snp_dev->dev, "Disabling vmpck_id %d to prevent IV reuse.\n" , |
110 | vmpck_id); |
111 | memzero_explicit(s: snp_dev->vmpck, VMPCK_KEY_LEN); |
112 | snp_dev->vmpck = NULL; |
113 | } |
114 | |
115 | static inline u64 __snp_get_msg_seqno(struct snp_guest_dev *snp_dev) |
116 | { |
117 | u64 count; |
118 | |
119 | lockdep_assert_held(&snp_cmd_mutex); |
120 | |
121 | /* Read the current message sequence counter from secrets pages */ |
122 | count = *snp_dev->os_area_msg_seqno; |
123 | |
124 | return count + 1; |
125 | } |
126 | |
127 | /* Return a non-zero on success */ |
128 | static u64 snp_get_msg_seqno(struct snp_guest_dev *snp_dev) |
129 | { |
130 | u64 count = __snp_get_msg_seqno(snp_dev); |
131 | |
132 | /* |
133 | * The message sequence counter for the SNP guest request is a 64-bit |
134 | * value but the version 2 of GHCB specification defines a 32-bit storage |
135 | * for it. If the counter exceeds the 32-bit value then return zero. |
136 | * The caller should check the return value, but if the caller happens to |
137 | * not check the value and use it, then the firmware treats zero as an |
138 | * invalid number and will fail the message request. |
139 | */ |
140 | if (count >= UINT_MAX) { |
141 | dev_err(snp_dev->dev, "request message sequence counter overflow\n" ); |
142 | return 0; |
143 | } |
144 | |
145 | return count; |
146 | } |
147 | |
148 | static void snp_inc_msg_seqno(struct snp_guest_dev *snp_dev) |
149 | { |
150 | /* |
151 | * The counter is also incremented by the PSP, so increment it by 2 |
152 | * and save in secrets page. |
153 | */ |
154 | *snp_dev->os_area_msg_seqno += 2; |
155 | } |
156 | |
157 | static inline struct snp_guest_dev *to_snp_dev(struct file *file) |
158 | { |
159 | struct miscdevice *dev = file->private_data; |
160 | |
161 | return container_of(dev, struct snp_guest_dev, misc); |
162 | } |
163 | |
164 | static struct snp_guest_crypto *init_crypto(struct snp_guest_dev *snp_dev, u8 *key, size_t keylen) |
165 | { |
166 | struct snp_guest_crypto *crypto; |
167 | |
168 | crypto = kzalloc(size: sizeof(*crypto), GFP_KERNEL_ACCOUNT); |
169 | if (!crypto) |
170 | return NULL; |
171 | |
172 | crypto->tfm = crypto_alloc_aead(alg_name: "gcm(aes)" , type: 0, mask: 0); |
173 | if (IS_ERR(ptr: crypto->tfm)) |
174 | goto e_free; |
175 | |
176 | if (crypto_aead_setkey(tfm: crypto->tfm, key, keylen)) |
177 | goto e_free_crypto; |
178 | |
179 | crypto->iv_len = crypto_aead_ivsize(tfm: crypto->tfm); |
180 | crypto->iv = kmalloc(size: crypto->iv_len, GFP_KERNEL_ACCOUNT); |
181 | if (!crypto->iv) |
182 | goto e_free_crypto; |
183 | |
184 | if (crypto_aead_authsize(tfm: crypto->tfm) > MAX_AUTHTAG_LEN) { |
185 | if (crypto_aead_setauthsize(tfm: crypto->tfm, MAX_AUTHTAG_LEN)) { |
186 | dev_err(snp_dev->dev, "failed to set authsize to %d\n" , MAX_AUTHTAG_LEN); |
187 | goto e_free_iv; |
188 | } |
189 | } |
190 | |
191 | crypto->a_len = crypto_aead_authsize(tfm: crypto->tfm); |
192 | crypto->authtag = kmalloc(size: crypto->a_len, GFP_KERNEL_ACCOUNT); |
193 | if (!crypto->authtag) |
194 | goto e_free_iv; |
195 | |
196 | return crypto; |
197 | |
198 | e_free_iv: |
199 | kfree(objp: crypto->iv); |
200 | e_free_crypto: |
201 | crypto_free_aead(tfm: crypto->tfm); |
202 | e_free: |
203 | kfree(objp: crypto); |
204 | |
205 | return NULL; |
206 | } |
207 | |
208 | static void deinit_crypto(struct snp_guest_crypto *crypto) |
209 | { |
210 | crypto_free_aead(tfm: crypto->tfm); |
211 | kfree(objp: crypto->iv); |
212 | kfree(objp: crypto->authtag); |
213 | kfree(objp: crypto); |
214 | } |
215 | |
216 | static int enc_dec_message(struct snp_guest_crypto *crypto, struct snp_guest_msg *msg, |
217 | u8 *src_buf, u8 *dst_buf, size_t len, bool enc) |
218 | { |
219 | struct snp_guest_msg_hdr *hdr = &msg->hdr; |
220 | struct scatterlist src[3], dst[3]; |
221 | DECLARE_CRYPTO_WAIT(wait); |
222 | struct aead_request *req; |
223 | int ret; |
224 | |
225 | req = aead_request_alloc(tfm: crypto->tfm, GFP_KERNEL); |
226 | if (!req) |
227 | return -ENOMEM; |
228 | |
229 | /* |
230 | * AEAD memory operations: |
231 | * +------ AAD -------+------- DATA -----+---- AUTHTAG----+ |
232 | * | msg header | plaintext | hdr->authtag | |
233 | * | bytes 30h - 5Fh | or | | |
234 | * | | cipher | | |
235 | * +------------------+------------------+----------------+ |
236 | */ |
237 | sg_init_table(src, 3); |
238 | sg_set_buf(sg: &src[0], buf: &hdr->algo, AAD_LEN); |
239 | sg_set_buf(sg: &src[1], buf: src_buf, buflen: hdr->msg_sz); |
240 | sg_set_buf(sg: &src[2], buf: hdr->authtag, buflen: crypto->a_len); |
241 | |
242 | sg_init_table(dst, 3); |
243 | sg_set_buf(sg: &dst[0], buf: &hdr->algo, AAD_LEN); |
244 | sg_set_buf(sg: &dst[1], buf: dst_buf, buflen: hdr->msg_sz); |
245 | sg_set_buf(sg: &dst[2], buf: hdr->authtag, buflen: crypto->a_len); |
246 | |
247 | aead_request_set_ad(req, AAD_LEN); |
248 | aead_request_set_tfm(req, tfm: crypto->tfm); |
249 | aead_request_set_callback(req, flags: 0, compl: crypto_req_done, data: &wait); |
250 | |
251 | aead_request_set_crypt(req, src, dst, cryptlen: len, iv: crypto->iv); |
252 | ret = crypto_wait_req(err: enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req), wait: &wait); |
253 | |
254 | aead_request_free(req); |
255 | return ret; |
256 | } |
257 | |
258 | static int __enc_payload(struct snp_guest_dev *snp_dev, struct snp_guest_msg *msg, |
259 | void *plaintext, size_t len) |
260 | { |
261 | struct snp_guest_crypto *crypto = snp_dev->crypto; |
262 | struct snp_guest_msg_hdr *hdr = &msg->hdr; |
263 | |
264 | memset(crypto->iv, 0, crypto->iv_len); |
265 | memcpy(crypto->iv, &hdr->msg_seqno, sizeof(hdr->msg_seqno)); |
266 | |
267 | return enc_dec_message(crypto, msg, src_buf: plaintext, dst_buf: msg->payload, len, enc: true); |
268 | } |
269 | |
270 | static int dec_payload(struct snp_guest_dev *snp_dev, struct snp_guest_msg *msg, |
271 | void *plaintext, size_t len) |
272 | { |
273 | struct snp_guest_crypto *crypto = snp_dev->crypto; |
274 | struct snp_guest_msg_hdr *hdr = &msg->hdr; |
275 | |
276 | /* Build IV with response buffer sequence number */ |
277 | memset(crypto->iv, 0, crypto->iv_len); |
278 | memcpy(crypto->iv, &hdr->msg_seqno, sizeof(hdr->msg_seqno)); |
279 | |
280 | return enc_dec_message(crypto, msg, src_buf: msg->payload, dst_buf: plaintext, len, enc: false); |
281 | } |
282 | |
283 | static int verify_and_dec_payload(struct snp_guest_dev *snp_dev, void *payload, u32 sz) |
284 | { |
285 | struct snp_guest_crypto *crypto = snp_dev->crypto; |
286 | struct snp_guest_msg *resp = &snp_dev->secret_response; |
287 | struct snp_guest_msg *req = &snp_dev->secret_request; |
288 | struct snp_guest_msg_hdr *req_hdr = &req->hdr; |
289 | struct snp_guest_msg_hdr *resp_hdr = &resp->hdr; |
290 | |
291 | dev_dbg(snp_dev->dev, "response [seqno %lld type %d version %d sz %d]\n" , |
292 | resp_hdr->msg_seqno, resp_hdr->msg_type, resp_hdr->msg_version, resp_hdr->msg_sz); |
293 | |
294 | /* Copy response from shared memory to encrypted memory. */ |
295 | memcpy(resp, snp_dev->response, sizeof(*resp)); |
296 | |
297 | /* Verify that the sequence counter is incremented by 1 */ |
298 | if (unlikely(resp_hdr->msg_seqno != (req_hdr->msg_seqno + 1))) |
299 | return -EBADMSG; |
300 | |
301 | /* Verify response message type and version number. */ |
302 | if (resp_hdr->msg_type != (req_hdr->msg_type + 1) || |
303 | resp_hdr->msg_version != req_hdr->msg_version) |
304 | return -EBADMSG; |
305 | |
306 | /* |
307 | * If the message size is greater than our buffer length then return |
308 | * an error. |
309 | */ |
310 | if (unlikely((resp_hdr->msg_sz + crypto->a_len) > sz)) |
311 | return -EBADMSG; |
312 | |
313 | /* Decrypt the payload */ |
314 | return dec_payload(snp_dev, msg: resp, plaintext: payload, len: resp_hdr->msg_sz + crypto->a_len); |
315 | } |
316 | |
317 | static int enc_payload(struct snp_guest_dev *snp_dev, u64 seqno, int version, u8 type, |
318 | void *payload, size_t sz) |
319 | { |
320 | struct snp_guest_msg *req = &snp_dev->secret_request; |
321 | struct snp_guest_msg_hdr *hdr = &req->hdr; |
322 | |
323 | memset(req, 0, sizeof(*req)); |
324 | |
325 | hdr->algo = SNP_AEAD_AES_256_GCM; |
326 | hdr->hdr_version = MSG_HDR_VER; |
327 | hdr->hdr_sz = sizeof(*hdr); |
328 | hdr->msg_type = type; |
329 | hdr->msg_version = version; |
330 | hdr->msg_seqno = seqno; |
331 | hdr->msg_vmpck = vmpck_id; |
332 | hdr->msg_sz = sz; |
333 | |
334 | /* Verify the sequence number is non-zero */ |
335 | if (!hdr->msg_seqno) |
336 | return -ENOSR; |
337 | |
338 | dev_dbg(snp_dev->dev, "request [seqno %lld type %d version %d sz %d]\n" , |
339 | hdr->msg_seqno, hdr->msg_type, hdr->msg_version, hdr->msg_sz); |
340 | |
341 | return __enc_payload(snp_dev, msg: req, plaintext: payload, len: sz); |
342 | } |
343 | |
344 | static int __handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code, |
345 | struct snp_guest_request_ioctl *rio) |
346 | { |
347 | unsigned long req_start = jiffies; |
348 | unsigned int override_npages = 0; |
349 | u64 override_err = 0; |
350 | int rc; |
351 | |
352 | retry_request: |
353 | /* |
354 | * Call firmware to process the request. In this function the encrypted |
355 | * message enters shared memory with the host. So after this call the |
356 | * sequence number must be incremented or the VMPCK must be deleted to |
357 | * prevent reuse of the IV. |
358 | */ |
359 | rc = snp_issue_guest_request(exit_code, input: &snp_dev->input, rio); |
360 | switch (rc) { |
361 | case -ENOSPC: |
362 | /* |
363 | * If the extended guest request fails due to having too |
364 | * small of a certificate data buffer, retry the same |
365 | * guest request without the extended data request in |
366 | * order to increment the sequence number and thus avoid |
367 | * IV reuse. |
368 | */ |
369 | override_npages = snp_dev->input.data_npages; |
370 | exit_code = SVM_VMGEXIT_GUEST_REQUEST; |
371 | |
372 | /* |
373 | * Override the error to inform callers the given extended |
374 | * request buffer size was too small and give the caller the |
375 | * required buffer size. |
376 | */ |
377 | override_err = SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN); |
378 | |
379 | /* |
380 | * If this call to the firmware succeeds, the sequence number can |
381 | * be incremented allowing for continued use of the VMPCK. If |
382 | * there is an error reflected in the return value, this value |
383 | * is checked further down and the result will be the deletion |
384 | * of the VMPCK and the error code being propagated back to the |
385 | * user as an ioctl() return code. |
386 | */ |
387 | goto retry_request; |
388 | |
389 | /* |
390 | * The host may return SNP_GUEST_VMM_ERR_BUSY if the request has been |
391 | * throttled. Retry in the driver to avoid returning and reusing the |
392 | * message sequence number on a different message. |
393 | */ |
394 | case -EAGAIN: |
395 | if (jiffies - req_start > SNP_REQ_MAX_RETRY_DURATION) { |
396 | rc = -ETIMEDOUT; |
397 | break; |
398 | } |
399 | schedule_timeout_killable(SNP_REQ_RETRY_DELAY); |
400 | goto retry_request; |
401 | } |
402 | |
403 | /* |
404 | * Increment the message sequence number. There is no harm in doing |
405 | * this now because decryption uses the value stored in the response |
406 | * structure and any failure will wipe the VMPCK, preventing further |
407 | * use anyway. |
408 | */ |
409 | snp_inc_msg_seqno(snp_dev); |
410 | |
411 | if (override_err) { |
412 | rio->exitinfo2 = override_err; |
413 | |
414 | /* |
415 | * If an extended guest request was issued and the supplied certificate |
416 | * buffer was not large enough, a standard guest request was issued to |
417 | * prevent IV reuse. If the standard request was successful, return -EIO |
418 | * back to the caller as would have originally been returned. |
419 | */ |
420 | if (!rc && override_err == SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN)) |
421 | rc = -EIO; |
422 | } |
423 | |
424 | if (override_npages) |
425 | snp_dev->input.data_npages = override_npages; |
426 | |
427 | return rc; |
428 | } |
429 | |
430 | static int handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code, |
431 | struct snp_guest_request_ioctl *rio, u8 type, |
432 | void *req_buf, size_t req_sz, void *resp_buf, |
433 | u32 resp_sz) |
434 | { |
435 | u64 seqno; |
436 | int rc; |
437 | |
438 | /* Get message sequence and verify that its a non-zero */ |
439 | seqno = snp_get_msg_seqno(snp_dev); |
440 | if (!seqno) |
441 | return -EIO; |
442 | |
443 | /* Clear shared memory's response for the host to populate. */ |
444 | memset(snp_dev->response, 0, sizeof(struct snp_guest_msg)); |
445 | |
446 | /* Encrypt the userspace provided payload in snp_dev->secret_request. */ |
447 | rc = enc_payload(snp_dev, seqno, version: rio->msg_version, type, payload: req_buf, sz: req_sz); |
448 | if (rc) |
449 | return rc; |
450 | |
451 | /* |
452 | * Write the fully encrypted request to the shared unencrypted |
453 | * request page. |
454 | */ |
455 | memcpy(snp_dev->request, &snp_dev->secret_request, |
456 | sizeof(snp_dev->secret_request)); |
457 | |
458 | rc = __handle_guest_request(snp_dev, exit_code, rio); |
459 | if (rc) { |
460 | if (rc == -EIO && |
461 | rio->exitinfo2 == SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN)) |
462 | return rc; |
463 | |
464 | dev_alert(snp_dev->dev, |
465 | "Detected error from ASP request. rc: %d, exitinfo2: 0x%llx\n" , |
466 | rc, rio->exitinfo2); |
467 | |
468 | snp_disable_vmpck(snp_dev); |
469 | return rc; |
470 | } |
471 | |
472 | rc = verify_and_dec_payload(snp_dev, payload: resp_buf, sz: resp_sz); |
473 | if (rc) { |
474 | dev_alert(snp_dev->dev, "Detected unexpected decode failure from ASP. rc: %d\n" , rc); |
475 | snp_disable_vmpck(snp_dev); |
476 | return rc; |
477 | } |
478 | |
479 | return 0; |
480 | } |
481 | |
482 | struct snp_req_resp { |
483 | sockptr_t req_data; |
484 | sockptr_t resp_data; |
485 | }; |
486 | |
487 | static int get_report(struct snp_guest_dev *snp_dev, struct snp_guest_request_ioctl *arg) |
488 | { |
489 | struct snp_guest_crypto *crypto = snp_dev->crypto; |
490 | struct snp_report_req *req = &snp_dev->req.report; |
491 | struct snp_report_resp *resp; |
492 | int rc, resp_len; |
493 | |
494 | lockdep_assert_held(&snp_cmd_mutex); |
495 | |
496 | if (!arg->req_data || !arg->resp_data) |
497 | return -EINVAL; |
498 | |
499 | if (copy_from_user(to: req, from: (void __user *)arg->req_data, n: sizeof(*req))) |
500 | return -EFAULT; |
501 | |
502 | /* |
503 | * The intermediate response buffer is used while decrypting the |
504 | * response payload. Make sure that it has enough space to cover the |
505 | * authtag. |
506 | */ |
507 | resp_len = sizeof(resp->data) + crypto->a_len; |
508 | resp = kzalloc(size: resp_len, GFP_KERNEL_ACCOUNT); |
509 | if (!resp) |
510 | return -ENOMEM; |
511 | |
512 | rc = handle_guest_request(snp_dev, SVM_VMGEXIT_GUEST_REQUEST, rio: arg, |
513 | type: SNP_MSG_REPORT_REQ, req_buf: req, req_sz: sizeof(*req), resp_buf: resp->data, |
514 | resp_sz: resp_len); |
515 | if (rc) |
516 | goto e_free; |
517 | |
518 | if (copy_to_user(to: (void __user *)arg->resp_data, from: resp, n: sizeof(*resp))) |
519 | rc = -EFAULT; |
520 | |
521 | e_free: |
522 | kfree(objp: resp); |
523 | return rc; |
524 | } |
525 | |
526 | static int get_derived_key(struct snp_guest_dev *snp_dev, struct snp_guest_request_ioctl *arg) |
527 | { |
528 | struct snp_derived_key_req *req = &snp_dev->req.derived_key; |
529 | struct snp_guest_crypto *crypto = snp_dev->crypto; |
530 | struct snp_derived_key_resp resp = {0}; |
531 | int rc, resp_len; |
532 | /* Response data is 64 bytes and max authsize for GCM is 16 bytes. */ |
533 | u8 buf[64 + 16]; |
534 | |
535 | lockdep_assert_held(&snp_cmd_mutex); |
536 | |
537 | if (!arg->req_data || !arg->resp_data) |
538 | return -EINVAL; |
539 | |
540 | /* |
541 | * The intermediate response buffer is used while decrypting the |
542 | * response payload. Make sure that it has enough space to cover the |
543 | * authtag. |
544 | */ |
545 | resp_len = sizeof(resp.data) + crypto->a_len; |
546 | if (sizeof(buf) < resp_len) |
547 | return -ENOMEM; |
548 | |
549 | if (copy_from_user(to: req, from: (void __user *)arg->req_data, n: sizeof(*req))) |
550 | return -EFAULT; |
551 | |
552 | rc = handle_guest_request(snp_dev, SVM_VMGEXIT_GUEST_REQUEST, rio: arg, |
553 | type: SNP_MSG_KEY_REQ, req_buf: req, req_sz: sizeof(*req), resp_buf: buf, resp_sz: resp_len); |
554 | if (rc) |
555 | return rc; |
556 | |
557 | memcpy(resp.data, buf, sizeof(resp.data)); |
558 | if (copy_to_user(to: (void __user *)arg->resp_data, from: &resp, n: sizeof(resp))) |
559 | rc = -EFAULT; |
560 | |
561 | /* The response buffer contains the sensitive data, explicitly clear it. */ |
562 | memzero_explicit(s: buf, count: sizeof(buf)); |
563 | memzero_explicit(s: &resp, count: sizeof(resp)); |
564 | return rc; |
565 | } |
566 | |
567 | static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_request_ioctl *arg, |
568 | struct snp_req_resp *io) |
569 | |
570 | { |
571 | struct snp_ext_report_req *req = &snp_dev->req.ext_report; |
572 | struct snp_guest_crypto *crypto = snp_dev->crypto; |
573 | struct snp_report_resp *resp; |
574 | int ret, npages = 0, resp_len; |
575 | sockptr_t certs_address; |
576 | |
577 | lockdep_assert_held(&snp_cmd_mutex); |
578 | |
579 | if (sockptr_is_null(sockptr: io->req_data) || sockptr_is_null(sockptr: io->resp_data)) |
580 | return -EINVAL; |
581 | |
582 | if (copy_from_sockptr(dst: req, src: io->req_data, size: sizeof(*req))) |
583 | return -EFAULT; |
584 | |
585 | /* caller does not want certificate data */ |
586 | if (!req->certs_len || !req->certs_address) |
587 | goto cmd; |
588 | |
589 | if (req->certs_len > SEV_FW_BLOB_MAX_SIZE || |
590 | !IS_ALIGNED(req->certs_len, PAGE_SIZE)) |
591 | return -EINVAL; |
592 | |
593 | if (sockptr_is_kernel(sockptr: io->resp_data)) { |
594 | certs_address = KERNEL_SOCKPTR(p: (void *)req->certs_address); |
595 | } else { |
596 | certs_address = USER_SOCKPTR(p: (void __user *)req->certs_address); |
597 | if (!access_ok(certs_address.user, req->certs_len)) |
598 | return -EFAULT; |
599 | } |
600 | |
601 | /* |
602 | * Initialize the intermediate buffer with all zeros. This buffer |
603 | * is used in the guest request message to get the certs blob from |
604 | * the host. If host does not supply any certs in it, then copy |
605 | * zeros to indicate that certificate data was not provided. |
606 | */ |
607 | memset(snp_dev->certs_data, 0, req->certs_len); |
608 | npages = req->certs_len >> PAGE_SHIFT; |
609 | cmd: |
610 | /* |
611 | * The intermediate response buffer is used while decrypting the |
612 | * response payload. Make sure that it has enough space to cover the |
613 | * authtag. |
614 | */ |
615 | resp_len = sizeof(resp->data) + crypto->a_len; |
616 | resp = kzalloc(size: resp_len, GFP_KERNEL_ACCOUNT); |
617 | if (!resp) |
618 | return -ENOMEM; |
619 | |
620 | snp_dev->input.data_npages = npages; |
621 | ret = handle_guest_request(snp_dev, SVM_VMGEXIT_EXT_GUEST_REQUEST, rio: arg, |
622 | type: SNP_MSG_REPORT_REQ, req_buf: &req->data, |
623 | req_sz: sizeof(req->data), resp_buf: resp->data, resp_sz: resp_len); |
624 | |
625 | /* If certs length is invalid then copy the returned length */ |
626 | if (arg->vmm_error == SNP_GUEST_VMM_ERR_INVALID_LEN) { |
627 | req->certs_len = snp_dev->input.data_npages << PAGE_SHIFT; |
628 | |
629 | if (copy_to_sockptr(dst: io->req_data, src: req, size: sizeof(*req))) |
630 | ret = -EFAULT; |
631 | } |
632 | |
633 | if (ret) |
634 | goto e_free; |
635 | |
636 | if (npages && copy_to_sockptr(dst: certs_address, src: snp_dev->certs_data, size: req->certs_len)) { |
637 | ret = -EFAULT; |
638 | goto e_free; |
639 | } |
640 | |
641 | if (copy_to_sockptr(dst: io->resp_data, src: resp, size: sizeof(*resp))) |
642 | ret = -EFAULT; |
643 | |
644 | e_free: |
645 | kfree(objp: resp); |
646 | return ret; |
647 | } |
648 | |
649 | static long snp_guest_ioctl(struct file *file, unsigned int ioctl, unsigned long arg) |
650 | { |
651 | struct snp_guest_dev *snp_dev = to_snp_dev(file); |
652 | void __user *argp = (void __user *)arg; |
653 | struct snp_guest_request_ioctl input; |
654 | struct snp_req_resp io; |
655 | int ret = -ENOTTY; |
656 | |
657 | if (copy_from_user(to: &input, from: argp, n: sizeof(input))) |
658 | return -EFAULT; |
659 | |
660 | input.exitinfo2 = 0xff; |
661 | |
662 | /* Message version must be non-zero */ |
663 | if (!input.msg_version) |
664 | return -EINVAL; |
665 | |
666 | mutex_lock(&snp_cmd_mutex); |
667 | |
668 | /* Check if the VMPCK is not empty */ |
669 | if (is_vmpck_empty(snp_dev)) { |
670 | dev_err_ratelimited(snp_dev->dev, "VMPCK is disabled\n" ); |
671 | mutex_unlock(lock: &snp_cmd_mutex); |
672 | return -ENOTTY; |
673 | } |
674 | |
675 | switch (ioctl) { |
676 | case SNP_GET_REPORT: |
677 | ret = get_report(snp_dev, arg: &input); |
678 | break; |
679 | case SNP_GET_DERIVED_KEY: |
680 | ret = get_derived_key(snp_dev, arg: &input); |
681 | break; |
682 | case SNP_GET_EXT_REPORT: |
683 | /* |
684 | * As get_ext_report() may be called from the ioctl() path and a |
685 | * kernel internal path (configfs-tsm), decorate the passed |
686 | * buffers as user pointers. |
687 | */ |
688 | io.req_data = USER_SOCKPTR(p: (void __user *)input.req_data); |
689 | io.resp_data = USER_SOCKPTR(p: (void __user *)input.resp_data); |
690 | ret = get_ext_report(snp_dev, arg: &input, io: &io); |
691 | break; |
692 | default: |
693 | break; |
694 | } |
695 | |
696 | mutex_unlock(lock: &snp_cmd_mutex); |
697 | |
698 | if (input.exitinfo2 && copy_to_user(to: argp, from: &input, n: sizeof(input))) |
699 | return -EFAULT; |
700 | |
701 | return ret; |
702 | } |
703 | |
704 | static void free_shared_pages(void *buf, size_t sz) |
705 | { |
706 | unsigned int npages = PAGE_ALIGN(sz) >> PAGE_SHIFT; |
707 | int ret; |
708 | |
709 | if (!buf) |
710 | return; |
711 | |
712 | ret = set_memory_encrypted(addr: (unsigned long)buf, numpages: npages); |
713 | if (ret) { |
714 | WARN_ONCE(ret, "failed to restore encryption mask (leak it)\n" ); |
715 | return; |
716 | } |
717 | |
718 | __free_pages(virt_to_page(buf), order: get_order(size: sz)); |
719 | } |
720 | |
721 | static void *alloc_shared_pages(struct device *dev, size_t sz) |
722 | { |
723 | unsigned int npages = PAGE_ALIGN(sz) >> PAGE_SHIFT; |
724 | struct page *page; |
725 | int ret; |
726 | |
727 | page = alloc_pages(GFP_KERNEL_ACCOUNT, order: get_order(size: sz)); |
728 | if (!page) |
729 | return NULL; |
730 | |
731 | ret = set_memory_decrypted(addr: (unsigned long)page_address(page), numpages: npages); |
732 | if (ret) { |
733 | dev_err(dev, "failed to mark page shared, ret=%d\n" , ret); |
734 | __free_pages(page, order: get_order(size: sz)); |
735 | return NULL; |
736 | } |
737 | |
738 | return page_address(page); |
739 | } |
740 | |
741 | static const struct file_operations snp_guest_fops = { |
742 | .owner = THIS_MODULE, |
743 | .unlocked_ioctl = snp_guest_ioctl, |
744 | }; |
745 | |
746 | static u8 *get_vmpck(int id, struct snp_secrets_page_layout *layout, u32 **seqno) |
747 | { |
748 | u8 *key = NULL; |
749 | |
750 | switch (id) { |
751 | case 0: |
752 | *seqno = &layout->os_area.msg_seqno_0; |
753 | key = layout->vmpck0; |
754 | break; |
755 | case 1: |
756 | *seqno = &layout->os_area.msg_seqno_1; |
757 | key = layout->vmpck1; |
758 | break; |
759 | case 2: |
760 | *seqno = &layout->os_area.msg_seqno_2; |
761 | key = layout->vmpck2; |
762 | break; |
763 | case 3: |
764 | *seqno = &layout->os_area.msg_seqno_3; |
765 | key = layout->vmpck3; |
766 | break; |
767 | default: |
768 | break; |
769 | } |
770 | |
771 | return key; |
772 | } |
773 | |
774 | struct snp_msg_report_resp_hdr { |
775 | u32 status; |
776 | u32 report_size; |
777 | u8 rsvd[24]; |
778 | }; |
779 | |
780 | struct snp_msg_cert_entry { |
781 | guid_t guid; |
782 | u32 offset; |
783 | u32 length; |
784 | }; |
785 | |
786 | static int sev_report_new(struct tsm_report *report, void *data) |
787 | { |
788 | struct snp_msg_cert_entry *cert_table; |
789 | struct tsm_desc *desc = &report->desc; |
790 | struct snp_guest_dev *snp_dev = data; |
791 | struct snp_msg_report_resp_hdr hdr; |
792 | const u32 report_size = SZ_4K; |
793 | const u32 ext_size = SEV_FW_BLOB_MAX_SIZE; |
794 | u32 certs_size, i, size = report_size + ext_size; |
795 | int ret; |
796 | |
797 | if (desc->inblob_len != SNP_REPORT_USER_DATA_SIZE) |
798 | return -EINVAL; |
799 | |
800 | void *buf __free(kvfree) = kvzalloc(size, GFP_KERNEL); |
801 | if (!buf) |
802 | return -ENOMEM; |
803 | |
804 | guard(mutex)(T: &snp_cmd_mutex); |
805 | |
806 | /* Check if the VMPCK is not empty */ |
807 | if (is_vmpck_empty(snp_dev)) { |
808 | dev_err_ratelimited(snp_dev->dev, "VMPCK is disabled\n" ); |
809 | return -ENOTTY; |
810 | } |
811 | |
812 | cert_table = buf + report_size; |
813 | struct snp_ext_report_req ext_req = { |
814 | .data = { .vmpl = desc->privlevel }, |
815 | .certs_address = (__u64)cert_table, |
816 | .certs_len = ext_size, |
817 | }; |
818 | memcpy(&ext_req.data.user_data, desc->inblob, desc->inblob_len); |
819 | |
820 | struct snp_guest_request_ioctl input = { |
821 | .msg_version = 1, |
822 | .req_data = (__u64)&ext_req, |
823 | .resp_data = (__u64)buf, |
824 | .exitinfo2 = 0xff, |
825 | }; |
826 | struct snp_req_resp io = { |
827 | .req_data = KERNEL_SOCKPTR(p: &ext_req), |
828 | .resp_data = KERNEL_SOCKPTR(p: buf), |
829 | }; |
830 | |
831 | ret = get_ext_report(snp_dev, arg: &input, io: &io); |
832 | if (ret) |
833 | return ret; |
834 | |
835 | memcpy(&hdr, buf, sizeof(hdr)); |
836 | if (hdr.status == SEV_RET_INVALID_PARAM) |
837 | return -EINVAL; |
838 | if (hdr.status == SEV_RET_INVALID_KEY) |
839 | return -EINVAL; |
840 | if (hdr.status) |
841 | return -ENXIO; |
842 | if ((hdr.report_size + sizeof(hdr)) > report_size) |
843 | return -ENOMEM; |
844 | |
845 | void *rbuf __free(kvfree) = kvzalloc(size: hdr.report_size, GFP_KERNEL); |
846 | if (!rbuf) |
847 | return -ENOMEM; |
848 | |
849 | memcpy(rbuf, buf + sizeof(hdr), hdr.report_size); |
850 | report->outblob = no_free_ptr(rbuf); |
851 | report->outblob_len = hdr.report_size; |
852 | |
853 | certs_size = 0; |
854 | for (i = 0; i < ext_size / sizeof(struct snp_msg_cert_entry); i++) { |
855 | struct snp_msg_cert_entry *ent = &cert_table[i]; |
856 | |
857 | if (guid_is_null(guid: &ent->guid) && !ent->offset && !ent->length) |
858 | break; |
859 | certs_size = max(certs_size, ent->offset + ent->length); |
860 | } |
861 | |
862 | /* Suspicious that the response populated entries without populating size */ |
863 | if (!certs_size && i) |
864 | dev_warn_ratelimited(snp_dev->dev, "certificate slots conveyed without size\n" ); |
865 | |
866 | /* No certs to report */ |
867 | if (!certs_size) |
868 | return 0; |
869 | |
870 | /* Suspicious that the certificate blob size contract was violated |
871 | */ |
872 | if (certs_size > ext_size) { |
873 | dev_warn_ratelimited(snp_dev->dev, "certificate data truncated\n" ); |
874 | certs_size = ext_size; |
875 | } |
876 | |
877 | void *cbuf __free(kvfree) = kvzalloc(size: certs_size, GFP_KERNEL); |
878 | if (!cbuf) |
879 | return -ENOMEM; |
880 | |
881 | memcpy(cbuf, cert_table, certs_size); |
882 | report->auxblob = no_free_ptr(cbuf); |
883 | report->auxblob_len = certs_size; |
884 | |
885 | return 0; |
886 | } |
887 | |
888 | static const struct tsm_ops sev_tsm_ops = { |
889 | .name = KBUILD_MODNAME, |
890 | .report_new = sev_report_new, |
891 | }; |
892 | |
893 | static void unregister_sev_tsm(void *data) |
894 | { |
895 | tsm_unregister(ops: &sev_tsm_ops); |
896 | } |
897 | |
898 | static int __init sev_guest_probe(struct platform_device *pdev) |
899 | { |
900 | struct snp_secrets_page_layout *layout; |
901 | struct sev_guest_platform_data *data; |
902 | struct device *dev = &pdev->dev; |
903 | struct snp_guest_dev *snp_dev; |
904 | struct miscdevice *misc; |
905 | void __iomem *mapping; |
906 | int ret; |
907 | |
908 | if (!cc_platform_has(attr: CC_ATTR_GUEST_SEV_SNP)) |
909 | return -ENODEV; |
910 | |
911 | if (!dev->platform_data) |
912 | return -ENODEV; |
913 | |
914 | data = (struct sev_guest_platform_data *)dev->platform_data; |
915 | mapping = ioremap_encrypted(phys_addr: data->secrets_gpa, PAGE_SIZE); |
916 | if (!mapping) |
917 | return -ENODEV; |
918 | |
919 | layout = (__force void *)mapping; |
920 | |
921 | ret = -ENOMEM; |
922 | snp_dev = devm_kzalloc(dev: &pdev->dev, size: sizeof(struct snp_guest_dev), GFP_KERNEL); |
923 | if (!snp_dev) |
924 | goto e_unmap; |
925 | |
926 | ret = -EINVAL; |
927 | snp_dev->vmpck = get_vmpck(id: vmpck_id, layout, seqno: &snp_dev->os_area_msg_seqno); |
928 | if (!snp_dev->vmpck) { |
929 | dev_err(dev, "invalid vmpck id %d\n" , vmpck_id); |
930 | goto e_unmap; |
931 | } |
932 | |
933 | /* Verify that VMPCK is not zero. */ |
934 | if (is_vmpck_empty(snp_dev)) { |
935 | dev_err(dev, "vmpck id %d is null\n" , vmpck_id); |
936 | goto e_unmap; |
937 | } |
938 | |
939 | platform_set_drvdata(pdev, data: snp_dev); |
940 | snp_dev->dev = dev; |
941 | snp_dev->layout = layout; |
942 | |
943 | /* Allocate the shared page used for the request and response message. */ |
944 | snp_dev->request = alloc_shared_pages(dev, sz: sizeof(struct snp_guest_msg)); |
945 | if (!snp_dev->request) |
946 | goto e_unmap; |
947 | |
948 | snp_dev->response = alloc_shared_pages(dev, sz: sizeof(struct snp_guest_msg)); |
949 | if (!snp_dev->response) |
950 | goto e_free_request; |
951 | |
952 | snp_dev->certs_data = alloc_shared_pages(dev, SEV_FW_BLOB_MAX_SIZE); |
953 | if (!snp_dev->certs_data) |
954 | goto e_free_response; |
955 | |
956 | ret = -EIO; |
957 | snp_dev->crypto = init_crypto(snp_dev, key: snp_dev->vmpck, VMPCK_KEY_LEN); |
958 | if (!snp_dev->crypto) |
959 | goto e_free_cert_data; |
960 | |
961 | misc = &snp_dev->misc; |
962 | misc->minor = MISC_DYNAMIC_MINOR; |
963 | misc->name = DEVICE_NAME; |
964 | misc->fops = &snp_guest_fops; |
965 | |
966 | /* initial the input address for guest request */ |
967 | snp_dev->input.req_gpa = __pa(snp_dev->request); |
968 | snp_dev->input.resp_gpa = __pa(snp_dev->response); |
969 | snp_dev->input.data_gpa = __pa(snp_dev->certs_data); |
970 | |
971 | ret = tsm_register(ops: &sev_tsm_ops, priv: snp_dev, type: &tsm_report_extra_type); |
972 | if (ret) |
973 | goto e_free_cert_data; |
974 | |
975 | ret = devm_add_action_or_reset(&pdev->dev, unregister_sev_tsm, NULL); |
976 | if (ret) |
977 | goto e_free_cert_data; |
978 | |
979 | ret = misc_register(misc); |
980 | if (ret) |
981 | goto e_free_cert_data; |
982 | |
983 | dev_info(dev, "Initialized SEV guest driver (using vmpck_id %d)\n" , vmpck_id); |
984 | return 0; |
985 | |
986 | e_free_cert_data: |
987 | free_shared_pages(buf: snp_dev->certs_data, SEV_FW_BLOB_MAX_SIZE); |
988 | e_free_response: |
989 | free_shared_pages(buf: snp_dev->response, sz: sizeof(struct snp_guest_msg)); |
990 | e_free_request: |
991 | free_shared_pages(buf: snp_dev->request, sz: sizeof(struct snp_guest_msg)); |
992 | e_unmap: |
993 | iounmap(addr: mapping); |
994 | return ret; |
995 | } |
996 | |
997 | static void __exit sev_guest_remove(struct platform_device *pdev) |
998 | { |
999 | struct snp_guest_dev *snp_dev = platform_get_drvdata(pdev); |
1000 | |
1001 | free_shared_pages(buf: snp_dev->certs_data, SEV_FW_BLOB_MAX_SIZE); |
1002 | free_shared_pages(buf: snp_dev->response, sz: sizeof(struct snp_guest_msg)); |
1003 | free_shared_pages(buf: snp_dev->request, sz: sizeof(struct snp_guest_msg)); |
1004 | deinit_crypto(crypto: snp_dev->crypto); |
1005 | misc_deregister(misc: &snp_dev->misc); |
1006 | } |
1007 | |
1008 | /* |
1009 | * This driver is meant to be a common SEV guest interface driver and to |
1010 | * support any SEV guest API. As such, even though it has been introduced |
1011 | * with the SEV-SNP support, it is named "sev-guest". |
1012 | */ |
1013 | static struct platform_driver sev_guest_driver = { |
1014 | .remove_new = __exit_p(sev_guest_remove), |
1015 | .driver = { |
1016 | .name = "sev-guest" , |
1017 | }, |
1018 | }; |
1019 | |
1020 | module_platform_driver_probe(sev_guest_driver, sev_guest_probe); |
1021 | |
1022 | MODULE_AUTHOR("Brijesh Singh <brijesh.singh@amd.com>" ); |
1023 | MODULE_LICENSE("GPL" ); |
1024 | MODULE_VERSION("1.0.0" ); |
1025 | MODULE_DESCRIPTION("AMD SEV Guest Driver" ); |
1026 | MODULE_ALIAS("platform:sev-guest" ); |
1027 | |