1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */
3
4#include <linux/jump_label.h>
5#include <linux/uaccess.h>
6#include <linux/export.h>
7#include <linux/string.h>
8#include <linux/types.h>
9
10#include <asm/mce.h>
11
12#ifdef CONFIG_X86_MCE
13static DEFINE_STATIC_KEY_FALSE(copy_mc_fragile_key);
14
15void enable_copy_mc_fragile(void)
16{
17 static_branch_inc(&copy_mc_fragile_key);
18}
19#define copy_mc_fragile_enabled (static_branch_unlikely(&copy_mc_fragile_key))
20
21/*
22 * Similar to copy_user_handle_tail, probe for the write fault point, or
23 * source exception point.
24 */
25__visible notrace unsigned long
26copy_mc_fragile_handle_tail(char *to, char *from, unsigned len)
27{
28 for (; len; --len, to++, from++)
29 if (copy_mc_fragile(dst: to, src: from, cnt: 1))
30 break;
31 return len;
32}
33#else
34/*
35 * No point in doing careful copying, or consulting a static key when
36 * there is no #MC handler in the CONFIG_X86_MCE=n case.
37 */
38void enable_copy_mc_fragile(void)
39{
40}
41#define copy_mc_fragile_enabled (0)
42#endif
43
44unsigned long copy_mc_enhanced_fast_string(void *dst, const void *src, unsigned len);
45
46/**
47 * copy_mc_to_kernel - memory copy that handles source exceptions
48 *
49 * @dst: destination address
50 * @src: source address
51 * @len: number of bytes to copy
52 *
53 * Call into the 'fragile' version on systems that benefit from avoiding
54 * corner case poison consumption scenarios, For example, accessing
55 * poison across 2 cachelines with a single instruction. Almost all
56 * other uses case can use copy_mc_enhanced_fast_string() for a fast
57 * recoverable copy, or fallback to plain memcpy.
58 *
59 * Return 0 for success, or number of bytes not copied if there was an
60 * exception.
61 */
62unsigned long __must_check copy_mc_to_kernel(void *dst, const void *src, unsigned len)
63{
64 if (copy_mc_fragile_enabled)
65 return copy_mc_fragile(dst, src, cnt: len);
66 if (static_cpu_has(X86_FEATURE_ERMS))
67 return copy_mc_enhanced_fast_string(dst, src, len);
68 memcpy(dst, src, len);
69 return 0;
70}
71EXPORT_SYMBOL_GPL(copy_mc_to_kernel);
72
73unsigned long __must_check copy_mc_to_user(void __user *dst, const void *src, unsigned len)
74{
75 unsigned long ret;
76
77 if (copy_mc_fragile_enabled) {
78 __uaccess_begin();
79 ret = copy_mc_fragile(dst: (__force void *)dst, src, cnt: len);
80 __uaccess_end();
81 return ret;
82 }
83
84 if (static_cpu_has(X86_FEATURE_ERMS)) {
85 __uaccess_begin();
86 ret = copy_mc_enhanced_fast_string(dst: (__force void *)dst, src, len);
87 __uaccess_end();
88 return ret;
89 }
90
91 return copy_user_generic(to: (__force void *)dst, from: src, len);
92}
93

source code of linux/arch/x86/lib/copy_mc.c