1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* |
3 | * String handling functions. |
4 | * |
5 | * Copyright IBM Corp. 2012 |
6 | */ |
7 | |
8 | #include <linux/export.h> |
9 | #include <linux/linkage.h> |
10 | #include <asm/nospec-insn.h> |
11 | |
12 | GEN_BR_THUNK %r14 |
13 | |
14 | /* |
15 | * void *memmove(void *dest, const void *src, size_t n) |
16 | */ |
17 | SYM_FUNC_START(__memmove) |
18 | ltgr %r4,%r4 |
19 | lgr %r1,%r2 |
20 | jz .Lmemmove_exit |
21 | aghi %r4,-1 |
22 | clgr %r2,%r3 |
23 | jnh .Lmemmove_forward |
24 | la %r5,1(%r4,%r3) |
25 | clgr %r2,%r5 |
26 | jl .Lmemmove_reverse |
27 | .Lmemmove_forward: |
28 | srlg %r0,%r4,8 |
29 | ltgr %r0,%r0 |
30 | jz .Lmemmove_forward_remainder |
31 | .Lmemmove_forward_loop: |
32 | mvc 0(256,%r1),0(%r3) |
33 | la %r1,256(%r1) |
34 | la %r3,256(%r3) |
35 | brctg %r0,.Lmemmove_forward_loop |
36 | .Lmemmove_forward_remainder: |
37 | larl %r5,.Lmemmove_mvc |
38 | ex %r4,0(%r5) |
39 | .Lmemmove_exit: |
40 | BR_EX %r14 |
41 | .Lmemmove_reverse: |
42 | ic %r0,0(%r4,%r3) |
43 | stc %r0,0(%r4,%r1) |
44 | brctg %r4,.Lmemmove_reverse |
45 | ic %r0,0(%r4,%r3) |
46 | stc %r0,0(%r4,%r1) |
47 | BR_EX %r14 |
48 | .Lmemmove_mvc: |
49 | mvc 0(1,%r1),0(%r3) |
50 | SYM_FUNC_END(__memmove) |
51 | EXPORT_SYMBOL(__memmove) |
52 | |
53 | SYM_FUNC_ALIAS(memmove, __memmove) |
54 | EXPORT_SYMBOL(memmove) |
55 | |
56 | /* |
57 | * memset implementation |
58 | * |
59 | * This code corresponds to the C construct below. We do distinguish |
60 | * between clearing (c == 0) and setting a memory array (c != 0) simply |
61 | * because nearly all memset invocations in the kernel clear memory and |
62 | * the xc instruction is preferred in such cases. |
63 | * |
64 | * void *memset(void *s, int c, size_t n) |
65 | * { |
66 | * if (likely(c == 0)) |
67 | * return __builtin_memset(s, 0, n); |
68 | * return __builtin_memset(s, c, n); |
69 | * } |
70 | */ |
71 | SYM_FUNC_START(__memset) |
72 | ltgr %r4,%r4 |
73 | jz .Lmemset_exit |
74 | ltgr %r3,%r3 |
75 | jnz .Lmemset_fill |
76 | aghi %r4,-1 |
77 | srlg %r3,%r4,8 |
78 | ltgr %r3,%r3 |
79 | lgr %r1,%r2 |
80 | jz .Lmemset_clear_remainder |
81 | .Lmemset_clear_loop: |
82 | xc 0(256,%r1),0(%r1) |
83 | la %r1,256(%r1) |
84 | brctg %r3,.Lmemset_clear_loop |
85 | .Lmemset_clear_remainder: |
86 | larl %r3,.Lmemset_xc |
87 | ex %r4,0(%r3) |
88 | .Lmemset_exit: |
89 | BR_EX %r14 |
90 | .Lmemset_fill: |
91 | cghi %r4,1 |
92 | lgr %r1,%r2 |
93 | je .Lmemset_fill_exit |
94 | aghi %r4,-2 |
95 | srlg %r5,%r4,8 |
96 | ltgr %r5,%r5 |
97 | jz .Lmemset_fill_remainder |
98 | .Lmemset_fill_loop: |
99 | stc %r3,0(%r1) |
100 | mvc 1(255,%r1),0(%r1) |
101 | la %r1,256(%r1) |
102 | brctg %r5,.Lmemset_fill_loop |
103 | .Lmemset_fill_remainder: |
104 | stc %r3,0(%r1) |
105 | larl %r5,.Lmemset_mvc |
106 | ex %r4,0(%r5) |
107 | BR_EX %r14 |
108 | .Lmemset_fill_exit: |
109 | stc %r3,0(%r1) |
110 | BR_EX %r14 |
111 | .Lmemset_xc: |
112 | xc 0(1,%r1),0(%r1) |
113 | .Lmemset_mvc: |
114 | mvc 1(1,%r1),0(%r1) |
115 | SYM_FUNC_END(__memset) |
116 | EXPORT_SYMBOL(__memset) |
117 | |
118 | SYM_FUNC_ALIAS(memset, __memset) |
119 | EXPORT_SYMBOL(memset) |
120 | |
121 | /* |
122 | * memcpy implementation |
123 | * |
124 | * void *memcpy(void *dest, const void *src, size_t n) |
125 | */ |
126 | SYM_FUNC_START(__memcpy) |
127 | ltgr %r4,%r4 |
128 | jz .Lmemcpy_exit |
129 | aghi %r4,-1 |
130 | srlg %r5,%r4,8 |
131 | ltgr %r5,%r5 |
132 | lgr %r1,%r2 |
133 | jnz .Lmemcpy_loop |
134 | .Lmemcpy_remainder: |
135 | larl %r5,.Lmemcpy_mvc |
136 | ex %r4,0(%r5) |
137 | .Lmemcpy_exit: |
138 | BR_EX %r14 |
139 | .Lmemcpy_loop: |
140 | mvc 0(256,%r1),0(%r3) |
141 | la %r1,256(%r1) |
142 | la %r3,256(%r3) |
143 | brctg %r5,.Lmemcpy_loop |
144 | j .Lmemcpy_remainder |
145 | .Lmemcpy_mvc: |
146 | mvc 0(1,%r1),0(%r3) |
147 | SYM_FUNC_END(__memcpy) |
148 | EXPORT_SYMBOL(__memcpy) |
149 | |
150 | SYM_FUNC_ALIAS(memcpy, __memcpy) |
151 | EXPORT_SYMBOL(memcpy) |
152 | |
153 | /* |
154 | * __memset16/32/64 |
155 | * |
156 | * void *__memset16(uint16_t *s, uint16_t v, size_t count) |
157 | * void *__memset32(uint32_t *s, uint32_t v, size_t count) |
158 | * void *__memset64(uint64_t *s, uint64_t v, size_t count) |
159 | */ |
160 | .macro __MEMSET bits,bytes,insn |
161 | SYM_FUNC_START(__memset\bits) |
162 | ltgr %r4,%r4 |
163 | jz .L__memset_exit\bits |
164 | cghi %r4,\bytes |
165 | je .L__memset_store\bits |
166 | aghi %r4,-(\bytes+1) |
167 | srlg %r5,%r4,8 |
168 | ltgr %r5,%r5 |
169 | lgr %r1,%r2 |
170 | jz .L__memset_remainder\bits |
171 | .L__memset_loop\bits: |
172 | \insn %r3,0(%r1) |
173 | mvc \bytes(256-\bytes,%r1),0(%r1) |
174 | la %r1,256(%r1) |
175 | brctg %r5,.L__memset_loop\bits |
176 | .L__memset_remainder\bits: |
177 | \insn %r3,0(%r1) |
178 | larl %r5,.L__memset_mvc\bits |
179 | ex %r4,0(%r5) |
180 | BR_EX %r14 |
181 | .L__memset_store\bits: |
182 | \insn %r3,0(%r2) |
183 | .L__memset_exit\bits: |
184 | BR_EX %r14 |
185 | .L__memset_mvc\bits: |
186 | mvc \bytes(1,%r1),0(%r1) |
187 | SYM_FUNC_END(__memset\bits) |
188 | .endm |
189 | |
190 | __MEMSET 16,2,sth |
191 | EXPORT_SYMBOL(__memset16) |
192 | |
193 | __MEMSET 32,4,st |
194 | EXPORT_SYMBOL(__memset32) |
195 | |
196 | __MEMSET 64,8,stg |
197 | EXPORT_SYMBOL(__memset64) |
198 | |