1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * String handling functions.
4 *
5 * Copyright IBM Corp. 2012
6 */
7
8#include <linux/export.h>
9#include <linux/linkage.h>
10#include <asm/nospec-insn.h>
11
12 GEN_BR_THUNK %r14
13
14/*
15 * void *memmove(void *dest, const void *src, size_t n)
16 */
17SYM_FUNC_START(__memmove)
18 ltgr %r4,%r4
19 lgr %r1,%r2
20 jz .Lmemmove_exit
21 aghi %r4,-1
22 clgr %r2,%r3
23 jnh .Lmemmove_forward
24 la %r5,1(%r4,%r3)
25 clgr %r2,%r5
26 jl .Lmemmove_reverse
27.Lmemmove_forward:
28 srlg %r0,%r4,8
29 ltgr %r0,%r0
30 jz .Lmemmove_forward_remainder
31.Lmemmove_forward_loop:
32 mvc 0(256,%r1),0(%r3)
33 la %r1,256(%r1)
34 la %r3,256(%r3)
35 brctg %r0,.Lmemmove_forward_loop
36.Lmemmove_forward_remainder:
37 exrl %r4,.Lmemmove_mvc
38.Lmemmove_exit:
39 BR_EX %r14
40.Lmemmove_reverse:
41 ic %r0,0(%r4,%r3)
42 stc %r0,0(%r4,%r1)
43 brctg %r4,.Lmemmove_reverse
44 ic %r0,0(%r4,%r3)
45 stc %r0,0(%r4,%r1)
46 BR_EX %r14
47.Lmemmove_mvc:
48 mvc 0(1,%r1),0(%r3)
49SYM_FUNC_END(__memmove)
50EXPORT_SYMBOL(__memmove)
51
52SYM_FUNC_ALIAS(memmove, __memmove)
53EXPORT_SYMBOL(memmove)
54
55/*
56 * memset implementation
57 *
58 * This code corresponds to the C construct below. We do distinguish
59 * between clearing (c == 0) and setting a memory array (c != 0) simply
60 * because nearly all memset invocations in the kernel clear memory and
61 * the xc instruction is preferred in such cases.
62 *
63 * void *memset(void *s, int c, size_t n)
64 * {
65 * if (likely(c == 0))
66 * return __builtin_memset(s, 0, n);
67 * return __builtin_memset(s, c, n);
68 * }
69 */
70SYM_FUNC_START(__memset)
71 ltgr %r4,%r4
72 jz .Lmemset_exit
73 ltgr %r3,%r3
74 jnz .Lmemset_fill
75 aghi %r4,-1
76 srlg %r3,%r4,8
77 ltgr %r3,%r3
78 lgr %r1,%r2
79 jz .Lmemset_clear_remainder
80.Lmemset_clear_loop:
81 xc 0(256,%r1),0(%r1)
82 la %r1,256(%r1)
83 brctg %r3,.Lmemset_clear_loop
84.Lmemset_clear_remainder:
85 exrl %r4,.Lmemset_xc
86.Lmemset_exit:
87 BR_EX %r14
88.Lmemset_fill:
89 cghi %r4,1
90 lgr %r1,%r2
91 je .Lmemset_fill_exit
92 aghi %r4,-2
93 srlg %r5,%r4,8
94 ltgr %r5,%r5
95 jz .Lmemset_fill_remainder
96.Lmemset_fill_loop:
97 stc %r3,0(%r1)
98 mvc 1(255,%r1),0(%r1)
99 la %r1,256(%r1)
100 brctg %r5,.Lmemset_fill_loop
101.Lmemset_fill_remainder:
102 stc %r3,0(%r1)
103 exrl %r4,.Lmemset_mvc
104 BR_EX %r14
105.Lmemset_fill_exit:
106 stc %r3,0(%r1)
107 BR_EX %r14
108.Lmemset_xc:
109 xc 0(1,%r1),0(%r1)
110.Lmemset_mvc:
111 mvc 1(1,%r1),0(%r1)
112SYM_FUNC_END(__memset)
113EXPORT_SYMBOL(__memset)
114
115SYM_FUNC_ALIAS(memset, __memset)
116EXPORT_SYMBOL(memset)
117
118/*
119 * memcpy implementation
120 *
121 * void *memcpy(void *dest, const void *src, size_t n)
122 */
123SYM_FUNC_START(__memcpy)
124 ltgr %r4,%r4
125 jz .Lmemcpy_exit
126 aghi %r4,-1
127 srlg %r5,%r4,8
128 ltgr %r5,%r5
129 lgr %r1,%r2
130 jnz .Lmemcpy_loop
131.Lmemcpy_remainder:
132 exrl %r4,.Lmemcpy_mvc
133.Lmemcpy_exit:
134 BR_EX %r14
135.Lmemcpy_loop:
136 mvc 0(256,%r1),0(%r3)
137 la %r1,256(%r1)
138 la %r3,256(%r3)
139 brctg %r5,.Lmemcpy_loop
140 j .Lmemcpy_remainder
141.Lmemcpy_mvc:
142 mvc 0(1,%r1),0(%r3)
143SYM_FUNC_END(__memcpy)
144EXPORT_SYMBOL(__memcpy)
145
146SYM_FUNC_ALIAS(memcpy, __memcpy)
147EXPORT_SYMBOL(memcpy)
148
149/*
150 * __memset16/32/64
151 *
152 * void *__memset16(uint16_t *s, uint16_t v, size_t count)
153 * void *__memset32(uint32_t *s, uint32_t v, size_t count)
154 * void *__memset64(uint64_t *s, uint64_t v, size_t count)
155 */
156.macro __MEMSET bits,bytes,insn
157SYM_FUNC_START(__memset\bits)
158 ltgr %r4,%r4
159 jz .L__memset_exit\bits
160 cghi %r4,\bytes
161 je .L__memset_store\bits
162 aghi %r4,-(\bytes+1)
163 srlg %r5,%r4,8
164 ltgr %r5,%r5
165 lgr %r1,%r2
166 jz .L__memset_remainder\bits
167.L__memset_loop\bits:
168 \insn %r3,0(%r1)
169 mvc \bytes(256-\bytes,%r1),0(%r1)
170 la %r1,256(%r1)
171 brctg %r5,.L__memset_loop\bits
172.L__memset_remainder\bits:
173 \insn %r3,0(%r1)
174 exrl %r4,.L__memset_mvc\bits
175 BR_EX %r14
176.L__memset_store\bits:
177 \insn %r3,0(%r2)
178.L__memset_exit\bits:
179 BR_EX %r14
180.L__memset_mvc\bits:
181 mvc \bytes(1,%r1),0(%r1)
182SYM_FUNC_END(__memset\bits)
183.endm
184
185__MEMSET 16,2,sth
186EXPORT_SYMBOL(__memset16)
187
188__MEMSET 32,4,st
189EXPORT_SYMBOL(__memset32)
190
191__MEMSET 64,8,stg
192EXPORT_SYMBOL(__memset64)
193

source code of linux/arch/s390/lib/mem.S