1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * arch/alpha/lib/ev6-stxcpy.S
4 * 21264 version contributed by Rick Gorton <rick.gorton@alpha-processor.com>
5 *
6 * Copy a null-terminated string from SRC to DST.
7 *
8 * This is an internal routine used by strcpy, stpcpy, and strcat.
9 * As such, it uses special linkage conventions to make implementation
10 * of these public functions more efficient.
11 *
12 * On input:
13 * t9 = return address
14 * a0 = DST
15 * a1 = SRC
16 *
17 * On output:
18 * t12 = bitmask (with one bit set) indicating the last byte written
19 * a0 = unaligned address of the last *word* written
20 *
21 * Furthermore, v0, a3-a5, t11, and t12 are untouched.
22 *
23 * Much of the information about 21264 scheduling/coding comes from:
24 * Compiler Writer's Guide for the Alpha 21264
25 * abbreviated as 'CWG' in other comments here
26 * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html
27 * Scheduling notation:
28 * E - either cluster
29 * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1
30 * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1
31 * Try not to change the actual algorithm if possible for consistency.
32 */
33
34#include <asm/regdef.h>
35
36 .set noat
37 .set noreorder
38
39 .text
40
41/* There is a problem with either gdb (as of 4.16) or gas (as of 2.7) that
42 doesn't like putting the entry point for a procedure somewhere in the
43 middle of the procedure descriptor. Work around this by putting the
44 aligned copy in its own procedure descriptor */
45
46
47 .ent stxcpy_aligned
48 .align 4
49stxcpy_aligned:
50 .frame sp, 0, t9
51 .prologue 0
52
53 /* On entry to this basic block:
54 t0 == the first destination word for masking back in
55 t1 == the first source word. */
56
57 /* Create the 1st output word and detect 0's in the 1st input word. */
58 lda t2, -1 # E : build a mask against false zero
59 mskqh t2, a1, t2 # U : detection in the src word (stall)
60 mskqh t1, a1, t3 # U :
61 ornot t1, t2, t2 # E : (stall)
62
63 mskql t0, a1, t0 # U : assemble the first output word
64 cmpbge zero, t2, t8 # E : bits set iff null found
65 or t0, t3, t1 # E : (stall)
66 bne t8, $a_eos # U : (stall)
67
68 /* On entry to this basic block:
69 t0 == the first destination word for masking back in
70 t1 == a source word not containing a null. */
71 /* Nops here to separate store quads from load quads */
72
73$a_loop:
74 stq_u t1, 0(a0) # L :
75 addq a0, 8, a0 # E :
76 nop
77 nop
78
79 ldq_u t1, 0(a1) # L : Latency=3
80 addq a1, 8, a1 # E :
81 cmpbge zero, t1, t8 # E : (3 cycle stall)
82 beq t8, $a_loop # U : (stall for t8)
83
84 /* Take care of the final (partial) word store.
85 On entry to this basic block we have:
86 t1 == the source word containing the null
87 t8 == the cmpbge mask that found it. */
88$a_eos:
89 negq t8, t6 # E : find low bit set
90 and t8, t6, t12 # E : (stall)
91 /* For the sake of the cache, don't read a destination word
92 if we're not going to need it. */
93 and t12, 0x80, t6 # E : (stall)
94 bne t6, 1f # U : (stall)
95
96 /* We're doing a partial word store and so need to combine
97 our source and original destination words. */
98 ldq_u t0, 0(a0) # L : Latency=3
99 subq t12, 1, t6 # E :
100 zapnot t1, t6, t1 # U : clear src bytes >= null (stall)
101 or t12, t6, t8 # E : (stall)
102
103 zap t0, t8, t0 # E : clear dst bytes <= null
104 or t0, t1, t1 # E : (stall)
105 nop
106 nop
107
1081: stq_u t1, 0(a0) # L :
109 ret (t9) # L0 : Latency=3
110 nop
111 nop
112
113 .end stxcpy_aligned
114
115 .align 4
116 .ent __stxcpy
117 .globl __stxcpy
118__stxcpy:
119 .frame sp, 0, t9
120 .prologue 0
121
122 /* Are source and destination co-aligned? */
123 xor a0, a1, t0 # E :
124 unop # E :
125 and t0, 7, t0 # E : (stall)
126 bne t0, $unaligned # U : (stall)
127
128 /* We are co-aligned; take care of a partial first word. */
129 ldq_u t1, 0(a1) # L : load first src word
130 and a0, 7, t0 # E : take care not to load a word ...
131 addq a1, 8, a1 # E :
132 beq t0, stxcpy_aligned # U : ... if we wont need it (stall)
133
134 ldq_u t0, 0(a0) # L :
135 br stxcpy_aligned # L0 : Latency=3
136 nop
137 nop
138
139
140/* The source and destination are not co-aligned. Align the destination
141 and cope. We have to be very careful about not reading too much and
142 causing a SEGV. */
143
144 .align 4
145$u_head:
146 /* We know just enough now to be able to assemble the first
147 full source word. We can still find a zero at the end of it
148 that prevents us from outputting the whole thing.
149
150 On entry to this basic block:
151 t0 == the first dest word, for masking back in, if needed else 0
152 t1 == the low bits of the first source word
153 t6 == bytemask that is -1 in dest word bytes */
154
155 ldq_u t2, 8(a1) # L :
156 addq a1, 8, a1 # E :
157 extql t1, a1, t1 # U : (stall on a1)
158 extqh t2, a1, t4 # U : (stall on a1)
159
160 mskql t0, a0, t0 # U :
161 or t1, t4, t1 # E :
162 mskqh t1, a0, t1 # U : (stall on t1)
163 or t0, t1, t1 # E : (stall on t1)
164
165 or t1, t6, t6 # E :
166 cmpbge zero, t6, t8 # E : (stall)
167 lda t6, -1 # E : for masking just below
168 bne t8, $u_final # U : (stall)
169
170 mskql t6, a1, t6 # U : mask out the bits we have
171 or t6, t2, t2 # E : already extracted before (stall)
172 cmpbge zero, t2, t8 # E : testing eos (stall)
173 bne t8, $u_late_head_exit # U : (stall)
174
175 /* Finally, we've got all the stupid leading edge cases taken care
176 of and we can set up to enter the main loop. */
177
178 stq_u t1, 0(a0) # L : store first output word
179 addq a0, 8, a0 # E :
180 extql t2, a1, t0 # U : position ho-bits of lo word
181 ldq_u t2, 8(a1) # U : read next high-order source word
182
183 addq a1, 8, a1 # E :
184 cmpbge zero, t2, t8 # E : (stall for t2)
185 nop # E :
186 bne t8, $u_eos # U : (stall)
187
188 /* Unaligned copy main loop. In order to avoid reading too much,
189 the loop is structured to detect zeros in aligned source words.
190 This has, unfortunately, effectively pulled half of a loop
191 iteration out into the head and half into the tail, but it does
192 prevent nastiness from accumulating in the very thing we want
193 to run as fast as possible.
194
195 On entry to this basic block:
196 t0 == the shifted high-order bits from the previous source word
197 t2 == the unshifted current source word
198
199 We further know that t2 does not contain a null terminator. */
200
201 .align 3
202$u_loop:
203 extqh t2, a1, t1 # U : extract high bits for current word
204 addq a1, 8, a1 # E : (stall)
205 extql t2, a1, t3 # U : extract low bits for next time (stall)
206 addq a0, 8, a0 # E :
207
208 or t0, t1, t1 # E : current dst word now complete
209 ldq_u t2, 0(a1) # L : Latency=3 load high word for next time
210 stq_u t1, -8(a0) # L : save the current word (stall)
211 mov t3, t0 # E :
212
213 cmpbge zero, t2, t8 # E : test new word for eos
214 beq t8, $u_loop # U : (stall)
215 nop
216 nop
217
218 /* We've found a zero somewhere in the source word we just read.
219 If it resides in the lower half, we have one (probably partial)
220 word to write out, and if it resides in the upper half, we
221 have one full and one partial word left to write out.
222
223 On entry to this basic block:
224 t0 == the shifted high-order bits from the previous source word
225 t2 == the unshifted current source word. */
226$u_eos:
227 extqh t2, a1, t1 # U :
228 or t0, t1, t1 # E : first (partial) source word complete (stall)
229 cmpbge zero, t1, t8 # E : is the null in this first bit? (stall)
230 bne t8, $u_final # U : (stall)
231
232$u_late_head_exit:
233 stq_u t1, 0(a0) # L : the null was in the high-order bits
234 addq a0, 8, a0 # E :
235 extql t2, a1, t1 # U :
236 cmpbge zero, t1, t8 # E : (stall)
237
238 /* Take care of a final (probably partial) result word.
239 On entry to this basic block:
240 t1 == assembled source word
241 t8 == cmpbge mask that found the null. */
242$u_final:
243 negq t8, t6 # E : isolate low bit set
244 and t6, t8, t12 # E : (stall)
245 and t12, 0x80, t6 # E : avoid dest word load if we can (stall)
246 bne t6, 1f # U : (stall)
247
248 ldq_u t0, 0(a0) # E :
249 subq t12, 1, t6 # E :
250 or t6, t12, t8 # E : (stall)
251 zapnot t1, t6, t1 # U : kill source bytes >= null (stall)
252
253 zap t0, t8, t0 # U : kill dest bytes <= null (2 cycle data stall)
254 or t0, t1, t1 # E : (stall)
255 nop
256 nop
257
2581: stq_u t1, 0(a0) # L :
259 ret (t9) # L0 : Latency=3
260 nop
261 nop
262
263 /* Unaligned copy entry point. */
264 .align 4
265$unaligned:
266
267 ldq_u t1, 0(a1) # L : load first source word
268 and a0, 7, t4 # E : find dest misalignment
269 and a1, 7, t5 # E : find src misalignment
270 /* Conditionally load the first destination word and a bytemask
271 with 0xff indicating that the destination byte is sacrosanct. */
272 mov zero, t0 # E :
273
274 mov zero, t6 # E :
275 beq t4, 1f # U :
276 ldq_u t0, 0(a0) # L :
277 lda t6, -1 # E :
278
279 mskql t6, a0, t6 # U :
280 nop
281 nop
282 nop
2831:
284 subq a1, t4, a1 # E : sub dest misalignment from src addr
285 /* If source misalignment is larger than dest misalignment, we need
286 extra startup checks to avoid SEGV. */
287 cmplt t4, t5, t12 # E :
288 beq t12, $u_head # U :
289 lda t2, -1 # E : mask out leading garbage in source
290
291 mskqh t2, t5, t2 # U :
292 ornot t1, t2, t3 # E : (stall)
293 cmpbge zero, t3, t8 # E : is there a zero? (stall)
294 beq t8, $u_head # U : (stall)
295
296 /* At this point we've found a zero in the first partial word of
297 the source. We need to isolate the valid source data and mask
298 it into the original destination data. (Incidentally, we know
299 that we'll need at least one byte of that original dest word.) */
300
301 ldq_u t0, 0(a0) # L :
302 negq t8, t6 # E : build bitmask of bytes <= zero
303 and t6, t8, t12 # E : (stall)
304 and a1, 7, t5 # E :
305
306 subq t12, 1, t6 # E :
307 or t6, t12, t8 # E : (stall)
308 srl t12, t5, t12 # U : adjust final null return value
309 zapnot t2, t8, t2 # U : prepare source word; mirror changes (stall)
310
311 and t1, t2, t1 # E : to source validity mask
312 extql t2, a1, t2 # U :
313 extql t1, a1, t1 # U : (stall)
314 andnot t0, t2, t0 # .. e1 : zero place for source to reside (stall)
315
316 or t0, t1, t1 # e1 : and put it there
317 stq_u t1, 0(a0) # .. e0 : (stall)
318 ret (t9) # e1 :
319 nop
320
321 .end __stxcpy
322
323

source code of linux/arch/alpha/lib/ev6-stxcpy.S