1 | /* |
2 | * M7memcpy: Optimized SPARC M7 memcpy |
3 | * |
4 | * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. |
5 | */ |
6 | |
7 | .file "M7memcpy.S" |
8 | |
9 | /* |
10 | * memcpy(s1, s2, len) |
11 | * |
12 | * Copy s2 to s1, always copy n bytes. |
13 | * Note: this C code does not work for overlapped copies. |
14 | * |
15 | * Fast assembler language version of the following C-program for memcpy |
16 | * which represents the `standard' for the C-library. |
17 | * |
18 | * void * |
19 | * memcpy(void *s, const void *s0, size_t n) |
20 | * { |
21 | * if (n != 0) { |
22 | * char *s1 = s; |
23 | * const char *s2 = s0; |
24 | * do { |
25 | * *s1++ = *s2++; |
26 | * } while (--n != 0); |
27 | * } |
28 | * return (s); |
29 | * } |
30 | * |
31 | * |
32 | * SPARC T7/M7 Flow : |
33 | * |
34 | * if (count < SMALL_MAX) { |
35 | * if count < SHORTCOPY (SHORTCOPY=3) |
36 | * copy bytes; exit with dst addr |
37 | * if src & dst aligned on word boundary but not long word boundary, |
38 | * copy with ldw/stw; branch to finish_up |
39 | * if src & dst aligned on long word boundary |
40 | * copy with ldx/stx; branch to finish_up |
41 | * if src & dst not aligned and length <= SHORTCHECK (SHORTCHECK=14) |
42 | * copy bytes; exit with dst addr |
43 | * move enough bytes to get src to word boundary |
44 | * if dst now on word boundary |
45 | * move_words: |
46 | * copy words; branch to finish_up |
47 | * if dst now on half word boundary |
48 | * load words, shift half words, store words; branch to finish_up |
49 | * if dst on byte 1 |
50 | * load words, shift 3 bytes, store words; branch to finish_up |
51 | * if dst on byte 3 |
52 | * load words, shift 1 byte, store words; branch to finish_up |
53 | * finish_up: |
54 | * copy bytes; exit with dst addr |
55 | * } else { More than SMALL_MAX bytes |
56 | * move bytes until dst is on long word boundary |
57 | * if( src is on long word boundary ) { |
58 | * if (count < MED_MAX) { |
59 | * finish_long: src/dst aligned on 8 bytes |
60 | * copy with ldx/stx in 8-way unrolled loop; |
61 | * copy final 0-63 bytes; exit with dst addr |
62 | * } else { src/dst aligned; count > MED_MAX |
63 | * align dst on 64 byte boundary; for main data movement: |
64 | * prefetch src data to L2 cache; let HW prefetch move data to L1 cache |
65 | * Use BIS (block initializing store) to avoid copying store cache |
66 | * lines from memory. But pre-store first element of each cache line |
67 | * ST_CHUNK lines in advance of the rest of that cache line. That |
68 | * gives time for replacement cache lines to be written back without |
69 | * excess STQ and Miss Buffer filling. Repeat until near the end, |
70 | * then finish up storing before going to finish_long. |
71 | * } |
72 | * } else { src/dst not aligned on 8 bytes |
73 | * if src is word aligned and count < MED_WMAX |
74 | * move words in 8-way unrolled loop |
75 | * move final 0-31 bytes; exit with dst addr |
76 | * if count < MED_UMAX |
77 | * use alignaddr/faligndata combined with ldd/std in 8-way |
78 | * unrolled loop to move data. |
79 | * go to unalign_done |
80 | * else |
81 | * setup alignaddr for faligndata instructions |
82 | * align dst on 64 byte boundary; prefetch src data to L1 cache |
83 | * loadx8, falign, block-store, prefetch loop |
84 | * (only use block-init-store when src/dst on 8 byte boundaries.) |
85 | * unalign_done: |
86 | * move remaining bytes for unaligned cases. exit with dst addr. |
87 | * } |
88 | * |
89 | */ |
90 | |
91 | #include <asm/visasm.h> |
92 | #include <asm/asi.h> |
93 | |
94 | #if !defined(EX_LD) && !defined(EX_ST) |
95 | #define NON_USER_COPY |
96 | #endif |
97 | |
98 | #ifndef EX_LD |
99 | #define EX_LD(x,y) x |
100 | #endif |
101 | #ifndef EX_LD_FP |
102 | #define EX_LD_FP(x,y) x |
103 | #endif |
104 | |
105 | #ifndef EX_ST |
106 | #define EX_ST(x,y) x |
107 | #endif |
108 | #ifndef EX_ST_FP |
109 | #define EX_ST_FP(x,y) x |
110 | #endif |
111 | |
112 | #ifndef EX_RETVAL |
113 | #define EX_RETVAL(x) x |
114 | #endif |
115 | |
116 | #ifndef LOAD |
117 | #define LOAD(type,addr,dest) type [addr], dest |
118 | #endif |
119 | |
120 | #ifndef STORE |
121 | #define STORE(type,src,addr) type src, [addr] |
122 | #endif |
123 | |
124 | /* |
125 | * ASI_BLK_INIT_QUAD_LDD_P/ASI_BLK_INIT_QUAD_LDD_S marks the cache |
126 | * line as "least recently used" which means if many threads are |
127 | * active, it has a high probability of being pushed out of the cache |
128 | * between the first initializing store and the final stores. |
129 | * Thus, we use ASI_ST_BLKINIT_MRU_P/ASI_ST_BLKINIT_MRU_S which |
130 | * marks the cache line as "most recently used" for all |
131 | * but the last cache line |
132 | */ |
133 | #ifndef STORE_ASI |
134 | #ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA |
135 | #define STORE_ASI ASI_BLK_INIT_QUAD_LDD_P |
136 | #else |
137 | #define STORE_ASI 0x80 /* ASI_P */ |
138 | #endif |
139 | #endif |
140 | |
141 | #ifndef STORE_MRU_ASI |
142 | #ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA |
143 | #define STORE_MRU_ASI ASI_ST_BLKINIT_MRU_P |
144 | #else |
145 | #define STORE_MRU_ASI 0x80 /* ASI_P */ |
146 | #endif |
147 | #endif |
148 | |
149 | #ifndef STORE_INIT |
150 | #define STORE_INIT(src,addr) stxa src, [addr] STORE_ASI |
151 | #endif |
152 | |
153 | #ifndef STORE_INIT_MRU |
154 | #define STORE_INIT_MRU(src,addr) stxa src, [addr] STORE_MRU_ASI |
155 | #endif |
156 | |
157 | #ifndef FUNC_NAME |
158 | #define FUNC_NAME M7memcpy |
159 | #endif |
160 | |
161 | #ifndef PREAMBLE |
162 | #define PREAMBLE |
163 | #endif |
164 | |
165 | #define BLOCK_SIZE 64 |
166 | #define SHORTCOPY 3 |
167 | #define SHORTCHECK 14 |
168 | #define SHORT_LONG 64 /* max copy for short longword-aligned case */ |
169 | /* must be at least 64 */ |
170 | #define SMALL_MAX 128 |
171 | #define MED_UMAX 1024 /* max copy for medium un-aligned case */ |
172 | #define MED_WMAX 1024 /* max copy for medium word-aligned case */ |
173 | #define MED_MAX 1024 /* max copy for medium longword-aligned case */ |
174 | #define ST_CHUNK 24 /* ST_CHUNK - block of values for BIS Store */ |
175 | #define ALIGN_PRE 24 /* distance for aligned prefetch loop */ |
176 | |
177 | .register %g2,#scratch |
178 | |
179 | .section ".text" |
180 | .global FUNC_NAME |
181 | .type FUNC_NAME, #function |
182 | .align 16 |
183 | FUNC_NAME: |
184 | srlx %o2, 31, %g2 |
185 | cmp %g2, 0 |
186 | tne %xcc, 5 |
187 | PREAMBLE |
188 | mov %o0, %g1 ! save %o0 |
189 | brz,pn %o2, .Lsmallx |
190 | cmp %o2, 3 |
191 | ble,pn %icc, .Ltiny_cp |
192 | cmp %o2, 19 |
193 | ble,pn %icc, .Lsmall_cp |
194 | or %o0, %o1, %g2 |
195 | cmp %o2, SMALL_MAX |
196 | bl,pn %icc, .Lmedium_cp |
197 | nop |
198 | |
199 | .Lmedium: |
200 | neg %o0, %o5 |
201 | andcc %o5, 7, %o5 ! bytes till DST 8 byte aligned |
202 | brz,pt %o5, .Ldst_aligned_on_8 |
203 | |
204 | ! %o5 has the bytes to be written in partial store. |
205 | sub %o2, %o5, %o2 |
206 | sub %o1, %o0, %o1 ! %o1 gets the difference |
207 | 7: ! dst aligning loop |
208 | add %o1, %o0, %o4 |
209 | EX_LD(LOAD(ldub, %o4, %o4), memcpy_retl_o2_plus_o5) ! load one byte |
210 | subcc %o5, 1, %o5 |
211 | EX_ST(STORE(stb, %o4, %o0), memcpy_retl_o2_plus_o5_plus_1) |
212 | bgu,pt %xcc, 7b |
213 | add %o0, 1, %o0 ! advance dst |
214 | add %o1, %o0, %o1 ! restore %o1 |
215 | .Ldst_aligned_on_8: |
216 | andcc %o1, 7, %o5 |
217 | brnz,pt %o5, .Lsrc_dst_unaligned_on_8 |
218 | nop |
219 | |
220 | .Lsrc_dst_aligned_on_8: |
221 | ! check if we are copying MED_MAX or more bytes |
222 | set MED_MAX, %o3 |
223 | cmp %o2, %o3 ! limit to store buffer size |
224 | bgu,pn %xcc, .Llarge_align8_copy |
225 | nop |
226 | |
227 | /* |
228 | * Special case for handling when src and dest are both long word aligned |
229 | * and total data to move is less than MED_MAX bytes |
230 | */ |
231 | .Lmedlong: |
232 | subcc %o2, 63, %o2 ! adjust length to allow cc test |
233 | ble,pn %xcc, .Lmedl63 ! skip big loop if less than 64 bytes |
234 | nop |
235 | .Lmedl64: |
236 | EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_63) ! load |
237 | subcc %o2, 64, %o2 ! decrement length count |
238 | EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_63_64) ! and store |
239 | EX_LD(LOAD(ldx, %o1+8, %o3), memcpy_retl_o2_plus_63_56) ! a block of 64 |
240 | EX_ST(STORE(stx, %o3, %o0+8), memcpy_retl_o2_plus_63_56) |
241 | EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_63_48) |
242 | EX_ST(STORE(stx, %o4, %o0+16), memcpy_retl_o2_plus_63_48) |
243 | EX_LD(LOAD(ldx, %o1+24, %o3), memcpy_retl_o2_plus_63_40) |
244 | EX_ST(STORE(stx, %o3, %o0+24), memcpy_retl_o2_plus_63_40) |
245 | EX_LD(LOAD(ldx, %o1+32, %o4), memcpy_retl_o2_plus_63_32)! load and store |
246 | EX_ST(STORE(stx, %o4, %o0+32), memcpy_retl_o2_plus_63_32) |
247 | EX_LD(LOAD(ldx, %o1+40, %o3), memcpy_retl_o2_plus_63_24)! a block of 64 |
248 | add %o1, 64, %o1 ! increase src ptr by 64 |
249 | EX_ST(STORE(stx, %o3, %o0+40), memcpy_retl_o2_plus_63_24) |
250 | EX_LD(LOAD(ldx, %o1-16, %o4), memcpy_retl_o2_plus_63_16) |
251 | add %o0, 64, %o0 ! increase dst ptr by 64 |
252 | EX_ST(STORE(stx, %o4, %o0-16), memcpy_retl_o2_plus_63_16) |
253 | EX_LD(LOAD(ldx, %o1-8, %o3), memcpy_retl_o2_plus_63_8) |
254 | bgu,pt %xcc, .Lmedl64 ! repeat if at least 64 bytes left |
255 | EX_ST(STORE(stx, %o3, %o0-8), memcpy_retl_o2_plus_63_8) |
256 | .Lmedl63: |
257 | addcc %o2, 32, %o2 ! adjust remaining count |
258 | ble,pt %xcc, .Lmedl31 ! to skip if 31 or fewer bytes left |
259 | nop |
260 | EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_31) ! load |
261 | sub %o2, 32, %o2 ! decrement length count |
262 | EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_31_32) ! and store |
263 | EX_LD(LOAD(ldx, %o1+8, %o3), memcpy_retl_o2_plus_31_24) ! a block of 32 |
264 | add %o1, 32, %o1 ! increase src ptr by 32 |
265 | EX_ST(STORE(stx, %o3, %o0+8), memcpy_retl_o2_plus_31_24) |
266 | EX_LD(LOAD(ldx, %o1-16, %o4), memcpy_retl_o2_plus_31_16) |
267 | add %o0, 32, %o0 ! increase dst ptr by 32 |
268 | EX_ST(STORE(stx, %o4, %o0-16), memcpy_retl_o2_plus_31_16) |
269 | EX_LD(LOAD(ldx, %o1-8, %o3), memcpy_retl_o2_plus_31_8) |
270 | EX_ST(STORE(stx, %o3, %o0-8), memcpy_retl_o2_plus_31_8) |
271 | .Lmedl31: |
272 | addcc %o2, 16, %o2 ! adjust remaining count |
273 | ble,pt %xcc, .Lmedl15 ! skip if 15 or fewer bytes left |
274 | nop ! |
275 | EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_15) |
276 | add %o1, 16, %o1 ! increase src ptr by 16 |
277 | EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_15) |
278 | sub %o2, 16, %o2 ! decrease count by 16 |
279 | EX_LD(LOAD(ldx, %o1-8, %o3), memcpy_retl_o2_plus_15_8) |
280 | add %o0, 16, %o0 ! increase dst ptr by 16 |
281 | EX_ST(STORE(stx, %o3, %o0-8), memcpy_retl_o2_plus_15_8) |
282 | .Lmedl15: |
283 | addcc %o2, 15, %o2 ! restore count |
284 | bz,pt %xcc, .Lsmallx ! exit if finished |
285 | cmp %o2, 8 |
286 | blt,pt %xcc, .Lmedw7 ! skip if 7 or fewer bytes left |
287 | tst %o2 |
288 | EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2) ! load 8 bytes |
289 | add %o1, 8, %o1 ! increase src ptr by 8 |
290 | add %o0, 8, %o0 ! increase dst ptr by 8 |
291 | subcc %o2, 8, %o2 ! decrease count by 8 |
292 | bnz,pn %xcc, .Lmedw7 |
293 | EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8) ! and store 8 |
294 | retl |
295 | mov EX_RETVAL(%g1), %o0 ! restore %o0 |
296 | |
297 | .align 16 |
298 | .Lsrc_dst_unaligned_on_8: |
299 | ! DST is 8-byte aligned, src is not |
300 | 2: |
301 | andcc %o1, 0x3, %o5 ! test word alignment |
302 | bnz,pt %xcc, .Lunalignsetup ! branch to skip if not word aligned |
303 | nop |
304 | |
305 | /* |
306 | * Handle all cases where src and dest are aligned on word |
307 | * boundaries. Use unrolled loops for better performance. |
308 | * This option wins over standard large data move when |
309 | * source and destination is in cache for.Lmedium |
310 | * to short data moves. |
311 | */ |
312 | set MED_WMAX, %o3 |
313 | cmp %o2, %o3 ! limit to store buffer size |
314 | bge,pt %xcc, .Lunalignrejoin ! otherwise rejoin main loop |
315 | nop |
316 | |
317 | subcc %o2, 31, %o2 ! adjust length to allow cc test |
318 | ! for end of loop |
319 | ble,pt %xcc, .Lmedw31 ! skip big loop if less than 16 |
320 | .Lmedw32: |
321 | EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2_plus_31)! move a block of 32 |
322 | sllx %o4, 32, %o5 |
323 | EX_LD(LOAD(ld, %o1+4, %o4), memcpy_retl_o2_plus_31) |
324 | or %o4, %o5, %o5 |
325 | EX_ST(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_31) |
326 | subcc %o2, 32, %o2 ! decrement length count |
327 | EX_LD(LOAD(ld, %o1+8, %o4), memcpy_retl_o2_plus_31_24) |
328 | sllx %o4, 32, %o5 |
329 | EX_LD(LOAD(ld, %o1+12, %o4), memcpy_retl_o2_plus_31_24) |
330 | or %o4, %o5, %o5 |
331 | EX_ST(STORE(stx, %o5, %o0+8), memcpy_retl_o2_plus_31_24) |
332 | add %o1, 32, %o1 ! increase src ptr by 32 |
333 | EX_LD(LOAD(ld, %o1-16, %o4), memcpy_retl_o2_plus_31_16) |
334 | sllx %o4, 32, %o5 |
335 | EX_LD(LOAD(ld, %o1-12, %o4), memcpy_retl_o2_plus_31_16) |
336 | or %o4, %o5, %o5 |
337 | EX_ST(STORE(stx, %o5, %o0+16), memcpy_retl_o2_plus_31_16) |
338 | add %o0, 32, %o0 ! increase dst ptr by 32 |
339 | EX_LD(LOAD(ld, %o1-8, %o4), memcpy_retl_o2_plus_31_8) |
340 | sllx %o4, 32, %o5 |
341 | EX_LD(LOAD(ld, %o1-4, %o4), memcpy_retl_o2_plus_31_8) |
342 | or %o4, %o5, %o5 |
343 | bgu,pt %xcc, .Lmedw32 ! repeat if at least 32 bytes left |
344 | EX_ST(STORE(stx, %o5, %o0-8), memcpy_retl_o2_plus_31_8) |
345 | .Lmedw31: |
346 | addcc %o2, 31, %o2 ! restore count |
347 | |
348 | bz,pt %xcc, .Lsmallx ! exit if finished |
349 | nop |
350 | cmp %o2, 16 |
351 | blt,pt %xcc, .Lmedw15 |
352 | nop |
353 | EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2)! move a block of 16 bytes |
354 | sllx %o4, 32, %o5 |
355 | subcc %o2, 16, %o2 ! decrement length count |
356 | EX_LD(LOAD(ld, %o1+4, %o4), memcpy_retl_o2_plus_16) |
357 | or %o4, %o5, %o5 |
358 | EX_ST(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_16) |
359 | add %o1, 16, %o1 ! increase src ptr by 16 |
360 | EX_LD(LOAD(ld, %o1-8, %o4), memcpy_retl_o2_plus_8) |
361 | add %o0, 16, %o0 ! increase dst ptr by 16 |
362 | sllx %o4, 32, %o5 |
363 | EX_LD(LOAD(ld, %o1-4, %o4), memcpy_retl_o2_plus_8) |
364 | or %o4, %o5, %o5 |
365 | EX_ST(STORE(stx, %o5, %o0-8), memcpy_retl_o2_plus_8) |
366 | .Lmedw15: |
367 | bz,pt %xcc, .Lsmallx ! exit if finished |
368 | cmp %o2, 8 |
369 | blt,pn %xcc, .Lmedw7 ! skip if 7 or fewer bytes left |
370 | tst %o2 |
371 | EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2) ! load 4 bytes |
372 | subcc %o2, 8, %o2 ! decrease count by 8 |
373 | EX_ST(STORE(stw, %o4, %o0), memcpy_retl_o2_plus_8)! and store 4 bytes |
374 | add %o1, 8, %o1 ! increase src ptr by 8 |
375 | EX_LD(LOAD(ld, %o1-4, %o3), memcpy_retl_o2_plus_4) ! load 4 bytes |
376 | add %o0, 8, %o0 ! increase dst ptr by 8 |
377 | EX_ST(STORE(stw, %o3, %o0-4), memcpy_retl_o2_plus_4)! and store 4 bytes |
378 | bz,pt %xcc, .Lsmallx ! exit if finished |
379 | .Lmedw7: ! count is ge 1, less than 8 |
380 | cmp %o2, 4 ! check for 4 bytes left |
381 | blt,pn %xcc, .Lsmallleft3 ! skip if 3 or fewer bytes left |
382 | nop ! |
383 | EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2) ! load 4 bytes |
384 | add %o1, 4, %o1 ! increase src ptr by 4 |
385 | add %o0, 4, %o0 ! increase dst ptr by 4 |
386 | subcc %o2, 4, %o2 ! decrease count by 4 |
387 | bnz .Lsmallleft3 |
388 | EX_ST(STORE(stw, %o4, %o0-4), memcpy_retl_o2_plus_4)! and store 4 bytes |
389 | retl |
390 | mov EX_RETVAL(%g1), %o0 |
391 | |
392 | .align 16 |
393 | .Llarge_align8_copy: ! Src and dst share 8 byte alignment |
394 | ! align dst to 64 byte boundary |
395 | andcc %o0, 0x3f, %o3 ! %o3 == 0 means dst is 64 byte aligned |
396 | brz,pn %o3, .Laligned_to_64 |
397 | andcc %o0, 8, %o3 ! odd long words to move? |
398 | brz,pt %o3, .Laligned_to_16 |
399 | nop |
400 | EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2) |
401 | sub %o2, 8, %o2 |
402 | add %o1, 8, %o1 ! increment src ptr |
403 | add %o0, 8, %o0 ! increment dst ptr |
404 | EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8) |
405 | .Laligned_to_16: |
406 | andcc %o0, 16, %o3 ! pair of long words to move? |
407 | brz,pt %o3, .Laligned_to_32 |
408 | nop |
409 | EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2) |
410 | sub %o2, 16, %o2 |
411 | EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_16) |
412 | add %o1, 16, %o1 ! increment src ptr |
413 | EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_8) |
414 | add %o0, 16, %o0 ! increment dst ptr |
415 | EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8) |
416 | .Laligned_to_32: |
417 | andcc %o0, 32, %o3 ! four long words to move? |
418 | brz,pt %o3, .Laligned_to_64 |
419 | nop |
420 | EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2) |
421 | sub %o2, 32, %o2 |
422 | EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_32) |
423 | EX_LD(LOAD(ldx, %o1+8, %o4), memcpy_retl_o2_plus_24) |
424 | EX_ST(STORE(stx, %o4, %o0+8), memcpy_retl_o2_plus_24) |
425 | EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_16) |
426 | EX_ST(STORE(stx, %o4, %o0+16), memcpy_retl_o2_plus_16) |
427 | add %o1, 32, %o1 ! increment src ptr |
428 | EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_8) |
429 | add %o0, 32, %o0 ! increment dst ptr |
430 | EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8) |
431 | .Laligned_to_64: |
432 | ! |
433 | ! Using block init store (BIS) instructions to avoid fetching cache |
434 | ! lines from memory. Use ST_CHUNK stores to first element of each cache |
435 | ! line (similar to prefetching) to avoid overfilling STQ or miss buffers. |
436 | ! Gives existing cache lines time to be moved out of L1/L2/L3 cache. |
437 | ! Initial stores using MRU version of BIS to keep cache line in |
438 | ! cache until we are ready to store final element of cache line. |
439 | ! Then store last element using the LRU version of BIS. |
440 | ! |
441 | andn %o2, 0x3f, %o5 ! %o5 is multiple of block size |
442 | and %o2, 0x3f, %o2 ! residue bytes in %o2 |
443 | ! |
444 | ! We use STORE_MRU_ASI for the first seven stores to each cache line |
445 | ! followed by STORE_ASI (mark as LRU) for the last store. That |
446 | ! mixed approach reduces the probability that the cache line is removed |
447 | ! before we finish setting it, while minimizing the effects on |
448 | ! other cached values during a large memcpy |
449 | ! |
450 | ! ST_CHUNK batches up initial BIS operations for several cache lines |
451 | ! to allow multiple requests to not be blocked by overflowing the |
452 | ! the store miss buffer. Then the matching stores for all those |
453 | ! BIS operations are executed. |
454 | ! |
455 | |
456 | sub %o0, 8, %o0 ! adjust %o0 for ASI alignment |
457 | .Lalign_loop: |
458 | cmp %o5, ST_CHUNK*64 |
459 | blu,pt %xcc, .Lalign_loop_fin |
460 | mov ST_CHUNK,%o3 |
461 | .Lalign_loop_start: |
462 | prefetch [%o1 + (ALIGN_PRE * BLOCK_SIZE)], 21 |
463 | subcc %o3, 1, %o3 |
464 | EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_o5) |
465 | add %o1, 64, %o1 |
466 | add %o0, 8, %o0 |
467 | EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) |
468 | bgu %xcc,.Lalign_loop_start |
469 | add %o0, 56, %o0 |
470 | |
471 | mov ST_CHUNK,%o3 |
472 | sllx %o3, 6, %o4 ! ST_CHUNK*64 |
473 | sub %o1, %o4, %o1 ! reset %o1 |
474 | sub %o0, %o4, %o0 ! reset %o0 |
475 | |
476 | .Lalign_loop_rest: |
477 | EX_LD(LOAD(ldx, %o1+8, %o4), memcpy_retl_o2_plus_o5) |
478 | add %o0, 16, %o0 |
479 | EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) |
480 | EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_o5) |
481 | add %o0, 8, %o0 |
482 | EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) |
483 | subcc %o3, 1, %o3 |
484 | EX_LD(LOAD(ldx, %o1+24, %o4), memcpy_retl_o2_plus_o5) |
485 | add %o0, 8, %o0 |
486 | EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) |
487 | EX_LD(LOAD(ldx, %o1+32, %o4), memcpy_retl_o2_plus_o5) |
488 | add %o0, 8, %o0 |
489 | EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) |
490 | EX_LD(LOAD(ldx, %o1+40, %o4), memcpy_retl_o2_plus_o5) |
491 | add %o0, 8, %o0 |
492 | EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) |
493 | EX_LD(LOAD(ldx, %o1+48, %o4), memcpy_retl_o2_plus_o5) |
494 | add %o1, 64, %o1 |
495 | add %o0, 8, %o0 |
496 | EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) |
497 | add %o0, 8, %o0 |
498 | EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_o5) |
499 | sub %o5, 64, %o5 |
500 | bgu %xcc,.Lalign_loop_rest |
501 | ! mark cache line as LRU |
502 | EX_ST(STORE_INIT(%o4, %o0), memcpy_retl_o2_plus_o5_plus_64) |
503 | |
504 | cmp %o5, ST_CHUNK*64 |
505 | bgu,pt %xcc, .Lalign_loop_start |
506 | mov ST_CHUNK,%o3 |
507 | |
508 | cmp %o5, 0 |
509 | beq .Lalign_done |
510 | nop |
511 | .Lalign_loop_fin: |
512 | EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_o5) |
513 | EX_ST(STORE(stx, %o4, %o0+8), memcpy_retl_o2_plus_o5) |
514 | EX_LD(LOAD(ldx, %o1+8, %o4), memcpy_retl_o2_plus_o5) |
515 | EX_ST(STORE(stx, %o4, %o0+8+8), memcpy_retl_o2_plus_o5) |
516 | EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_o5) |
517 | EX_ST(STORE(stx, %o4, %o0+8+16), memcpy_retl_o2_plus_o5) |
518 | subcc %o5, 64, %o5 |
519 | EX_LD(LOAD(ldx, %o1+24, %o4), memcpy_retl_o2_plus_o5_64) |
520 | EX_ST(STORE(stx, %o4, %o0+8+24), memcpy_retl_o2_plus_o5_64) |
521 | EX_LD(LOAD(ldx, %o1+32, %o4), memcpy_retl_o2_plus_o5_64) |
522 | EX_ST(STORE(stx, %o4, %o0+8+32), memcpy_retl_o2_plus_o5_64) |
523 | EX_LD(LOAD(ldx, %o1+40, %o4), memcpy_retl_o2_plus_o5_64) |
524 | EX_ST(STORE(stx, %o4, %o0+8+40), memcpy_retl_o2_plus_o5_64) |
525 | EX_LD(LOAD(ldx, %o1+48, %o4), memcpy_retl_o2_plus_o5_64) |
526 | add %o1, 64, %o1 |
527 | EX_ST(STORE(stx, %o4, %o0+8+48), memcpy_retl_o2_plus_o5_64) |
528 | add %o0, 64, %o0 |
529 | EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_o5_64) |
530 | bgu %xcc,.Lalign_loop_fin |
531 | EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_o5_64) |
532 | |
533 | .Lalign_done: |
534 | add %o0, 8, %o0 ! restore %o0 from ASI alignment |
535 | membar #StoreStore |
536 | sub %o2, 63, %o2 ! adjust length to allow cc test |
537 | ba .Lmedl63 ! in .Lmedl63 |
538 | nop |
539 | |
540 | .align 16 |
541 | ! Dst is on 8 byte boundary; src is not; remaining count > SMALL_MAX |
542 | .Lunalignsetup: |
543 | .Lunalignrejoin: |
544 | mov %g1, %o3 ! save %g1 as VISEntryHalf clobbers it |
545 | #ifdef NON_USER_COPY |
546 | VISEntryHalfFast(.Lmedium_vis_entry_fail_cp) |
547 | #else |
548 | VISEntryHalf |
549 | #endif |
550 | mov %o3, %g1 ! restore %g1 |
551 | |
552 | set MED_UMAX, %o3 |
553 | cmp %o2, %o3 ! check for.Lmedium unaligned limit |
554 | bge,pt %xcc,.Lunalign_large |
555 | prefetch [%o1 + (4 * BLOCK_SIZE)], 20 |
556 | andn %o2, 0x3f, %o5 ! %o5 is multiple of block size |
557 | and %o2, 0x3f, %o2 ! residue bytes in %o2 |
558 | cmp %o2, 8 ! Insure we do not load beyond |
559 | bgt .Lunalign_adjust ! end of source buffer |
560 | andn %o1, 0x7, %o4 ! %o4 has long word aligned src address |
561 | add %o2, 64, %o2 ! adjust to leave loop |
562 | sub %o5, 64, %o5 ! early if necessary |
563 | .Lunalign_adjust: |
564 | alignaddr %o1, %g0, %g0 ! generate %gsr |
565 | add %o1, %o5, %o1 ! advance %o1 to after blocks |
566 | EX_LD_FP(LOAD(ldd, %o4, %f0), memcpy_retl_o2_plus_o5) |
567 | .Lunalign_loop: |
568 | EX_LD_FP(LOAD(ldd, %o4+8, %f2), memcpy_retl_o2_plus_o5) |
569 | faligndata %f0, %f2, %f16 |
570 | EX_LD_FP(LOAD(ldd, %o4+16, %f4), memcpy_retl_o2_plus_o5) |
571 | subcc %o5, BLOCK_SIZE, %o5 |
572 | EX_ST_FP(STORE(std, %f16, %o0), memcpy_retl_o2_plus_o5_plus_64) |
573 | faligndata %f2, %f4, %f18 |
574 | EX_LD_FP(LOAD(ldd, %o4+24, %f6), memcpy_retl_o2_plus_o5_plus_56) |
575 | EX_ST_FP(STORE(std, %f18, %o0+8), memcpy_retl_o2_plus_o5_plus_56) |
576 | faligndata %f4, %f6, %f20 |
577 | EX_LD_FP(LOAD(ldd, %o4+32, %f8), memcpy_retl_o2_plus_o5_plus_48) |
578 | EX_ST_FP(STORE(std, %f20, %o0+16), memcpy_retl_o2_plus_o5_plus_48) |
579 | faligndata %f6, %f8, %f22 |
580 | EX_LD_FP(LOAD(ldd, %o4+40, %f10), memcpy_retl_o2_plus_o5_plus_40) |
581 | EX_ST_FP(STORE(std, %f22, %o0+24), memcpy_retl_o2_plus_o5_plus_40) |
582 | faligndata %f8, %f10, %f24 |
583 | EX_LD_FP(LOAD(ldd, %o4+48, %f12), memcpy_retl_o2_plus_o5_plus_32) |
584 | EX_ST_FP(STORE(std, %f24, %o0+32), memcpy_retl_o2_plus_o5_plus_32) |
585 | faligndata %f10, %f12, %f26 |
586 | EX_LD_FP(LOAD(ldd, %o4+56, %f14), memcpy_retl_o2_plus_o5_plus_24) |
587 | add %o4, BLOCK_SIZE, %o4 |
588 | EX_ST_FP(STORE(std, %f26, %o0+40), memcpy_retl_o2_plus_o5_plus_24) |
589 | faligndata %f12, %f14, %f28 |
590 | EX_LD_FP(LOAD(ldd, %o4, %f0), memcpy_retl_o2_plus_o5_plus_16) |
591 | EX_ST_FP(STORE(std, %f28, %o0+48), memcpy_retl_o2_plus_o5_plus_16) |
592 | faligndata %f14, %f0, %f30 |
593 | EX_ST_FP(STORE(std, %f30, %o0+56), memcpy_retl_o2_plus_o5_plus_8) |
594 | add %o0, BLOCK_SIZE, %o0 |
595 | bgu,pt %xcc, .Lunalign_loop |
596 | prefetch [%o4 + (5 * BLOCK_SIZE)], 20 |
597 | ba .Lunalign_done |
598 | nop |
599 | |
600 | .Lunalign_large: |
601 | andcc %o0, 0x3f, %o3 ! is dst 64-byte block aligned? |
602 | bz %xcc, .Lunalignsrc |
603 | sub %o3, 64, %o3 ! %o3 will be multiple of 8 |
604 | neg %o3 ! bytes until dest is 64 byte aligned |
605 | sub %o2, %o3, %o2 ! update cnt with bytes to be moved |
606 | ! Move bytes according to source alignment |
607 | andcc %o1, 0x1, %o5 |
608 | bnz %xcc, .Lunalignbyte ! check for byte alignment |
609 | nop |
610 | andcc %o1, 2, %o5 ! check for half word alignment |
611 | bnz %xcc, .Lunalignhalf |
612 | nop |
613 | ! Src is word aligned |
614 | .Lunalignword: |
615 | EX_LD_FP(LOAD(ld, %o1, %o4), memcpy_retl_o2_plus_o3) ! load 4 bytes |
616 | add %o1, 8, %o1 ! increase src ptr by 8 |
617 | EX_ST_FP(STORE(stw, %o4, %o0), memcpy_retl_o2_plus_o3) ! and store 4 |
618 | subcc %o3, 8, %o3 ! decrease count by 8 |
619 | EX_LD_FP(LOAD(ld, %o1-4, %o4), memcpy_retl_o2_plus_o3_plus_4)! load 4 |
620 | add %o0, 8, %o0 ! increase dst ptr by 8 |
621 | bnz %xcc, .Lunalignword |
622 | EX_ST_FP(STORE(stw, %o4, %o0-4), memcpy_retl_o2_plus_o3_plus_4) |
623 | ba .Lunalignsrc |
624 | nop |
625 | |
626 | ! Src is half-word aligned |
627 | .Lunalignhalf: |
628 | EX_LD_FP(LOAD(lduh, %o1, %o4), memcpy_retl_o2_plus_o3) ! load 2 bytes |
629 | sllx %o4, 32, %o5 ! shift left |
630 | EX_LD_FP(LOAD(lduw, %o1+2, %o4), memcpy_retl_o2_plus_o3) |
631 | or %o4, %o5, %o5 |
632 | sllx %o5, 16, %o5 |
633 | EX_LD_FP(LOAD(lduh, %o1+6, %o4), memcpy_retl_o2_plus_o3) |
634 | or %o4, %o5, %o5 |
635 | EX_ST_FP(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_o3) |
636 | add %o1, 8, %o1 |
637 | subcc %o3, 8, %o3 |
638 | bnz %xcc, .Lunalignhalf |
639 | add %o0, 8, %o0 |
640 | ba .Lunalignsrc |
641 | nop |
642 | |
643 | ! Src is Byte aligned |
644 | .Lunalignbyte: |
645 | sub %o0, %o1, %o0 ! share pointer advance |
646 | .Lunalignbyte_loop: |
647 | EX_LD_FP(LOAD(ldub, %o1, %o4), memcpy_retl_o2_plus_o3) |
648 | sllx %o4, 56, %o5 |
649 | EX_LD_FP(LOAD(lduh, %o1+1, %o4), memcpy_retl_o2_plus_o3) |
650 | sllx %o4, 40, %o4 |
651 | or %o4, %o5, %o5 |
652 | EX_LD_FP(LOAD(lduh, %o1+3, %o4), memcpy_retl_o2_plus_o3) |
653 | sllx %o4, 24, %o4 |
654 | or %o4, %o5, %o5 |
655 | EX_LD_FP(LOAD(lduh, %o1+5, %o4), memcpy_retl_o2_plus_o3) |
656 | sllx %o4, 8, %o4 |
657 | or %o4, %o5, %o5 |
658 | EX_LD_FP(LOAD(ldub, %o1+7, %o4), memcpy_retl_o2_plus_o3) |
659 | or %o4, %o5, %o5 |
660 | add %o0, %o1, %o0 |
661 | EX_ST_FP(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_o3) |
662 | sub %o0, %o1, %o0 |
663 | subcc %o3, 8, %o3 |
664 | bnz %xcc, .Lunalignbyte_loop |
665 | add %o1, 8, %o1 |
666 | add %o0,%o1, %o0 ! restore pointer |
667 | |
668 | ! Destination is now block (64 byte aligned) |
669 | .Lunalignsrc: |
670 | andn %o2, 0x3f, %o5 ! %o5 is multiple of block size |
671 | and %o2, 0x3f, %o2 ! residue bytes in %o2 |
672 | add %o2, 64, %o2 ! Insure we do not load beyond |
673 | sub %o5, 64, %o5 ! end of source buffer |
674 | |
675 | andn %o1, 0x7, %o4 ! %o4 has long word aligned src address |
676 | alignaddr %o1, %g0, %g0 ! generate %gsr |
677 | add %o1, %o5, %o1 ! advance %o1 to after blocks |
678 | |
679 | EX_LD_FP(LOAD(ldd, %o4, %f14), memcpy_retl_o2_plus_o5) |
680 | add %o4, 8, %o4 |
681 | .Lunalign_sloop: |
682 | EX_LD_FP(LOAD(ldd, %o4, %f16), memcpy_retl_o2_plus_o5) |
683 | faligndata %f14, %f16, %f0 |
684 | EX_LD_FP(LOAD(ldd, %o4+8, %f18), memcpy_retl_o2_plus_o5) |
685 | faligndata %f16, %f18, %f2 |
686 | EX_LD_FP(LOAD(ldd, %o4+16, %f20), memcpy_retl_o2_plus_o5) |
687 | faligndata %f18, %f20, %f4 |
688 | EX_ST_FP(STORE(std, %f0, %o0), memcpy_retl_o2_plus_o5) |
689 | subcc %o5, 64, %o5 |
690 | EX_LD_FP(LOAD(ldd, %o4+24, %f22), memcpy_retl_o2_plus_o5_plus_56) |
691 | faligndata %f20, %f22, %f6 |
692 | EX_ST_FP(STORE(std, %f2, %o0+8), memcpy_retl_o2_plus_o5_plus_56) |
693 | EX_LD_FP(LOAD(ldd, %o4+32, %f24), memcpy_retl_o2_plus_o5_plus_48) |
694 | faligndata %f22, %f24, %f8 |
695 | EX_ST_FP(STORE(std, %f4, %o0+16), memcpy_retl_o2_plus_o5_plus_48) |
696 | EX_LD_FP(LOAD(ldd, %o4+40, %f26), memcpy_retl_o2_plus_o5_plus_40) |
697 | faligndata %f24, %f26, %f10 |
698 | EX_ST_FP(STORE(std, %f6, %o0+24), memcpy_retl_o2_plus_o5_plus_40) |
699 | EX_LD_FP(LOAD(ldd, %o4+48, %f28), memcpy_retl_o2_plus_o5_plus_40) |
700 | faligndata %f26, %f28, %f12 |
701 | EX_ST_FP(STORE(std, %f8, %o0+32), memcpy_retl_o2_plus_o5_plus_40) |
702 | add %o4, 64, %o4 |
703 | EX_LD_FP(LOAD(ldd, %o4-8, %f30), memcpy_retl_o2_plus_o5_plus_40) |
704 | faligndata %f28, %f30, %f14 |
705 | EX_ST_FP(STORE(std, %f10, %o0+40), memcpy_retl_o2_plus_o5_plus_40) |
706 | EX_ST_FP(STORE(std, %f12, %o0+48), memcpy_retl_o2_plus_o5_plus_40) |
707 | add %o0, 64, %o0 |
708 | EX_ST_FP(STORE(std, %f14, %o0-8), memcpy_retl_o2_plus_o5_plus_40) |
709 | fsrc2 %f30, %f14 |
710 | bgu,pt %xcc, .Lunalign_sloop |
711 | prefetch [%o4 + (8 * BLOCK_SIZE)], 20 |
712 | |
713 | .Lunalign_done: |
714 | ! Handle trailing bytes, 64 to 127 |
715 | ! Dest long word aligned, Src not long word aligned |
716 | cmp %o2, 15 |
717 | bleu %xcc, .Lunalign_short |
718 | |
719 | andn %o2, 0x7, %o5 ! %o5 is multiple of 8 |
720 | and %o2, 0x7, %o2 ! residue bytes in %o2 |
721 | add %o2, 8, %o2 |
722 | sub %o5, 8, %o5 ! insure we do not load past end of src |
723 | andn %o1, 0x7, %o4 ! %o4 has long word aligned src address |
724 | add %o1, %o5, %o1 ! advance %o1 to after multiple of 8 |
725 | EX_LD_FP(LOAD(ldd, %o4, %f0), memcpy_retl_o2_plus_o5)! fetch partialword |
726 | .Lunalign_by8: |
727 | EX_LD_FP(LOAD(ldd, %o4+8, %f2), memcpy_retl_o2_plus_o5) |
728 | add %o4, 8, %o4 |
729 | faligndata %f0, %f2, %f16 |
730 | subcc %o5, 8, %o5 |
731 | EX_ST_FP(STORE(std, %f16, %o0), memcpy_retl_o2_plus_o5) |
732 | fsrc2 %f2, %f0 |
733 | bgu,pt %xcc, .Lunalign_by8 |
734 | add %o0, 8, %o0 |
735 | |
736 | .Lunalign_short: |
737 | #ifdef NON_USER_COPY |
738 | VISExitHalfFast |
739 | #else |
740 | VISExitHalf |
741 | #endif |
742 | ba .Lsmallrest |
743 | nop |
744 | |
745 | /* |
746 | * This is a special case of nested memcpy. This can happen when kernel |
747 | * calls unaligned memcpy back to back without saving FP registers. We need |
748 | * traps(context switch) to save/restore FP registers. If the kernel calls |
749 | * memcpy without this trap sequence we will hit FP corruption. Let's use |
750 | * the normal integer load/store method in this case. |
751 | */ |
752 | |
753 | #ifdef NON_USER_COPY |
754 | .Lmedium_vis_entry_fail_cp: |
755 | or %o0, %o1, %g2 |
756 | #endif |
757 | .Lmedium_cp: |
758 | LOAD(prefetch, %o1 + 0x40, #n_reads_strong) |
759 | andcc %g2, 0x7, %g0 |
760 | bne,pn %xcc, .Lmedium_unaligned_cp |
761 | nop |
762 | |
763 | .Lmedium_noprefetch_cp: |
764 | andncc %o2, 0x20 - 1, %o5 |
765 | be,pn %xcc, 2f |
766 | sub %o2, %o5, %o2 |
767 | 1: EX_LD(LOAD(ldx, %o1 + 0x00, %o3), memcpy_retl_o2_plus_o5) |
768 | EX_LD(LOAD(ldx, %o1 + 0x08, %g2), memcpy_retl_o2_plus_o5) |
769 | EX_LD(LOAD(ldx, %o1 + 0x10, %g7), memcpy_retl_o2_plus_o5) |
770 | EX_LD(LOAD(ldx, %o1 + 0x18, %o4), memcpy_retl_o2_plus_o5) |
771 | add %o1, 0x20, %o1 |
772 | subcc %o5, 0x20, %o5 |
773 | EX_ST(STORE(stx, %o3, %o0 + 0x00), memcpy_retl_o2_plus_o5_plus_32) |
774 | EX_ST(STORE(stx, %g2, %o0 + 0x08), memcpy_retl_o2_plus_o5_plus_24) |
775 | EX_ST(STORE(stx, %g7, %o0 + 0x10), memcpy_retl_o2_plus_o5_plus_24) |
776 | EX_ST(STORE(stx, %o4, %o0 + 0x18), memcpy_retl_o2_plus_o5_plus_8) |
777 | bne,pt %xcc, 1b |
778 | add %o0, 0x20, %o0 |
779 | 2: andcc %o2, 0x18, %o5 |
780 | be,pt %xcc, 3f |
781 | sub %o2, %o5, %o2 |
782 | 1: EX_LD(LOAD(ldx, %o1 + 0x00, %o3), memcpy_retl_o2_plus_o5) |
783 | add %o1, 0x08, %o1 |
784 | add %o0, 0x08, %o0 |
785 | subcc %o5, 0x08, %o5 |
786 | bne,pt %xcc, 1b |
787 | EX_ST(STORE(stx, %o3, %o0 - 0x08), memcpy_retl_o2_plus_o5_plus_8) |
788 | 3: brz,pt %o2, .Lexit_cp |
789 | cmp %o2, 0x04 |
790 | bl,pn %xcc, .Ltiny_cp |
791 | nop |
792 | EX_LD(LOAD(lduw, %o1 + 0x00, %o3), memcpy_retl_o2) |
793 | add %o1, 0x04, %o1 |
794 | add %o0, 0x04, %o0 |
795 | subcc %o2, 0x04, %o2 |
796 | bne,pn %xcc, .Ltiny_cp |
797 | EX_ST(STORE(stw, %o3, %o0 - 0x04), memcpy_retl_o2_plus_4) |
798 | ba,a,pt %xcc, .Lexit_cp |
799 | |
800 | .Lmedium_unaligned_cp: |
801 | /* First get dest 8 byte aligned. */ |
802 | sub %g0, %o0, %o3 |
803 | and %o3, 0x7, %o3 |
804 | brz,pt %o3, 2f |
805 | sub %o2, %o3, %o2 |
806 | |
807 | 1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), memcpy_retl_o2_plus_g1) |
808 | add %o1, 1, %o1 |
809 | subcc %o3, 1, %o3 |
810 | add %o0, 1, %o0 |
811 | bne,pt %xcc, 1b |
812 | EX_ST(STORE(stb, %g2, %o0 - 0x01), memcpy_retl_o2_plus_g1_plus_1) |
813 | 2: |
814 | and %o1, 0x7, %o3 |
815 | brz,pn %o3, .Lmedium_noprefetch_cp |
816 | sll %o3, 3, %o3 |
817 | mov 64, %g2 |
818 | sub %g2, %o3, %g2 |
819 | andn %o1, 0x7, %o1 |
820 | EX_LD(LOAD(ldx, %o1 + 0x00, %o4), memcpy_retl_o2) |
821 | sllx %o4, %o3, %o4 |
822 | andn %o2, 0x08 - 1, %o5 |
823 | sub %o2, %o5, %o2 |
824 | |
825 | 1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3), memcpy_retl_o2_plus_o5) |
826 | add %o1, 0x08, %o1 |
827 | subcc %o5, 0x08, %o5 |
828 | srlx %g3, %g2, %g7 |
829 | or %g7, %o4, %g7 |
830 | EX_ST(STORE(stx, %g7, %o0 + 0x00), memcpy_retl_o2_plus_o5_plus_8) |
831 | add %o0, 0x08, %o0 |
832 | bne,pt %xcc, 1b |
833 | sllx %g3, %o3, %o4 |
834 | srl %o3, 3, %o3 |
835 | add %o1, %o3, %o1 |
836 | brz,pn %o2, .Lexit_cp |
837 | nop |
838 | ba,pt %xcc, .Lsmall_unaligned_cp |
839 | |
840 | .Ltiny_cp: |
841 | EX_LD(LOAD(ldub, %o1 + 0x00, %o3), memcpy_retl_o2) |
842 | subcc %o2, 1, %o2 |
843 | be,pn %xcc, .Lexit_cp |
844 | EX_ST(STORE(stb, %o3, %o0 + 0x00), memcpy_retl_o2_plus_1) |
845 | EX_LD(LOAD(ldub, %o1 + 0x01, %o3), memcpy_retl_o2) |
846 | subcc %o2, 1, %o2 |
847 | be,pn %xcc, .Lexit_cp |
848 | EX_ST(STORE(stb, %o3, %o0 + 0x01), memcpy_retl_o2_plus_1) |
849 | EX_LD(LOAD(ldub, %o1 + 0x02, %o3), memcpy_retl_o2) |
850 | ba,pt %xcc, .Lexit_cp |
851 | EX_ST(STORE(stb, %o3, %o0 + 0x02), memcpy_retl_o2) |
852 | |
853 | .Lsmall_cp: |
854 | andcc %g2, 0x3, %g0 |
855 | bne,pn %xcc, .Lsmall_unaligned_cp |
856 | andn %o2, 0x4 - 1, %o5 |
857 | sub %o2, %o5, %o2 |
858 | 1: |
859 | EX_LD(LOAD(lduw, %o1 + 0x00, %o3), memcpy_retl_o2_plus_o5) |
860 | add %o1, 0x04, %o1 |
861 | subcc %o5, 0x04, %o5 |
862 | add %o0, 0x04, %o0 |
863 | bne,pt %xcc, 1b |
864 | EX_ST(STORE(stw, %o3, %o0 - 0x04), memcpy_retl_o2_plus_o5_plus_4) |
865 | brz,pt %o2, .Lexit_cp |
866 | nop |
867 | ba,a,pt %xcc, .Ltiny_cp |
868 | |
869 | .Lsmall_unaligned_cp: |
870 | 1: EX_LD(LOAD(ldub, %o1 + 0x00, %o3), memcpy_retl_o2) |
871 | add %o1, 1, %o1 |
872 | add %o0, 1, %o0 |
873 | subcc %o2, 1, %o2 |
874 | bne,pt %xcc, 1b |
875 | EX_ST(STORE(stb, %o3, %o0 - 0x01), memcpy_retl_o2_plus_1) |
876 | ba,a,pt %xcc, .Lexit_cp |
877 | |
878 | .Lsmallrest: |
879 | tst %o2 |
880 | bz,pt %xcc, .Lsmallx |
881 | cmp %o2, 4 |
882 | blt,pn %xcc, .Lsmallleft3 |
883 | nop |
884 | sub %o2, 3, %o2 |
885 | .Lsmallnotalign4: |
886 | EX_LD(LOAD(ldub, %o1, %o3), memcpy_retl_o2_plus_3)! read byte |
887 | subcc %o2, 4, %o2 ! reduce count by 4 |
888 | EX_ST(STORE(stb, %o3, %o0), memcpy_retl_o2_plus_7)! write byte & repeat |
889 | EX_LD(LOAD(ldub, %o1+1, %o3), memcpy_retl_o2_plus_6)! for total of 4 |
890 | add %o1, 4, %o1 ! advance SRC by 4 |
891 | EX_ST(STORE(stb, %o3, %o0+1), memcpy_retl_o2_plus_6) |
892 | EX_LD(LOAD(ldub, %o1-2, %o3), memcpy_retl_o2_plus_5) |
893 | add %o0, 4, %o0 ! advance DST by 4 |
894 | EX_ST(STORE(stb, %o3, %o0-2), memcpy_retl_o2_plus_5) |
895 | EX_LD(LOAD(ldub, %o1-1, %o3), memcpy_retl_o2_plus_4) |
896 | bgu,pt %xcc, .Lsmallnotalign4 ! loop til 3 or fewer bytes remain |
897 | EX_ST(STORE(stb, %o3, %o0-1), memcpy_retl_o2_plus_4) |
898 | addcc %o2, 3, %o2 ! restore count |
899 | bz,pt %xcc, .Lsmallx |
900 | .Lsmallleft3: ! 1, 2, or 3 bytes remain |
901 | subcc %o2, 1, %o2 |
902 | EX_LD(LOAD(ldub, %o1, %o3), memcpy_retl_o2_plus_1) ! load one byte |
903 | bz,pt %xcc, .Lsmallx |
904 | EX_ST(STORE(stb, %o3, %o0), memcpy_retl_o2_plus_1) ! store one byte |
905 | EX_LD(LOAD(ldub, %o1+1, %o3), memcpy_retl_o2) ! load second byte |
906 | subcc %o2, 1, %o2 |
907 | bz,pt %xcc, .Lsmallx |
908 | EX_ST(STORE(stb, %o3, %o0+1), memcpy_retl_o2_plus_1)! store second byte |
909 | EX_LD(LOAD(ldub, %o1+2, %o3), memcpy_retl_o2) ! load third byte |
910 | EX_ST(STORE(stb, %o3, %o0+2), memcpy_retl_o2) ! store third byte |
911 | .Lsmallx: |
912 | retl |
913 | mov EX_RETVAL(%g1), %o0 |
914 | .Lsmallfin: |
915 | tst %o2 |
916 | bnz,pn %xcc, .Lsmallleft3 |
917 | nop |
918 | retl |
919 | mov EX_RETVAL(%g1), %o0 ! restore %o0 |
920 | .Lexit_cp: |
921 | retl |
922 | mov EX_RETVAL(%g1), %o0 |
923 | .size FUNC_NAME, .-FUNC_NAME |
924 | |