1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * arch/alpha/lib/ev6-clear_user.S
4 * 21264 version contributed by Rick Gorton <rick.gorton@alpha-processor.com>
5 *
6 * Zero user space, handling exceptions as we go.
7 *
8 * We have to make sure that $0 is always up-to-date and contains the
9 * right "bytes left to zero" value (and that it is updated only _after_
10 * a successful copy). There is also some rather minor exception setup
11 * stuff.
12 *
13 * Much of the information about 21264 scheduling/coding comes from:
14 * Compiler Writer's Guide for the Alpha 21264
15 * abbreviated as 'CWG' in other comments here
16 * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html
17 * Scheduling notation:
18 * E - either cluster
19 * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1
20 * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1
21 * Try not to change the actual algorithm if possible for consistency.
22 * Determining actual stalls (other than slotting) doesn't appear to be easy to do.
23 * From perusing the source code context where this routine is called, it is
24 * a fair assumption that significant fractions of entire pages are zeroed, so
25 * it's going to be worth the effort to hand-unroll a big loop, and use wh64.
26 * ASSUMPTION:
27 * The believed purpose of only updating $0 after a store is that a signal
28 * may come along during the execution of this chunk of code, and we don't
29 * want to leave a hole (and we also want to avoid repeating lots of work)
30 */
31
32#include <linux/export.h>
33/* Allow an exception for an insn; exit if we get one. */
34#define EX(x,y...) \
35 99: x,##y; \
36 .section __ex_table,"a"; \
37 .long 99b - .; \
38 lda $31, $exception-99b($31); \
39 .previous
40
41 .set noat
42 .set noreorder
43 .align 4
44
45 .globl __clear_user
46 .ent __clear_user
47 .frame $30, 0, $26
48 .prologue 0
49
50 # Pipeline info : Slotting & Comments
51__clear_user:
52 and $17, $17, $0
53 and $16, 7, $4 # .. E .. .. : find dest head misalignment
54 beq $0, $zerolength # U .. .. .. : U L U L
55
56 addq $0, $4, $1 # .. .. .. E : bias counter
57 and $1, 7, $2 # .. .. E .. : number of misaligned bytes in tail
58# Note - we never actually use $2, so this is a moot computation
59# and we can rewrite this later...
60 srl $1, 3, $1 # .. E .. .. : number of quadwords to clear
61 beq $4, $headalign # U .. .. .. : U L U L
62
63/*
64 * Head is not aligned. Write (8 - $4) bytes to head of destination
65 * This means $16 is known to be misaligned
66 */
67 EX( ldq_u $5, 0($16) ) # .. .. .. L : load dst word to mask back in
68 beq $1, $onebyte # .. .. U .. : sub-word store?
69 mskql $5, $16, $5 # .. U .. .. : take care of misaligned head
70 addq $16, 8, $16 # E .. .. .. : L U U L
71
72 EX( stq_u $5, -8($16) ) # .. .. .. L :
73 subq $1, 1, $1 # .. .. E .. :
74 addq $0, $4, $0 # .. E .. .. : bytes left -= 8 - misalignment
75 subq $0, 8, $0 # E .. .. .. : U L U L
76
77 .align 4
78/*
79 * (The .align directive ought to be a moot point)
80 * values upon initial entry to the loop
81 * $1 is number of quadwords to clear (zero is a valid value)
82 * $2 is number of trailing bytes (0..7) ($2 never used...)
83 * $16 is known to be aligned 0mod8
84 */
85$headalign:
86 subq $1, 16, $4 # .. .. .. E : If < 16, we can not use the huge loop
87 and $16, 0x3f, $2 # .. .. E .. : Forward work for huge loop
88 subq $2, 0x40, $3 # .. E .. .. : bias counter (huge loop)
89 blt $4, $trailquad # U .. .. .. : U L U L
90
91/*
92 * We know that we're going to do at least 16 quads, which means we are
93 * going to be able to use the large block clear loop at least once.
94 * Figure out how many quads we need to clear before we are 0mod64 aligned
95 * so we can use the wh64 instruction.
96 */
97
98 nop # .. .. .. E
99 nop # .. .. E ..
100 nop # .. E .. ..
101 beq $3, $bigalign # U .. .. .. : U L U L : Aligned 0mod64
102
103$alignmod64:
104 EX( stq_u $31, 0($16) ) # .. .. .. L
105 addq $3, 8, $3 # .. .. E ..
106 subq $0, 8, $0 # .. E .. ..
107 nop # E .. .. .. : U L U L
108
109 nop # .. .. .. E
110 subq $1, 1, $1 # .. .. E ..
111 addq $16, 8, $16 # .. E .. ..
112 blt $3, $alignmod64 # U .. .. .. : U L U L
113
114$bigalign:
115/*
116 * $0 is the number of bytes left
117 * $1 is the number of quads left
118 * $16 is aligned 0mod64
119 * we know that we'll be taking a minimum of one trip through
120 * CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle
121 * We are _not_ going to update $0 after every single store. That
122 * would be silly, because there will be cross-cluster dependencies
123 * no matter how the code is scheduled. By doing it in slightly
124 * staggered fashion, we can still do this loop in 5 fetches
125 * The worse case will be doing two extra quads in some future execution,
126 * in the event of an interrupted clear.
127 * Assumes the wh64 needs to be for 2 trips through the loop in the future
128 * The wh64 is issued on for the starting destination address for trip +2
129 * through the loop, and if there are less than two trips left, the target
130 * address will be for the current trip.
131 */
132 nop # E :
133 nop # E :
134 nop # E :
135 bis $16,$16,$3 # E : U L U L : Initial wh64 address is dest
136 /* This might actually help for the current trip... */
137
138$do_wh64:
139 wh64 ($3) # .. .. .. L1 : memory subsystem hint
140 subq $1, 16, $4 # .. .. E .. : Forward calculation - repeat the loop?
141 EX( stq_u $31, 0($16) ) # .. L .. ..
142 subq $0, 8, $0 # E .. .. .. : U L U L
143
144 addq $16, 128, $3 # E : Target address of wh64
145 EX( stq_u $31, 8($16) ) # L :
146 EX( stq_u $31, 16($16) ) # L :
147 subq $0, 16, $0 # E : U L L U
148
149 nop # E :
150 EX( stq_u $31, 24($16) ) # L :
151 EX( stq_u $31, 32($16) ) # L :
152 subq $0, 168, $5 # E : U L L U : two trips through the loop left?
153 /* 168 = 192 - 24, since we've already completed some stores */
154
155 subq $0, 16, $0 # E :
156 EX( stq_u $31, 40($16) ) # L :
157 EX( stq_u $31, 48($16) ) # L :
158 cmovlt $5, $16, $3 # E : U L L U : Latency 2, extra mapping cycle
159
160 subq $1, 8, $1 # E :
161 subq $0, 16, $0 # E :
162 EX( stq_u $31, 56($16) ) # L :
163 nop # E : U L U L
164
165 nop # E :
166 subq $0, 8, $0 # E :
167 addq $16, 64, $16 # E :
168 bge $4, $do_wh64 # U : U L U L
169
170$trailquad:
171 # zero to 16 quadwords left to store, plus any trailing bytes
172 # $1 is the number of quadwords left to go.
173 #
174 nop # .. .. .. E
175 nop # .. .. E ..
176 nop # .. E .. ..
177 beq $1, $trailbytes # U .. .. .. : U L U L : Only 0..7 bytes to go
178
179$onequad:
180 EX( stq_u $31, 0($16) ) # .. .. .. L
181 subq $1, 1, $1 # .. .. E ..
182 subq $0, 8, $0 # .. E .. ..
183 nop # E .. .. .. : U L U L
184
185 nop # .. .. .. E
186 nop # .. .. E ..
187 addq $16, 8, $16 # .. E .. ..
188 bgt $1, $onequad # U .. .. .. : U L U L
189
190 # We have an unknown number of bytes left to go.
191$trailbytes:
192 nop # .. .. .. E
193 nop # .. .. E ..
194 nop # .. E .. ..
195 beq $0, $zerolength # U .. .. .. : U L U L
196
197 # $0 contains the number of bytes left to copy (0..31)
198 # so we will use $0 as the loop counter
199 # We know for a fact that $0 > 0 zero due to previous context
200$onebyte:
201 EX( stb $31, 0($16) ) # .. .. .. L
202 subq $0, 1, $0 # .. .. E .. :
203 addq $16, 1, $16 # .. E .. .. :
204 bgt $0, $onebyte # U .. .. .. : U L U L
205
206$zerolength:
207$exception: # Destination for exception recovery(?)
208 nop # .. .. .. E :
209 nop # .. .. E .. :
210 nop # .. E .. .. :
211 ret $31, ($26), 1 # L0 .. .. .. : L U L U
212 .end __clear_user
213 EXPORT_SYMBOL(__clear_user)
214

source code of linux/arch/alpha/lib/ev6-clear_user.S