1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* |
3 | * hypersparc.S: High speed Hypersparc mmu/cache operations. |
4 | * |
5 | * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) |
6 | */ |
7 | |
8 | #include <asm/ptrace.h> |
9 | #include <asm/psr.h> |
10 | #include <asm/asm-offsets.h> |
11 | #include <asm/asi.h> |
12 | #include <asm/page.h> |
13 | #include <asm/pgtable.h> |
14 | #include <asm/pgtsrmmu.h> |
15 | #include <linux/init.h> |
16 | |
17 | .text |
18 | .align 4 |
19 | |
20 | .globl hypersparc_flush_cache_all, hypersparc_flush_cache_mm |
21 | .globl hypersparc_flush_cache_range, hypersparc_flush_cache_page |
22 | .globl hypersparc_flush_page_to_ram |
23 | .globl hypersparc_flush_page_for_dma, hypersparc_flush_sig_insns |
24 | .globl hypersparc_flush_tlb_all, hypersparc_flush_tlb_mm |
25 | .globl hypersparc_flush_tlb_range, hypersparc_flush_tlb_page |
26 | |
27 | hypersparc_flush_cache_all: |
28 | WINDOW_FLUSH(%g4, %g5) |
29 | sethi %hi(vac_cache_size), %g4 |
30 | ld [%g4 + %lo(vac_cache_size)], %g5 |
31 | sethi %hi(vac_line_size), %g1 |
32 | ld [%g1 + %lo(vac_line_size)], %g2 |
33 | 1: |
34 | subcc %g5, %g2, %g5 ! hyper_flush_unconditional_combined |
35 | bne 1b |
36 | sta %g0, [%g5] ASI_M_FLUSH_CTX |
37 | retl |
38 | sta %g0, [%g0] ASI_M_FLUSH_IWHOLE ! hyper_flush_whole_icache |
39 | |
40 | /* We expand the window flush to get maximum performance. */ |
41 | hypersparc_flush_cache_mm: |
42 | #ifndef CONFIG_SMP |
43 | ld [%o0 + AOFF_mm_context], %g1 |
44 | cmp %g1, -1 |
45 | be hypersparc_flush_cache_mm_out |
46 | #endif |
47 | WINDOW_FLUSH(%g4, %g5) |
48 | |
49 | sethi %hi(vac_line_size), %g1 |
50 | ld [%g1 + %lo(vac_line_size)], %o1 |
51 | sethi %hi(vac_cache_size), %g2 |
52 | ld [%g2 + %lo(vac_cache_size)], %o0 |
53 | add %o1, %o1, %g1 |
54 | add %o1, %g1, %g2 |
55 | add %o1, %g2, %g3 |
56 | add %o1, %g3, %g4 |
57 | add %o1, %g4, %g5 |
58 | add %o1, %g5, %o4 |
59 | add %o1, %o4, %o5 |
60 | |
61 | /* BLAMMO! */ |
62 | 1: |
63 | subcc %o0, %o5, %o0 ! hyper_flush_cache_user |
64 | sta %g0, [%o0 + %g0] ASI_M_FLUSH_USER |
65 | sta %g0, [%o0 + %o1] ASI_M_FLUSH_USER |
66 | sta %g0, [%o0 + %g1] ASI_M_FLUSH_USER |
67 | sta %g0, [%o0 + %g2] ASI_M_FLUSH_USER |
68 | sta %g0, [%o0 + %g3] ASI_M_FLUSH_USER |
69 | sta %g0, [%o0 + %g4] ASI_M_FLUSH_USER |
70 | sta %g0, [%o0 + %g5] ASI_M_FLUSH_USER |
71 | bne 1b |
72 | sta %g0, [%o0 + %o4] ASI_M_FLUSH_USER |
73 | hypersparc_flush_cache_mm_out: |
74 | retl |
75 | nop |
76 | |
77 | /* The things we do for performance... */ |
78 | hypersparc_flush_cache_range: |
79 | ld [%o0 + VMA_VM_MM], %o0 |
80 | #ifndef CONFIG_SMP |
81 | ld [%o0 + AOFF_mm_context], %g1 |
82 | cmp %g1, -1 |
83 | be hypersparc_flush_cache_range_out |
84 | #endif |
85 | WINDOW_FLUSH(%g4, %g5) |
86 | |
87 | sethi %hi(vac_line_size), %g1 |
88 | ld [%g1 + %lo(vac_line_size)], %o4 |
89 | sethi %hi(vac_cache_size), %g2 |
90 | ld [%g2 + %lo(vac_cache_size)], %o3 |
91 | |
92 | /* Here comes the fun part... */ |
93 | add %o2, (PAGE_SIZE - 1), %o2 |
94 | andn %o1, (PAGE_SIZE - 1), %o1 |
95 | add %o4, %o4, %o5 |
96 | andn %o2, (PAGE_SIZE - 1), %o2 |
97 | add %o4, %o5, %g1 |
98 | sub %o2, %o1, %g4 |
99 | add %o4, %g1, %g2 |
100 | sll %o3, 2, %g5 |
101 | add %o4, %g2, %g3 |
102 | cmp %g4, %g5 |
103 | add %o4, %g3, %g4 |
104 | blu 0f |
105 | add %o4, %g4, %g5 |
106 | add %o4, %g5, %g7 |
107 | |
108 | /* Flush entire user space, believe it or not this is quicker |
109 | * than page at a time flushings for range > (cache_size<<2). |
110 | */ |
111 | 1: |
112 | subcc %o3, %g7, %o3 |
113 | sta %g0, [%o3 + %g0] ASI_M_FLUSH_USER |
114 | sta %g0, [%o3 + %o4] ASI_M_FLUSH_USER |
115 | sta %g0, [%o3 + %o5] ASI_M_FLUSH_USER |
116 | sta %g0, [%o3 + %g1] ASI_M_FLUSH_USER |
117 | sta %g0, [%o3 + %g2] ASI_M_FLUSH_USER |
118 | sta %g0, [%o3 + %g3] ASI_M_FLUSH_USER |
119 | sta %g0, [%o3 + %g4] ASI_M_FLUSH_USER |
120 | bne 1b |
121 | sta %g0, [%o3 + %g5] ASI_M_FLUSH_USER |
122 | retl |
123 | nop |
124 | |
125 | /* Below our threshold, flush one page at a time. */ |
126 | 0: |
127 | ld [%o0 + AOFF_mm_context], %o0 |
128 | mov SRMMU_CTX_REG, %g7 |
129 | lda [%g7] ASI_M_MMUREGS, %o3 |
130 | sta %o0, [%g7] ASI_M_MMUREGS |
131 | add %o2, -PAGE_SIZE, %o0 |
132 | 1: |
133 | or %o0, 0x400, %g7 |
134 | lda [%g7] ASI_M_FLUSH_PROBE, %g7 |
135 | orcc %g7, 0, %g0 |
136 | be,a 3f |
137 | mov %o0, %o2 |
138 | add %o4, %g5, %g7 |
139 | 2: |
140 | sub %o2, %g7, %o2 |
141 | sta %g0, [%o2 + %g0] ASI_M_FLUSH_PAGE |
142 | sta %g0, [%o2 + %o4] ASI_M_FLUSH_PAGE |
143 | sta %g0, [%o2 + %o5] ASI_M_FLUSH_PAGE |
144 | sta %g0, [%o2 + %g1] ASI_M_FLUSH_PAGE |
145 | sta %g0, [%o2 + %g2] ASI_M_FLUSH_PAGE |
146 | sta %g0, [%o2 + %g3] ASI_M_FLUSH_PAGE |
147 | andcc %o2, 0xffc, %g0 |
148 | sta %g0, [%o2 + %g4] ASI_M_FLUSH_PAGE |
149 | bne 2b |
150 | sta %g0, [%o2 + %g5] ASI_M_FLUSH_PAGE |
151 | 3: |
152 | cmp %o2, %o1 |
153 | bne 1b |
154 | add %o2, -PAGE_SIZE, %o0 |
155 | mov SRMMU_FAULT_STATUS, %g5 |
156 | lda [%g5] ASI_M_MMUREGS, %g0 |
157 | mov SRMMU_CTX_REG, %g7 |
158 | sta %o3, [%g7] ASI_M_MMUREGS |
159 | hypersparc_flush_cache_range_out: |
160 | retl |
161 | nop |
162 | |
163 | /* HyperSparc requires a valid mapping where we are about to flush |
164 | * in order to check for a physical tag match during the flush. |
165 | */ |
166 | /* Verified, my ass... */ |
167 | hypersparc_flush_cache_page: |
168 | ld [%o0 + VMA_VM_MM], %o0 |
169 | ld [%o0 + AOFF_mm_context], %g2 |
170 | #ifndef CONFIG_SMP |
171 | cmp %g2, -1 |
172 | be hypersparc_flush_cache_page_out |
173 | #endif |
174 | WINDOW_FLUSH(%g4, %g5) |
175 | |
176 | sethi %hi(vac_line_size), %g1 |
177 | ld [%g1 + %lo(vac_line_size)], %o4 |
178 | mov SRMMU_CTX_REG, %o3 |
179 | andn %o1, (PAGE_SIZE - 1), %o1 |
180 | lda [%o3] ASI_M_MMUREGS, %o2 |
181 | sta %g2, [%o3] ASI_M_MMUREGS |
182 | or %o1, 0x400, %o5 |
183 | lda [%o5] ASI_M_FLUSH_PROBE, %g1 |
184 | orcc %g0, %g1, %g0 |
185 | be 2f |
186 | add %o4, %o4, %o5 |
187 | sub %o1, -PAGE_SIZE, %o1 |
188 | add %o4, %o5, %g1 |
189 | add %o4, %g1, %g2 |
190 | add %o4, %g2, %g3 |
191 | add %o4, %g3, %g4 |
192 | add %o4, %g4, %g5 |
193 | add %o4, %g5, %g7 |
194 | |
195 | /* BLAMMO! */ |
196 | 1: |
197 | sub %o1, %g7, %o1 |
198 | sta %g0, [%o1 + %g0] ASI_M_FLUSH_PAGE |
199 | sta %g0, [%o1 + %o4] ASI_M_FLUSH_PAGE |
200 | sta %g0, [%o1 + %o5] ASI_M_FLUSH_PAGE |
201 | sta %g0, [%o1 + %g1] ASI_M_FLUSH_PAGE |
202 | sta %g0, [%o1 + %g2] ASI_M_FLUSH_PAGE |
203 | sta %g0, [%o1 + %g3] ASI_M_FLUSH_PAGE |
204 | andcc %o1, 0xffc, %g0 |
205 | sta %g0, [%o1 + %g4] ASI_M_FLUSH_PAGE |
206 | bne 1b |
207 | sta %g0, [%o1 + %g5] ASI_M_FLUSH_PAGE |
208 | 2: |
209 | mov SRMMU_FAULT_STATUS, %g7 |
210 | mov SRMMU_CTX_REG, %g4 |
211 | lda [%g7] ASI_M_MMUREGS, %g0 |
212 | sta %o2, [%g4] ASI_M_MMUREGS |
213 | hypersparc_flush_cache_page_out: |
214 | retl |
215 | nop |
216 | |
217 | hypersparc_flush_sig_insns: |
218 | flush %o1 |
219 | retl |
220 | flush %o1 + 4 |
221 | |
222 | /* HyperSparc is copy-back. */ |
223 | hypersparc_flush_page_to_ram: |
224 | sethi %hi(vac_line_size), %g1 |
225 | ld [%g1 + %lo(vac_line_size)], %o4 |
226 | andn %o0, (PAGE_SIZE - 1), %o0 |
227 | add %o4, %o4, %o5 |
228 | or %o0, 0x400, %g7 |
229 | lda [%g7] ASI_M_FLUSH_PROBE, %g5 |
230 | add %o4, %o5, %g1 |
231 | orcc %g5, 0, %g0 |
232 | be 2f |
233 | add %o4, %g1, %g2 |
234 | add %o4, %g2, %g3 |
235 | sub %o0, -PAGE_SIZE, %o0 |
236 | add %o4, %g3, %g4 |
237 | add %o4, %g4, %g5 |
238 | add %o4, %g5, %g7 |
239 | |
240 | /* BLAMMO! */ |
241 | 1: |
242 | sub %o0, %g7, %o0 |
243 | sta %g0, [%o0 + %g0] ASI_M_FLUSH_PAGE |
244 | sta %g0, [%o0 + %o4] ASI_M_FLUSH_PAGE |
245 | sta %g0, [%o0 + %o5] ASI_M_FLUSH_PAGE |
246 | sta %g0, [%o0 + %g1] ASI_M_FLUSH_PAGE |
247 | sta %g0, [%o0 + %g2] ASI_M_FLUSH_PAGE |
248 | sta %g0, [%o0 + %g3] ASI_M_FLUSH_PAGE |
249 | andcc %o0, 0xffc, %g0 |
250 | sta %g0, [%o0 + %g4] ASI_M_FLUSH_PAGE |
251 | bne 1b |
252 | sta %g0, [%o0 + %g5] ASI_M_FLUSH_PAGE |
253 | 2: |
254 | mov SRMMU_FAULT_STATUS, %g1 |
255 | retl |
256 | lda [%g1] ASI_M_MMUREGS, %g0 |
257 | |
258 | /* HyperSparc is IO cache coherent. */ |
259 | hypersparc_flush_page_for_dma: |
260 | retl |
261 | nop |
262 | |
263 | /* It was noted that at boot time a TLB flush all in a delay slot |
264 | * can deliver an illegal instruction to the processor if the timing |
265 | * is just right... |
266 | */ |
267 | hypersparc_flush_tlb_all: |
268 | mov 0x400, %g1 |
269 | sta %g0, [%g1] ASI_M_FLUSH_PROBE |
270 | retl |
271 | nop |
272 | |
273 | hypersparc_flush_tlb_mm: |
274 | mov SRMMU_CTX_REG, %g1 |
275 | ld [%o0 + AOFF_mm_context], %o1 |
276 | lda [%g1] ASI_M_MMUREGS, %g5 |
277 | #ifndef CONFIG_SMP |
278 | cmp %o1, -1 |
279 | be hypersparc_flush_tlb_mm_out |
280 | #endif |
281 | mov 0x300, %g2 |
282 | sta %o1, [%g1] ASI_M_MMUREGS |
283 | sta %g0, [%g2] ASI_M_FLUSH_PROBE |
284 | hypersparc_flush_tlb_mm_out: |
285 | retl |
286 | sta %g5, [%g1] ASI_M_MMUREGS |
287 | |
288 | hypersparc_flush_tlb_range: |
289 | ld [%o0 + VMA_VM_MM], %o0 |
290 | mov SRMMU_CTX_REG, %g1 |
291 | ld [%o0 + AOFF_mm_context], %o3 |
292 | lda [%g1] ASI_M_MMUREGS, %g5 |
293 | #ifndef CONFIG_SMP |
294 | cmp %o3, -1 |
295 | be hypersparc_flush_tlb_range_out |
296 | #endif |
297 | sethi %hi(~((1 << PGDIR_SHIFT) - 1)), %o4 |
298 | sta %o3, [%g1] ASI_M_MMUREGS |
299 | and %o1, %o4, %o1 |
300 | add %o1, 0x200, %o1 |
301 | sta %g0, [%o1] ASI_M_FLUSH_PROBE |
302 | 1: |
303 | sub %o1, %o4, %o1 |
304 | cmp %o1, %o2 |
305 | blu,a 1b |
306 | sta %g0, [%o1] ASI_M_FLUSH_PROBE |
307 | hypersparc_flush_tlb_range_out: |
308 | retl |
309 | sta %g5, [%g1] ASI_M_MMUREGS |
310 | |
311 | hypersparc_flush_tlb_page: |
312 | ld [%o0 + VMA_VM_MM], %o0 |
313 | mov SRMMU_CTX_REG, %g1 |
314 | ld [%o0 + AOFF_mm_context], %o3 |
315 | andn %o1, (PAGE_SIZE - 1), %o1 |
316 | #ifndef CONFIG_SMP |
317 | cmp %o3, -1 |
318 | be hypersparc_flush_tlb_page_out |
319 | #endif |
320 | lda [%g1] ASI_M_MMUREGS, %g5 |
321 | sta %o3, [%g1] ASI_M_MMUREGS |
322 | sta %g0, [%o1] ASI_M_FLUSH_PROBE |
323 | hypersparc_flush_tlb_page_out: |
324 | retl |
325 | sta %g5, [%g1] ASI_M_MMUREGS |
326 | |
327 | __INIT |
328 | |
329 | /* High speed page clear/copy. */ |
330 | hypersparc_bzero_1page: |
331 | /* NOTE: This routine has to be shorter than 40insns --jj */ |
332 | clr %g1 |
333 | mov 32, %g2 |
334 | mov 64, %g3 |
335 | mov 96, %g4 |
336 | mov 128, %g5 |
337 | mov 160, %g7 |
338 | mov 192, %o2 |
339 | mov 224, %o3 |
340 | mov 16, %o1 |
341 | 1: |
342 | stda %g0, [%o0 + %g0] ASI_M_BFILL |
343 | stda %g0, [%o0 + %g2] ASI_M_BFILL |
344 | stda %g0, [%o0 + %g3] ASI_M_BFILL |
345 | stda %g0, [%o0 + %g4] ASI_M_BFILL |
346 | stda %g0, [%o0 + %g5] ASI_M_BFILL |
347 | stda %g0, [%o0 + %g7] ASI_M_BFILL |
348 | stda %g0, [%o0 + %o2] ASI_M_BFILL |
349 | stda %g0, [%o0 + %o3] ASI_M_BFILL |
350 | subcc %o1, 1, %o1 |
351 | bne 1b |
352 | add %o0, 256, %o0 |
353 | |
354 | retl |
355 | nop |
356 | |
357 | hypersparc_copy_1page: |
358 | /* NOTE: This routine has to be shorter than 70insns --jj */ |
359 | sub %o1, %o0, %o2 ! difference |
360 | mov 16, %g1 |
361 | 1: |
362 | sta %o0, [%o0 + %o2] ASI_M_BCOPY |
363 | add %o0, 32, %o0 |
364 | sta %o0, [%o0 + %o2] ASI_M_BCOPY |
365 | add %o0, 32, %o0 |
366 | sta %o0, [%o0 + %o2] ASI_M_BCOPY |
367 | add %o0, 32, %o0 |
368 | sta %o0, [%o0 + %o2] ASI_M_BCOPY |
369 | add %o0, 32, %o0 |
370 | sta %o0, [%o0 + %o2] ASI_M_BCOPY |
371 | add %o0, 32, %o0 |
372 | sta %o0, [%o0 + %o2] ASI_M_BCOPY |
373 | add %o0, 32, %o0 |
374 | sta %o0, [%o0 + %o2] ASI_M_BCOPY |
375 | add %o0, 32, %o0 |
376 | sta %o0, [%o0 + %o2] ASI_M_BCOPY |
377 | subcc %g1, 1, %g1 |
378 | bne 1b |
379 | add %o0, 32, %o0 |
380 | |
381 | retl |
382 | nop |
383 | |
384 | .globl hypersparc_setup_blockops |
385 | hypersparc_setup_blockops: |
386 | sethi %hi(bzero_1page), %o0 |
387 | or %o0, %lo(bzero_1page), %o0 |
388 | sethi %hi(hypersparc_bzero_1page), %o1 |
389 | or %o1, %lo(hypersparc_bzero_1page), %o1 |
390 | sethi %hi(hypersparc_copy_1page), %o2 |
391 | or %o2, %lo(hypersparc_copy_1page), %o2 |
392 | ld [%o1], %o4 |
393 | 1: |
394 | add %o1, 4, %o1 |
395 | st %o4, [%o0] |
396 | add %o0, 4, %o0 |
397 | cmp %o1, %o2 |
398 | bne 1b |
399 | ld [%o1], %o4 |
400 | sethi %hi(__copy_1page), %o0 |
401 | or %o0, %lo(__copy_1page), %o0 |
402 | sethi %hi(hypersparc_setup_blockops), %o2 |
403 | or %o2, %lo(hypersparc_setup_blockops), %o2 |
404 | ld [%o1], %o4 |
405 | 1: |
406 | add %o1, 4, %o1 |
407 | st %o4, [%o0] |
408 | add %o0, 4, %o0 |
409 | cmp %o1, %o2 |
410 | bne 1b |
411 | ld [%o1], %o4 |
412 | sta %g0, [%g0] ASI_M_FLUSH_IWHOLE |
413 | retl |
414 | nop |
415 | |