| 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
|---|
| 2 | /* |
|---|
| 3 | * ultra.S: Don't expand these all over the place... |
|---|
| 4 | * |
|---|
| 5 | * Copyright (C) 1997, 2000, 2008 David S. Miller (davem@davemloft.net) |
|---|
| 6 | */ |
|---|
| 7 | |
|---|
| 8 | #include <linux/pgtable.h> |
|---|
| 9 | #include <asm/asi.h> |
|---|
| 10 | #include <asm/page.h> |
|---|
| 11 | #include <asm/spitfire.h> |
|---|
| 12 | #include <asm/mmu_context.h> |
|---|
| 13 | #include <asm/mmu.h> |
|---|
| 14 | #include <asm/pil.h> |
|---|
| 15 | #include <asm/head.h> |
|---|
| 16 | #include <asm/thread_info.h> |
|---|
| 17 | #include <asm/cacheflush.h> |
|---|
| 18 | #include <asm/hypervisor.h> |
|---|
| 19 | #include <asm/cpudata.h> |
|---|
| 20 | |
|---|
| 21 | /* Basically, most of the Spitfire vs. Cheetah madness |
|---|
| 22 | * has to do with the fact that Cheetah does not support |
|---|
| 23 | * IMMU flushes out of the secondary context. Someone needs |
|---|
| 24 | * to throw a south lake birthday party for the folks |
|---|
| 25 | * in Microelectronics who refused to fix this shit. |
|---|
| 26 | */ |
|---|
| 27 | |
|---|
| 28 | /* This file is meant to be read efficiently by the CPU, not humans. |
|---|
| 29 | * Staraj sie tego nikomu nie pierdolnac... |
|---|
| 30 | */ |
|---|
| 31 | .text |
|---|
| 32 | .align 32 |
|---|
| 33 | .globl __flush_tlb_mm |
|---|
| 34 | __flush_tlb_mm: /* 19 insns */ |
|---|
| 35 | /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ |
|---|
| 36 | ldxa [%o1] ASI_DMMU, %g2 |
|---|
| 37 | cmp %g2, %o0 |
|---|
| 38 | bne,pn %icc, __spitfire_flush_tlb_mm_slow |
|---|
| 39 | mov 0x50, %g3 |
|---|
| 40 | stxa %g0, [%g3] ASI_DMMU_DEMAP |
|---|
| 41 | stxa %g0, [%g3] ASI_IMMU_DEMAP |
|---|
| 42 | sethi %hi(KERNBASE), %g3 |
|---|
| 43 | flush %g3 |
|---|
| 44 | retl |
|---|
| 45 | nop |
|---|
| 46 | nop |
|---|
| 47 | nop |
|---|
| 48 | nop |
|---|
| 49 | nop |
|---|
| 50 | nop |
|---|
| 51 | nop |
|---|
| 52 | nop |
|---|
| 53 | nop |
|---|
| 54 | nop |
|---|
| 55 | |
|---|
| 56 | .align 32 |
|---|
| 57 | .globl __flush_tlb_page |
|---|
| 58 | __flush_tlb_page: /* 22 insns */ |
|---|
| 59 | /* %o0 = context, %o1 = vaddr */ |
|---|
| 60 | rdpr %pstate, %g7 |
|---|
| 61 | andn %g7, PSTATE_IE, %g2 |
|---|
| 62 | wrpr %g2, %pstate |
|---|
| 63 | mov SECONDARY_CONTEXT, %o4 |
|---|
| 64 | ldxa [%o4] ASI_DMMU, %g2 |
|---|
| 65 | stxa %o0, [%o4] ASI_DMMU |
|---|
| 66 | andcc %o1, 1, %g0 |
|---|
| 67 | andn %o1, 1, %o3 |
|---|
| 68 | be,pn %icc, 1f |
|---|
| 69 | or %o3, 0x10, %o3 |
|---|
| 70 | stxa %g0, [%o3] ASI_IMMU_DEMAP |
|---|
| 71 | 1: stxa %g0, [%o3] ASI_DMMU_DEMAP |
|---|
| 72 | membar #Sync |
|---|
| 73 | stxa %g2, [%o4] ASI_DMMU |
|---|
| 74 | sethi %hi(KERNBASE), %o4 |
|---|
| 75 | flush %o4 |
|---|
| 76 | retl |
|---|
| 77 | wrpr %g7, 0x0, %pstate |
|---|
| 78 | nop |
|---|
| 79 | nop |
|---|
| 80 | nop |
|---|
| 81 | nop |
|---|
| 82 | |
|---|
| 83 | .align 32 |
|---|
| 84 | .globl __flush_tlb_pending |
|---|
| 85 | __flush_tlb_pending: /* 27 insns */ |
|---|
| 86 | /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ |
|---|
| 87 | rdpr %pstate, %g7 |
|---|
| 88 | sllx %o1, 3, %o1 |
|---|
| 89 | andn %g7, PSTATE_IE, %g2 |
|---|
| 90 | wrpr %g2, %pstate |
|---|
| 91 | mov SECONDARY_CONTEXT, %o4 |
|---|
| 92 | ldxa [%o4] ASI_DMMU, %g2 |
|---|
| 93 | stxa %o0, [%o4] ASI_DMMU |
|---|
| 94 | 1: sub %o1, (1 << 3), %o1 |
|---|
| 95 | ldx [%o2 + %o1], %o3 |
|---|
| 96 | andcc %o3, 1, %g0 |
|---|
| 97 | andn %o3, 1, %o3 |
|---|
| 98 | be,pn %icc, 2f |
|---|
| 99 | or %o3, 0x10, %o3 |
|---|
| 100 | stxa %g0, [%o3] ASI_IMMU_DEMAP |
|---|
| 101 | 2: stxa %g0, [%o3] ASI_DMMU_DEMAP |
|---|
| 102 | membar #Sync |
|---|
| 103 | brnz,pt %o1, 1b |
|---|
| 104 | nop |
|---|
| 105 | stxa %g2, [%o4] ASI_DMMU |
|---|
| 106 | sethi %hi(KERNBASE), %o4 |
|---|
| 107 | flush %o4 |
|---|
| 108 | retl |
|---|
| 109 | wrpr %g7, 0x0, %pstate |
|---|
| 110 | nop |
|---|
| 111 | nop |
|---|
| 112 | nop |
|---|
| 113 | nop |
|---|
| 114 | |
|---|
| 115 | .align 32 |
|---|
| 116 | .globl __flush_tlb_kernel_range |
|---|
| 117 | __flush_tlb_kernel_range: /* 31 insns */ |
|---|
| 118 | /* %o0=start, %o1=end */ |
|---|
| 119 | cmp %o0, %o1 |
|---|
| 120 | be,pn %xcc, 2f |
|---|
| 121 | sub %o1, %o0, %o3 |
|---|
| 122 | srlx %o3, 18, %o4 |
|---|
| 123 | brnz,pn %o4, __spitfire_flush_tlb_kernel_range_slow |
|---|
| 124 | sethi %hi(PAGE_SIZE), %o4 |
|---|
| 125 | sub %o3, %o4, %o3 |
|---|
| 126 | or %o0, 0x20, %o0 ! Nucleus |
|---|
| 127 | 1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP |
|---|
| 128 | stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP |
|---|
| 129 | membar #Sync |
|---|
| 130 | brnz,pt %o3, 1b |
|---|
| 131 | sub %o3, %o4, %o3 |
|---|
| 132 | 2: sethi %hi(KERNBASE), %o3 |
|---|
| 133 | flush %o3 |
|---|
| 134 | retl |
|---|
| 135 | nop |
|---|
| 136 | nop |
|---|
| 137 | nop |
|---|
| 138 | nop |
|---|
| 139 | nop |
|---|
| 140 | nop |
|---|
| 141 | nop |
|---|
| 142 | nop |
|---|
| 143 | nop |
|---|
| 144 | nop |
|---|
| 145 | nop |
|---|
| 146 | nop |
|---|
| 147 | nop |
|---|
| 148 | nop |
|---|
| 149 | nop |
|---|
| 150 | |
|---|
| 151 | __spitfire_flush_tlb_kernel_range_slow: |
|---|
| 152 | mov 63 * 8, %o4 |
|---|
| 153 | 1: ldxa [%o4] ASI_ITLB_DATA_ACCESS, %o3 |
|---|
| 154 | andcc %o3, 0x40, %g0 /* _PAGE_L_4U */ |
|---|
| 155 | bne,pn %xcc, 2f |
|---|
| 156 | mov TLB_TAG_ACCESS, %o3 |
|---|
| 157 | stxa %g0, [%o3] ASI_IMMU |
|---|
| 158 | stxa %g0, [%o4] ASI_ITLB_DATA_ACCESS |
|---|
| 159 | membar #Sync |
|---|
| 160 | 2: ldxa [%o4] ASI_DTLB_DATA_ACCESS, %o3 |
|---|
| 161 | andcc %o3, 0x40, %g0 |
|---|
| 162 | bne,pn %xcc, 2f |
|---|
| 163 | mov TLB_TAG_ACCESS, %o3 |
|---|
| 164 | stxa %g0, [%o3] ASI_DMMU |
|---|
| 165 | stxa %g0, [%o4] ASI_DTLB_DATA_ACCESS |
|---|
| 166 | membar #Sync |
|---|
| 167 | 2: sub %o4, 8, %o4 |
|---|
| 168 | brgez,pt %o4, 1b |
|---|
| 169 | nop |
|---|
| 170 | retl |
|---|
| 171 | nop |
|---|
| 172 | |
|---|
| 173 | __spitfire_flush_tlb_mm_slow: |
|---|
| 174 | rdpr %pstate, %g1 |
|---|
| 175 | wrpr %g1, PSTATE_IE, %pstate |
|---|
| 176 | stxa %o0, [%o1] ASI_DMMU |
|---|
| 177 | stxa %g0, [%g3] ASI_DMMU_DEMAP |
|---|
| 178 | stxa %g0, [%g3] ASI_IMMU_DEMAP |
|---|
| 179 | flush %g6 |
|---|
| 180 | stxa %g2, [%o1] ASI_DMMU |
|---|
| 181 | sethi %hi(KERNBASE), %o1 |
|---|
| 182 | flush %o1 |
|---|
| 183 | retl |
|---|
| 184 | wrpr %g1, 0, %pstate |
|---|
| 185 | |
|---|
| 186 | /* |
|---|
| 187 | * The following code flushes one page_size worth. |
|---|
| 188 | */ |
|---|
| 189 | .section .kprobes.text, "ax" |
|---|
| 190 | .align 32 |
|---|
| 191 | .globl __flush_icache_page |
|---|
| 192 | __flush_icache_page: /* %o0 = phys_page */ |
|---|
| 193 | srlx %o0, PAGE_SHIFT, %o0 |
|---|
| 194 | sethi %hi(PAGE_OFFSET), %g1 |
|---|
| 195 | sllx %o0, PAGE_SHIFT, %o0 |
|---|
| 196 | sethi %hi(PAGE_SIZE), %g2 |
|---|
| 197 | ldx [%g1 + %lo(PAGE_OFFSET)], %g1 |
|---|
| 198 | add %o0, %g1, %o0 |
|---|
| 199 | 1: subcc %g2, 32, %g2 |
|---|
| 200 | bne,pt %icc, 1b |
|---|
| 201 | flush %o0 + %g2 |
|---|
| 202 | retl |
|---|
| 203 | nop |
|---|
| 204 | |
|---|
| 205 | #ifdef DCACHE_ALIASING_POSSIBLE |
|---|
| 206 | |
|---|
| 207 | #if (PAGE_SHIFT != 13) |
|---|
| 208 | #error only page shift of 13 is supported by dcache flush |
|---|
| 209 | #endif |
|---|
| 210 | |
|---|
| 211 | #define DTAG_MASK 0x3 |
|---|
| 212 | |
|---|
| 213 | /* This routine is Spitfire specific so the hardcoded |
|---|
| 214 | * D-cache size and line-size are OK. |
|---|
| 215 | */ |
|---|
| 216 | .align 64 |
|---|
| 217 | .globl __flush_dcache_page |
|---|
| 218 | __flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */ |
|---|
| 219 | sethi %hi(PAGE_OFFSET), %g1 |
|---|
| 220 | ldx [%g1 + %lo(PAGE_OFFSET)], %g1 |
|---|
| 221 | sub %o0, %g1, %o0 ! physical address |
|---|
| 222 | srlx %o0, 11, %o0 ! make D-cache TAG |
|---|
| 223 | sethi %hi(1 << 14), %o2 ! D-cache size |
|---|
| 224 | sub %o2, (1 << 5), %o2 ! D-cache line size |
|---|
| 225 | 1: ldxa [%o2] ASI_DCACHE_TAG, %o3 ! load D-cache TAG |
|---|
| 226 | andcc %o3, DTAG_MASK, %g0 ! Valid? |
|---|
| 227 | be,pn %xcc, 2f ! Nope, branch |
|---|
| 228 | andn %o3, DTAG_MASK, %o3 ! Clear valid bits |
|---|
| 229 | cmp %o3, %o0 ! TAG match? |
|---|
| 230 | bne,pt %xcc, 2f ! Nope, branch |
|---|
| 231 | nop |
|---|
| 232 | stxa %g0, [%o2] ASI_DCACHE_TAG ! Invalidate TAG |
|---|
| 233 | membar #Sync |
|---|
| 234 | 2: brnz,pt %o2, 1b |
|---|
| 235 | sub %o2, (1 << 5), %o2 ! D-cache line size |
|---|
| 236 | |
|---|
| 237 | /* The I-cache does not snoop local stores so we |
|---|
| 238 | * better flush that too when necessary. |
|---|
| 239 | */ |
|---|
| 240 | brnz,pt %o1, __flush_icache_page |
|---|
| 241 | sllx %o0, 11, %o0 |
|---|
| 242 | retl |
|---|
| 243 | nop |
|---|
| 244 | |
|---|
| 245 | #endif /* DCACHE_ALIASING_POSSIBLE */ |
|---|
| 246 | |
|---|
| 247 | .previous |
|---|
| 248 | |
|---|
| 249 | /* Cheetah specific versions, patched at boot time. */ |
|---|
| 250 | __cheetah_flush_tlb_mm: /* 19 insns */ |
|---|
| 251 | rdpr %pstate, %g7 |
|---|
| 252 | andn %g7, PSTATE_IE, %g2 |
|---|
| 253 | wrpr %g2, 0x0, %pstate |
|---|
| 254 | wrpr %g0, 1, %tl |
|---|
| 255 | mov PRIMARY_CONTEXT, %o2 |
|---|
| 256 | mov 0x40, %g3 |
|---|
| 257 | ldxa [%o2] ASI_DMMU, %g2 |
|---|
| 258 | srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o1 |
|---|
| 259 | sllx %o1, CTX_PGSZ1_NUC_SHIFT, %o1 |
|---|
| 260 | or %o0, %o1, %o0 /* Preserve nucleus page size fields */ |
|---|
| 261 | stxa %o0, [%o2] ASI_DMMU |
|---|
| 262 | stxa %g0, [%g3] ASI_DMMU_DEMAP |
|---|
| 263 | stxa %g0, [%g3] ASI_IMMU_DEMAP |
|---|
| 264 | stxa %g2, [%o2] ASI_DMMU |
|---|
| 265 | sethi %hi(KERNBASE), %o2 |
|---|
| 266 | flush %o2 |
|---|
| 267 | wrpr %g0, 0, %tl |
|---|
| 268 | retl |
|---|
| 269 | wrpr %g7, 0x0, %pstate |
|---|
| 270 | |
|---|
| 271 | __cheetah_flush_tlb_page: /* 22 insns */ |
|---|
| 272 | /* %o0 = context, %o1 = vaddr */ |
|---|
| 273 | rdpr %pstate, %g7 |
|---|
| 274 | andn %g7, PSTATE_IE, %g2 |
|---|
| 275 | wrpr %g2, 0x0, %pstate |
|---|
| 276 | wrpr %g0, 1, %tl |
|---|
| 277 | mov PRIMARY_CONTEXT, %o4 |
|---|
| 278 | ldxa [%o4] ASI_DMMU, %g2 |
|---|
| 279 | srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 |
|---|
| 280 | sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 |
|---|
| 281 | or %o0, %o3, %o0 /* Preserve nucleus page size fields */ |
|---|
| 282 | stxa %o0, [%o4] ASI_DMMU |
|---|
| 283 | andcc %o1, 1, %g0 |
|---|
| 284 | be,pn %icc, 1f |
|---|
| 285 | andn %o1, 1, %o3 |
|---|
| 286 | stxa %g0, [%o3] ASI_IMMU_DEMAP |
|---|
| 287 | 1: stxa %g0, [%o3] ASI_DMMU_DEMAP |
|---|
| 288 | membar #Sync |
|---|
| 289 | stxa %g2, [%o4] ASI_DMMU |
|---|
| 290 | sethi %hi(KERNBASE), %o4 |
|---|
| 291 | flush %o4 |
|---|
| 292 | wrpr %g0, 0, %tl |
|---|
| 293 | retl |
|---|
| 294 | wrpr %g7, 0x0, %pstate |
|---|
| 295 | |
|---|
| 296 | __cheetah_flush_tlb_pending: /* 27 insns */ |
|---|
| 297 | /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ |
|---|
| 298 | rdpr %pstate, %g7 |
|---|
| 299 | sllx %o1, 3, %o1 |
|---|
| 300 | andn %g7, PSTATE_IE, %g2 |
|---|
| 301 | wrpr %g2, 0x0, %pstate |
|---|
| 302 | wrpr %g0, 1, %tl |
|---|
| 303 | mov PRIMARY_CONTEXT, %o4 |
|---|
| 304 | ldxa [%o4] ASI_DMMU, %g2 |
|---|
| 305 | srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 |
|---|
| 306 | sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 |
|---|
| 307 | or %o0, %o3, %o0 /* Preserve nucleus page size fields */ |
|---|
| 308 | stxa %o0, [%o4] ASI_DMMU |
|---|
| 309 | 1: sub %o1, (1 << 3), %o1 |
|---|
| 310 | ldx [%o2 + %o1], %o3 |
|---|
| 311 | andcc %o3, 1, %g0 |
|---|
| 312 | be,pn %icc, 2f |
|---|
| 313 | andn %o3, 1, %o3 |
|---|
| 314 | stxa %g0, [%o3] ASI_IMMU_DEMAP |
|---|
| 315 | 2: stxa %g0, [%o3] ASI_DMMU_DEMAP |
|---|
| 316 | membar #Sync |
|---|
| 317 | brnz,pt %o1, 1b |
|---|
| 318 | nop |
|---|
| 319 | stxa %g2, [%o4] ASI_DMMU |
|---|
| 320 | sethi %hi(KERNBASE), %o4 |
|---|
| 321 | flush %o4 |
|---|
| 322 | wrpr %g0, 0, %tl |
|---|
| 323 | retl |
|---|
| 324 | wrpr %g7, 0x0, %pstate |
|---|
| 325 | |
|---|
| 326 | __cheetah_flush_tlb_kernel_range: /* 31 insns */ |
|---|
| 327 | /* %o0=start, %o1=end */ |
|---|
| 328 | cmp %o0, %o1 |
|---|
| 329 | be,pn %xcc, 2f |
|---|
| 330 | sub %o1, %o0, %o3 |
|---|
| 331 | srlx %o3, 18, %o4 |
|---|
| 332 | brnz,pn %o4, 3f |
|---|
| 333 | sethi %hi(PAGE_SIZE), %o4 |
|---|
| 334 | sub %o3, %o4, %o3 |
|---|
| 335 | or %o0, 0x20, %o0 ! Nucleus |
|---|
| 336 | 1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP |
|---|
| 337 | stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP |
|---|
| 338 | membar #Sync |
|---|
| 339 | brnz,pt %o3, 1b |
|---|
| 340 | sub %o3, %o4, %o3 |
|---|
| 341 | 2: sethi %hi(KERNBASE), %o3 |
|---|
| 342 | flush %o3 |
|---|
| 343 | retl |
|---|
| 344 | nop |
|---|
| 345 | 3: mov 0x80, %o4 |
|---|
| 346 | stxa %g0, [%o4] ASI_DMMU_DEMAP |
|---|
| 347 | membar #Sync |
|---|
| 348 | stxa %g0, [%o4] ASI_IMMU_DEMAP |
|---|
| 349 | membar #Sync |
|---|
| 350 | retl |
|---|
| 351 | nop |
|---|
| 352 | nop |
|---|
| 353 | nop |
|---|
| 354 | nop |
|---|
| 355 | nop |
|---|
| 356 | nop |
|---|
| 357 | nop |
|---|
| 358 | nop |
|---|
| 359 | |
|---|
| 360 | #ifdef DCACHE_ALIASING_POSSIBLE |
|---|
| 361 | __cheetah_flush_dcache_page: /* 11 insns */ |
|---|
| 362 | sethi %hi(PAGE_OFFSET), %g1 |
|---|
| 363 | ldx [%g1 + %lo(PAGE_OFFSET)], %g1 |
|---|
| 364 | sub %o0, %g1, %o0 |
|---|
| 365 | sethi %hi(PAGE_SIZE), %o4 |
|---|
| 366 | 1: subcc %o4, (1 << 5), %o4 |
|---|
| 367 | stxa %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE |
|---|
| 368 | membar #Sync |
|---|
| 369 | bne,pt %icc, 1b |
|---|
| 370 | nop |
|---|
| 371 | retl /* I-cache flush never needed on Cheetah, see callers. */ |
|---|
| 372 | nop |
|---|
| 373 | #endif /* DCACHE_ALIASING_POSSIBLE */ |
|---|
| 374 | |
|---|
| 375 | /* Hypervisor specific versions, patched at boot time. */ |
|---|
| 376 | __hypervisor_tlb_tl0_error: |
|---|
| 377 | save %sp, -192, %sp |
|---|
| 378 | mov %i0, %o0 |
|---|
| 379 | call hypervisor_tlbop_error |
|---|
| 380 | mov %i1, %o1 |
|---|
| 381 | ret |
|---|
| 382 | restore |
|---|
| 383 | |
|---|
| 384 | __hypervisor_flush_tlb_mm: /* 19 insns */ |
|---|
| 385 | mov %o0, %o2 /* ARG2: mmu context */ |
|---|
| 386 | mov 0, %o0 /* ARG0: CPU lists unimplemented */ |
|---|
| 387 | mov 0, %o1 /* ARG1: CPU lists unimplemented */ |
|---|
| 388 | mov HV_MMU_ALL, %o3 /* ARG3: flags */ |
|---|
| 389 | mov HV_FAST_MMU_DEMAP_CTX, %o5 |
|---|
| 390 | ta HV_FAST_TRAP |
|---|
| 391 | brnz,pn %o0, 1f |
|---|
| 392 | mov HV_FAST_MMU_DEMAP_CTX, %o1 |
|---|
| 393 | retl |
|---|
| 394 | nop |
|---|
| 395 | 1: sethi %hi(__hypervisor_tlb_tl0_error), %o5 |
|---|
| 396 | jmpl %o5 + %lo(__hypervisor_tlb_tl0_error), %g0 |
|---|
| 397 | nop |
|---|
| 398 | nop |
|---|
| 399 | nop |
|---|
| 400 | nop |
|---|
| 401 | nop |
|---|
| 402 | nop |
|---|
| 403 | nop |
|---|
| 404 | |
|---|
| 405 | __hypervisor_flush_tlb_page: /* 22 insns */ |
|---|
| 406 | /* %o0 = context, %o1 = vaddr */ |
|---|
| 407 | mov %o0, %g2 |
|---|
| 408 | mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */ |
|---|
| 409 | mov %g2, %o1 /* ARG1: mmu context */ |
|---|
| 410 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ |
|---|
| 411 | srlx %o0, PAGE_SHIFT, %o0 |
|---|
| 412 | sllx %o0, PAGE_SHIFT, %o0 |
|---|
| 413 | ta HV_MMU_UNMAP_ADDR_TRAP |
|---|
| 414 | brnz,pn %o0, 1f |
|---|
| 415 | mov HV_MMU_UNMAP_ADDR_TRAP, %o1 |
|---|
| 416 | retl |
|---|
| 417 | nop |
|---|
| 418 | 1: sethi %hi(__hypervisor_tlb_tl0_error), %o2 |
|---|
| 419 | jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 |
|---|
| 420 | nop |
|---|
| 421 | nop |
|---|
| 422 | nop |
|---|
| 423 | nop |
|---|
| 424 | nop |
|---|
| 425 | nop |
|---|
| 426 | nop |
|---|
| 427 | nop |
|---|
| 428 | nop |
|---|
| 429 | |
|---|
| 430 | __hypervisor_flush_tlb_pending: /* 27 insns */ |
|---|
| 431 | /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ |
|---|
| 432 | sllx %o1, 3, %g1 |
|---|
| 433 | mov %o2, %g2 |
|---|
| 434 | mov %o0, %g3 |
|---|
| 435 | 1: sub %g1, (1 << 3), %g1 |
|---|
| 436 | ldx [%g2 + %g1], %o0 /* ARG0: vaddr + IMMU-bit */ |
|---|
| 437 | mov %g3, %o1 /* ARG1: mmu context */ |
|---|
| 438 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ |
|---|
| 439 | srlx %o0, PAGE_SHIFT, %o0 |
|---|
| 440 | sllx %o0, PAGE_SHIFT, %o0 |
|---|
| 441 | ta HV_MMU_UNMAP_ADDR_TRAP |
|---|
| 442 | brnz,pn %o0, 1f |
|---|
| 443 | mov HV_MMU_UNMAP_ADDR_TRAP, %o1 |
|---|
| 444 | brnz,pt %g1, 1b |
|---|
| 445 | nop |
|---|
| 446 | retl |
|---|
| 447 | nop |
|---|
| 448 | 1: sethi %hi(__hypervisor_tlb_tl0_error), %o2 |
|---|
| 449 | jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 |
|---|
| 450 | nop |
|---|
| 451 | nop |
|---|
| 452 | nop |
|---|
| 453 | nop |
|---|
| 454 | nop |
|---|
| 455 | nop |
|---|
| 456 | nop |
|---|
| 457 | nop |
|---|
| 458 | nop |
|---|
| 459 | |
|---|
| 460 | __hypervisor_flush_tlb_kernel_range: /* 31 insns */ |
|---|
| 461 | /* %o0=start, %o1=end */ |
|---|
| 462 | cmp %o0, %o1 |
|---|
| 463 | be,pn %xcc, 2f |
|---|
| 464 | sub %o1, %o0, %g2 |
|---|
| 465 | srlx %g2, 18, %g3 |
|---|
| 466 | brnz,pn %g3, 4f |
|---|
| 467 | mov %o0, %g1 |
|---|
| 468 | sethi %hi(PAGE_SIZE), %g3 |
|---|
| 469 | sub %g2, %g3, %g2 |
|---|
| 470 | 1: add %g1, %g2, %o0 /* ARG0: virtual address */ |
|---|
| 471 | mov 0, %o1 /* ARG1: mmu context */ |
|---|
| 472 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ |
|---|
| 473 | ta HV_MMU_UNMAP_ADDR_TRAP |
|---|
| 474 | brnz,pn %o0, 3f |
|---|
| 475 | mov HV_MMU_UNMAP_ADDR_TRAP, %o1 |
|---|
| 476 | brnz,pt %g2, 1b |
|---|
| 477 | sub %g2, %g3, %g2 |
|---|
| 478 | 2: retl |
|---|
| 479 | nop |
|---|
| 480 | 3: sethi %hi(__hypervisor_tlb_tl0_error), %o2 |
|---|
| 481 | jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 |
|---|
| 482 | nop |
|---|
| 483 | 4: mov 0, %o0 /* ARG0: CPU lists unimplemented */ |
|---|
| 484 | mov 0, %o1 /* ARG1: CPU lists unimplemented */ |
|---|
| 485 | mov 0, %o2 /* ARG2: mmu context == nucleus */ |
|---|
| 486 | mov HV_MMU_ALL, %o3 /* ARG3: flags */ |
|---|
| 487 | mov HV_FAST_MMU_DEMAP_CTX, %o5 |
|---|
| 488 | ta HV_FAST_TRAP |
|---|
| 489 | brnz,pn %o0, 3b |
|---|
| 490 | mov HV_FAST_MMU_DEMAP_CTX, %o1 |
|---|
| 491 | retl |
|---|
| 492 | nop |
|---|
| 493 | |
|---|
| 494 | #ifdef DCACHE_ALIASING_POSSIBLE |
|---|
| 495 | /* XXX Niagara and friends have an 8K cache, so no aliasing is |
|---|
| 496 | * XXX possible, but nothing explicit in the Hypervisor API |
|---|
| 497 | * XXX guarantees this. |
|---|
| 498 | */ |
|---|
| 499 | __hypervisor_flush_dcache_page: /* 2 insns */ |
|---|
| 500 | retl |
|---|
| 501 | nop |
|---|
| 502 | #endif |
|---|
| 503 | |
|---|
| 504 | tlb_patch_one: |
|---|
| 505 | 1: lduw [%o1], %g1 |
|---|
| 506 | stw %g1, [%o0] |
|---|
| 507 | flush %o0 |
|---|
| 508 | subcc %o2, 1, %o2 |
|---|
| 509 | add %o1, 4, %o1 |
|---|
| 510 | bne,pt %icc, 1b |
|---|
| 511 | add %o0, 4, %o0 |
|---|
| 512 | retl |
|---|
| 513 | nop |
|---|
| 514 | |
|---|
| 515 | #ifdef CONFIG_SMP |
|---|
| 516 | /* These are all called by the slaves of a cross call, at |
|---|
| 517 | * trap level 1, with interrupts fully disabled. |
|---|
| 518 | * |
|---|
| 519 | * Register usage: |
|---|
| 520 | * %g5 mm->context (all tlb flushes) |
|---|
| 521 | * %g1 address arg 1 (tlb page and range flushes) |
|---|
| 522 | * %g7 address arg 2 (tlb range flush only) |
|---|
| 523 | * |
|---|
| 524 | * %g6 scratch 1 |
|---|
| 525 | * %g2 scratch 2 |
|---|
| 526 | * %g3 scratch 3 |
|---|
| 527 | * %g4 scratch 4 |
|---|
| 528 | */ |
|---|
| 529 | .align 32 |
|---|
| 530 | .globl xcall_flush_tlb_mm |
|---|
| 531 | xcall_flush_tlb_mm: /* 24 insns */ |
|---|
| 532 | mov PRIMARY_CONTEXT, %g2 |
|---|
| 533 | ldxa [%g2] ASI_DMMU, %g3 |
|---|
| 534 | srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4 |
|---|
| 535 | sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 |
|---|
| 536 | or %g5, %g4, %g5 /* Preserve nucleus page size fields */ |
|---|
| 537 | stxa %g5, [%g2] ASI_DMMU |
|---|
| 538 | mov 0x40, %g4 |
|---|
| 539 | stxa %g0, [%g4] ASI_DMMU_DEMAP |
|---|
| 540 | stxa %g0, [%g4] ASI_IMMU_DEMAP |
|---|
| 541 | stxa %g3, [%g2] ASI_DMMU |
|---|
| 542 | retry |
|---|
| 543 | nop |
|---|
| 544 | nop |
|---|
| 545 | nop |
|---|
| 546 | nop |
|---|
| 547 | nop |
|---|
| 548 | nop |
|---|
| 549 | nop |
|---|
| 550 | nop |
|---|
| 551 | nop |
|---|
| 552 | nop |
|---|
| 553 | nop |
|---|
| 554 | nop |
|---|
| 555 | nop |
|---|
| 556 | |
|---|
| 557 | .globl xcall_flush_tlb_page |
|---|
| 558 | xcall_flush_tlb_page: /* 20 insns */ |
|---|
| 559 | /* %g5=context, %g1=vaddr */ |
|---|
| 560 | mov PRIMARY_CONTEXT, %g4 |
|---|
| 561 | ldxa [%g4] ASI_DMMU, %g2 |
|---|
| 562 | srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4 |
|---|
| 563 | sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 |
|---|
| 564 | or %g5, %g4, %g5 |
|---|
| 565 | mov PRIMARY_CONTEXT, %g4 |
|---|
| 566 | stxa %g5, [%g4] ASI_DMMU |
|---|
| 567 | andcc %g1, 0x1, %g0 |
|---|
| 568 | be,pn %icc, 2f |
|---|
| 569 | andn %g1, 0x1, %g5 |
|---|
| 570 | stxa %g0, [%g5] ASI_IMMU_DEMAP |
|---|
| 571 | 2: stxa %g0, [%g5] ASI_DMMU_DEMAP |
|---|
| 572 | membar #Sync |
|---|
| 573 | stxa %g2, [%g4] ASI_DMMU |
|---|
| 574 | retry |
|---|
| 575 | nop |
|---|
| 576 | nop |
|---|
| 577 | nop |
|---|
| 578 | nop |
|---|
| 579 | nop |
|---|
| 580 | |
|---|
| 581 | .globl xcall_flush_tlb_kernel_range |
|---|
| 582 | xcall_flush_tlb_kernel_range: /* 44 insns */ |
|---|
| 583 | sethi %hi(PAGE_SIZE - 1), %g2 |
|---|
| 584 | or %g2, %lo(PAGE_SIZE - 1), %g2 |
|---|
| 585 | andn %g1, %g2, %g1 |
|---|
| 586 | andn %g7, %g2, %g7 |
|---|
| 587 | sub %g7, %g1, %g3 |
|---|
| 588 | srlx %g3, 18, %g2 |
|---|
| 589 | brnz,pn %g2, 2f |
|---|
| 590 | sethi %hi(PAGE_SIZE), %g2 |
|---|
| 591 | sub %g3, %g2, %g3 |
|---|
| 592 | or %g1, 0x20, %g1 ! Nucleus |
|---|
| 593 | 1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP |
|---|
| 594 | stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP |
|---|
| 595 | membar #Sync |
|---|
| 596 | brnz,pt %g3, 1b |
|---|
| 597 | sub %g3, %g2, %g3 |
|---|
| 598 | retry |
|---|
| 599 | 2: mov 63 * 8, %g1 |
|---|
| 600 | 1: ldxa [%g1] ASI_ITLB_DATA_ACCESS, %g2 |
|---|
| 601 | andcc %g2, 0x40, %g0 /* _PAGE_L_4U */ |
|---|
| 602 | bne,pn %xcc, 2f |
|---|
| 603 | mov TLB_TAG_ACCESS, %g2 |
|---|
| 604 | stxa %g0, [%g2] ASI_IMMU |
|---|
| 605 | stxa %g0, [%g1] ASI_ITLB_DATA_ACCESS |
|---|
| 606 | membar #Sync |
|---|
| 607 | 2: ldxa [%g1] ASI_DTLB_DATA_ACCESS, %g2 |
|---|
| 608 | andcc %g2, 0x40, %g0 |
|---|
| 609 | bne,pn %xcc, 2f |
|---|
| 610 | mov TLB_TAG_ACCESS, %g2 |
|---|
| 611 | stxa %g0, [%g2] ASI_DMMU |
|---|
| 612 | stxa %g0, [%g1] ASI_DTLB_DATA_ACCESS |
|---|
| 613 | membar #Sync |
|---|
| 614 | 2: sub %g1, 8, %g1 |
|---|
| 615 | brgez,pt %g1, 1b |
|---|
| 616 | nop |
|---|
| 617 | retry |
|---|
| 618 | nop |
|---|
| 619 | nop |
|---|
| 620 | nop |
|---|
| 621 | nop |
|---|
| 622 | nop |
|---|
| 623 | nop |
|---|
| 624 | nop |
|---|
| 625 | nop |
|---|
| 626 | nop |
|---|
| 627 | |
|---|
| 628 | /* This runs in a very controlled environment, so we do |
|---|
| 629 | * not need to worry about BH races etc. |
|---|
| 630 | */ |
|---|
| 631 | .globl xcall_sync_tick |
|---|
| 632 | xcall_sync_tick: |
|---|
| 633 | |
|---|
| 634 | 661: rdpr %pstate, %g2 |
|---|
| 635 | wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate |
|---|
| 636 | .section .sun4v_2insn_patch, "ax" |
|---|
| 637 | .word 661b |
|---|
| 638 | nop |
|---|
| 639 | nop |
|---|
| 640 | .previous |
|---|
| 641 | |
|---|
| 642 | rdpr %pil, %g2 |
|---|
| 643 | wrpr %g0, PIL_NORMAL_MAX, %pil |
|---|
| 644 | sethi %hi(109f), %g7 |
|---|
| 645 | b,pt %xcc, etrap_irq |
|---|
| 646 | 109: or %g7, %lo(109b), %g7 |
|---|
| 647 | #ifdef CONFIG_TRACE_IRQFLAGS |
|---|
| 648 | call trace_hardirqs_off |
|---|
| 649 | nop |
|---|
| 650 | #endif |
|---|
| 651 | call smp_synchronize_tick_client |
|---|
| 652 | nop |
|---|
| 653 | b rtrap_xcall |
|---|
| 654 | ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 |
|---|
| 655 | |
|---|
| 656 | .globl xcall_fetch_glob_regs |
|---|
| 657 | xcall_fetch_glob_regs: |
|---|
| 658 | sethi %hi(global_cpu_snapshot), %g1 |
|---|
| 659 | or %g1, %lo(global_cpu_snapshot), %g1 |
|---|
| 660 | __GET_CPUID(%g2) |
|---|
| 661 | sllx %g2, 6, %g3 |
|---|
| 662 | add %g1, %g3, %g1 |
|---|
| 663 | rdpr %tstate, %g7 |
|---|
| 664 | stx %g7, [%g1 + GR_SNAP_TSTATE] |
|---|
| 665 | rdpr %tpc, %g7 |
|---|
| 666 | stx %g7, [%g1 + GR_SNAP_TPC] |
|---|
| 667 | rdpr %tnpc, %g7 |
|---|
| 668 | stx %g7, [%g1 + GR_SNAP_TNPC] |
|---|
| 669 | stx %o7, [%g1 + GR_SNAP_O7] |
|---|
| 670 | stx %i7, [%g1 + GR_SNAP_I7] |
|---|
| 671 | /* Don't try this at home kids... */ |
|---|
| 672 | rdpr %cwp, %g3 |
|---|
| 673 | sub %g3, 1, %g7 |
|---|
| 674 | wrpr %g7, %cwp |
|---|
| 675 | mov %i7, %g7 |
|---|
| 676 | wrpr %g3, %cwp |
|---|
| 677 | stx %g7, [%g1 + GR_SNAP_RPC] |
|---|
| 678 | sethi %hi(trap_block), %g7 |
|---|
| 679 | or %g7, %lo(trap_block), %g7 |
|---|
| 680 | sllx %g2, TRAP_BLOCK_SZ_SHIFT, %g2 |
|---|
| 681 | add %g7, %g2, %g7 |
|---|
| 682 | ldx [%g7 + TRAP_PER_CPU_THREAD], %g3 |
|---|
| 683 | stx %g3, [%g1 + GR_SNAP_THREAD] |
|---|
| 684 | retry |
|---|
| 685 | |
|---|
| 686 | .globl xcall_fetch_glob_pmu |
|---|
| 687 | xcall_fetch_glob_pmu: |
|---|
| 688 | sethi %hi(global_cpu_snapshot), %g1 |
|---|
| 689 | or %g1, %lo(global_cpu_snapshot), %g1 |
|---|
| 690 | __GET_CPUID(%g2) |
|---|
| 691 | sllx %g2, 6, %g3 |
|---|
| 692 | add %g1, %g3, %g1 |
|---|
| 693 | rd %pic, %g7 |
|---|
| 694 | stx %g7, [%g1 + (4 * 8)] |
|---|
| 695 | rd %pcr, %g7 |
|---|
| 696 | stx %g7, [%g1 + (0 * 8)] |
|---|
| 697 | retry |
|---|
| 698 | |
|---|
| 699 | .globl xcall_fetch_glob_pmu_n4 |
|---|
| 700 | xcall_fetch_glob_pmu_n4: |
|---|
| 701 | sethi %hi(global_cpu_snapshot), %g1 |
|---|
| 702 | or %g1, %lo(global_cpu_snapshot), %g1 |
|---|
| 703 | __GET_CPUID(%g2) |
|---|
| 704 | sllx %g2, 6, %g3 |
|---|
| 705 | add %g1, %g3, %g1 |
|---|
| 706 | |
|---|
| 707 | ldxa [%g0] ASI_PIC, %g7 |
|---|
| 708 | stx %g7, [%g1 + (4 * 8)] |
|---|
| 709 | mov 0x08, %g3 |
|---|
| 710 | ldxa [%g3] ASI_PIC, %g7 |
|---|
| 711 | stx %g7, [%g1 + (5 * 8)] |
|---|
| 712 | mov 0x10, %g3 |
|---|
| 713 | ldxa [%g3] ASI_PIC, %g7 |
|---|
| 714 | stx %g7, [%g1 + (6 * 8)] |
|---|
| 715 | mov 0x18, %g3 |
|---|
| 716 | ldxa [%g3] ASI_PIC, %g7 |
|---|
| 717 | stx %g7, [%g1 + (7 * 8)] |
|---|
| 718 | |
|---|
| 719 | mov %o0, %g2 |
|---|
| 720 | mov %o1, %g3 |
|---|
| 721 | mov %o5, %g7 |
|---|
| 722 | |
|---|
| 723 | mov HV_FAST_VT_GET_PERFREG, %o5 |
|---|
| 724 | mov 3, %o0 |
|---|
| 725 | ta HV_FAST_TRAP |
|---|
| 726 | stx %o1, [%g1 + (3 * 8)] |
|---|
| 727 | mov HV_FAST_VT_GET_PERFREG, %o5 |
|---|
| 728 | mov 2, %o0 |
|---|
| 729 | ta HV_FAST_TRAP |
|---|
| 730 | stx %o1, [%g1 + (2 * 8)] |
|---|
| 731 | mov HV_FAST_VT_GET_PERFREG, %o5 |
|---|
| 732 | mov 1, %o0 |
|---|
| 733 | ta HV_FAST_TRAP |
|---|
| 734 | stx %o1, [%g1 + (1 * 8)] |
|---|
| 735 | mov HV_FAST_VT_GET_PERFREG, %o5 |
|---|
| 736 | mov 0, %o0 |
|---|
| 737 | ta HV_FAST_TRAP |
|---|
| 738 | stx %o1, [%g1 + (0 * 8)] |
|---|
| 739 | |
|---|
| 740 | mov %g2, %o0 |
|---|
| 741 | mov %g3, %o1 |
|---|
| 742 | mov %g7, %o5 |
|---|
| 743 | |
|---|
| 744 | retry |
|---|
| 745 | |
|---|
| 746 | __cheetah_xcall_flush_tlb_kernel_range: /* 44 insns */ |
|---|
| 747 | sethi %hi(PAGE_SIZE - 1), %g2 |
|---|
| 748 | or %g2, %lo(PAGE_SIZE - 1), %g2 |
|---|
| 749 | andn %g1, %g2, %g1 |
|---|
| 750 | andn %g7, %g2, %g7 |
|---|
| 751 | sub %g7, %g1, %g3 |
|---|
| 752 | srlx %g3, 18, %g2 |
|---|
| 753 | brnz,pn %g2, 2f |
|---|
| 754 | sethi %hi(PAGE_SIZE), %g2 |
|---|
| 755 | sub %g3, %g2, %g3 |
|---|
| 756 | or %g1, 0x20, %g1 ! Nucleus |
|---|
| 757 | 1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP |
|---|
| 758 | stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP |
|---|
| 759 | membar #Sync |
|---|
| 760 | brnz,pt %g3, 1b |
|---|
| 761 | sub %g3, %g2, %g3 |
|---|
| 762 | retry |
|---|
| 763 | 2: mov 0x80, %g2 |
|---|
| 764 | stxa %g0, [%g2] ASI_DMMU_DEMAP |
|---|
| 765 | membar #Sync |
|---|
| 766 | stxa %g0, [%g2] ASI_IMMU_DEMAP |
|---|
| 767 | membar #Sync |
|---|
| 768 | retry |
|---|
| 769 | nop |
|---|
| 770 | nop |
|---|
| 771 | nop |
|---|
| 772 | nop |
|---|
| 773 | nop |
|---|
| 774 | nop |
|---|
| 775 | nop |
|---|
| 776 | nop |
|---|
| 777 | nop |
|---|
| 778 | nop |
|---|
| 779 | nop |
|---|
| 780 | nop |
|---|
| 781 | nop |
|---|
| 782 | nop |
|---|
| 783 | nop |
|---|
| 784 | nop |
|---|
| 785 | nop |
|---|
| 786 | nop |
|---|
| 787 | nop |
|---|
| 788 | nop |
|---|
| 789 | nop |
|---|
| 790 | nop |
|---|
| 791 | |
|---|
| 792 | #ifdef DCACHE_ALIASING_POSSIBLE |
|---|
| 793 | .align 32 |
|---|
| 794 | .globl xcall_flush_dcache_page_cheetah |
|---|
| 795 | xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */ |
|---|
| 796 | sethi %hi(PAGE_SIZE), %g3 |
|---|
| 797 | 1: subcc %g3, (1 << 5), %g3 |
|---|
| 798 | stxa %g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE |
|---|
| 799 | membar #Sync |
|---|
| 800 | bne,pt %icc, 1b |
|---|
| 801 | nop |
|---|
| 802 | retry |
|---|
| 803 | nop |
|---|
| 804 | #endif /* DCACHE_ALIASING_POSSIBLE */ |
|---|
| 805 | |
|---|
| 806 | .globl xcall_flush_dcache_page_spitfire |
|---|
| 807 | xcall_flush_dcache_page_spitfire: /* %g1 == physical page address |
|---|
| 808 | %g7 == kernel page virtual address |
|---|
| 809 | %g5 == (page->mapping != NULL) */ |
|---|
| 810 | #ifdef DCACHE_ALIASING_POSSIBLE |
|---|
| 811 | srlx %g1, (13 - 2), %g1 ! Form tag comparitor |
|---|
| 812 | sethi %hi(L1DCACHE_SIZE), %g3 ! D$ size == 16K |
|---|
| 813 | sub %g3, (1 << 5), %g3 ! D$ linesize == 32 |
|---|
| 814 | 1: ldxa [%g3] ASI_DCACHE_TAG, %g2 |
|---|
| 815 | andcc %g2, 0x3, %g0 |
|---|
| 816 | be,pn %xcc, 2f |
|---|
| 817 | andn %g2, 0x3, %g2 |
|---|
| 818 | cmp %g2, %g1 |
|---|
| 819 | |
|---|
| 820 | bne,pt %xcc, 2f |
|---|
| 821 | nop |
|---|
| 822 | stxa %g0, [%g3] ASI_DCACHE_TAG |
|---|
| 823 | membar #Sync |
|---|
| 824 | 2: cmp %g3, 0 |
|---|
| 825 | bne,pt %xcc, 1b |
|---|
| 826 | sub %g3, (1 << 5), %g3 |
|---|
| 827 | |
|---|
| 828 | brz,pn %g5, 2f |
|---|
| 829 | #endif /* DCACHE_ALIASING_POSSIBLE */ |
|---|
| 830 | sethi %hi(PAGE_SIZE), %g3 |
|---|
| 831 | |
|---|
| 832 | 1: flush %g7 |
|---|
| 833 | subcc %g3, (1 << 5), %g3 |
|---|
| 834 | bne,pt %icc, 1b |
|---|
| 835 | add %g7, (1 << 5), %g7 |
|---|
| 836 | |
|---|
| 837 | 2: retry |
|---|
| 838 | nop |
|---|
| 839 | nop |
|---|
| 840 | |
|---|
| 841 | /* %g5: error |
|---|
| 842 | * %g6: tlb op |
|---|
| 843 | */ |
|---|
| 844 | __hypervisor_tlb_xcall_error: |
|---|
| 845 | mov %g5, %g4 |
|---|
| 846 | mov %g6, %g5 |
|---|
| 847 | ba,pt %xcc, etrap |
|---|
| 848 | rd %pc, %g7 |
|---|
| 849 | mov %l4, %o0 |
|---|
| 850 | call hypervisor_tlbop_error_xcall |
|---|
| 851 | mov %l5, %o1 |
|---|
| 852 | ba,a,pt %xcc, rtrap |
|---|
| 853 | |
|---|
| 854 | .globl __hypervisor_xcall_flush_tlb_mm |
|---|
| 855 | __hypervisor_xcall_flush_tlb_mm: /* 24 insns */ |
|---|
| 856 | /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */ |
|---|
| 857 | mov %o0, %g2 |
|---|
| 858 | mov %o1, %g3 |
|---|
| 859 | mov %o2, %g4 |
|---|
| 860 | mov %o3, %g1 |
|---|
| 861 | mov %o5, %g7 |
|---|
| 862 | clr %o0 /* ARG0: CPU lists unimplemented */ |
|---|
| 863 | clr %o1 /* ARG1: CPU lists unimplemented */ |
|---|
| 864 | mov %g5, %o2 /* ARG2: mmu context */ |
|---|
| 865 | mov HV_MMU_ALL, %o3 /* ARG3: flags */ |
|---|
| 866 | mov HV_FAST_MMU_DEMAP_CTX, %o5 |
|---|
| 867 | ta HV_FAST_TRAP |
|---|
| 868 | mov HV_FAST_MMU_DEMAP_CTX, %g6 |
|---|
| 869 | brnz,pn %o0, 1f |
|---|
| 870 | mov %o0, %g5 |
|---|
| 871 | mov %g2, %o0 |
|---|
| 872 | mov %g3, %o1 |
|---|
| 873 | mov %g4, %o2 |
|---|
| 874 | mov %g1, %o3 |
|---|
| 875 | mov %g7, %o5 |
|---|
| 876 | membar #Sync |
|---|
| 877 | retry |
|---|
| 878 | 1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 |
|---|
| 879 | jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 |
|---|
| 880 | nop |
|---|
| 881 | |
|---|
| 882 | .globl __hypervisor_xcall_flush_tlb_page |
|---|
| 883 | __hypervisor_xcall_flush_tlb_page: /* 20 insns */ |
|---|
| 884 | /* %g5=ctx, %g1=vaddr */ |
|---|
| 885 | mov %o0, %g2 |
|---|
| 886 | mov %o1, %g3 |
|---|
| 887 | mov %o2, %g4 |
|---|
| 888 | mov %g1, %o0 /* ARG0: virtual address */ |
|---|
| 889 | mov %g5, %o1 /* ARG1: mmu context */ |
|---|
| 890 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ |
|---|
| 891 | srlx %o0, PAGE_SHIFT, %o0 |
|---|
| 892 | sllx %o0, PAGE_SHIFT, %o0 |
|---|
| 893 | ta HV_MMU_UNMAP_ADDR_TRAP |
|---|
| 894 | mov HV_MMU_UNMAP_ADDR_TRAP, %g6 |
|---|
| 895 | brnz,a,pn %o0, 1f |
|---|
| 896 | mov %o0, %g5 |
|---|
| 897 | mov %g2, %o0 |
|---|
| 898 | mov %g3, %o1 |
|---|
| 899 | mov %g4, %o2 |
|---|
| 900 | membar #Sync |
|---|
| 901 | retry |
|---|
| 902 | 1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 |
|---|
| 903 | jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 |
|---|
| 904 | nop |
|---|
| 905 | |
|---|
| 906 | .globl __hypervisor_xcall_flush_tlb_kernel_range |
|---|
| 907 | __hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */ |
|---|
| 908 | /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */ |
|---|
| 909 | sethi %hi(PAGE_SIZE - 1), %g2 |
|---|
| 910 | or %g2, %lo(PAGE_SIZE - 1), %g2 |
|---|
| 911 | andn %g1, %g2, %g1 |
|---|
| 912 | andn %g7, %g2, %g7 |
|---|
| 913 | sub %g7, %g1, %g3 |
|---|
| 914 | srlx %g3, 18, %g7 |
|---|
| 915 | add %g2, 1, %g2 |
|---|
| 916 | sub %g3, %g2, %g3 |
|---|
| 917 | mov %o0, %g2 |
|---|
| 918 | mov %o1, %g4 |
|---|
| 919 | brnz,pn %g7, 2f |
|---|
| 920 | mov %o2, %g7 |
|---|
| 921 | 1: add %g1, %g3, %o0 /* ARG0: virtual address */ |
|---|
| 922 | mov 0, %o1 /* ARG1: mmu context */ |
|---|
| 923 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ |
|---|
| 924 | ta HV_MMU_UNMAP_ADDR_TRAP |
|---|
| 925 | mov HV_MMU_UNMAP_ADDR_TRAP, %g6 |
|---|
| 926 | brnz,pn %o0, 1f |
|---|
| 927 | mov %o0, %g5 |
|---|
| 928 | sethi %hi(PAGE_SIZE), %o2 |
|---|
| 929 | brnz,pt %g3, 1b |
|---|
| 930 | sub %g3, %o2, %g3 |
|---|
| 931 | 5: mov %g2, %o0 |
|---|
| 932 | mov %g4, %o1 |
|---|
| 933 | mov %g7, %o2 |
|---|
| 934 | membar #Sync |
|---|
| 935 | retry |
|---|
| 936 | 1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 |
|---|
| 937 | jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 |
|---|
| 938 | nop |
|---|
| 939 | 2: mov %o3, %g1 |
|---|
| 940 | mov %o5, %g3 |
|---|
| 941 | mov 0, %o0 /* ARG0: CPU lists unimplemented */ |
|---|
| 942 | mov 0, %o1 /* ARG1: CPU lists unimplemented */ |
|---|
| 943 | mov 0, %o2 /* ARG2: mmu context == nucleus */ |
|---|
| 944 | mov HV_MMU_ALL, %o3 /* ARG3: flags */ |
|---|
| 945 | mov HV_FAST_MMU_DEMAP_CTX, %o5 |
|---|
| 946 | ta HV_FAST_TRAP |
|---|
| 947 | mov %g1, %o3 |
|---|
| 948 | brz,pt %o0, 5b |
|---|
| 949 | mov %g3, %o5 |
|---|
| 950 | mov HV_FAST_MMU_DEMAP_CTX, %g6 |
|---|
| 951 | ba,pt %xcc, 1b |
|---|
| 952 | clr %g5 |
|---|
| 953 | |
|---|
| 954 | /* These just get rescheduled to PIL vectors. */ |
|---|
| 955 | .globl xcall_call_function |
|---|
| 956 | xcall_call_function: |
|---|
| 957 | wr %g0, (1 << PIL_SMP_CALL_FUNC), %set_softint |
|---|
| 958 | retry |
|---|
| 959 | |
|---|
| 960 | .globl xcall_call_function_single |
|---|
| 961 | xcall_call_function_single: |
|---|
| 962 | wr %g0, (1 << PIL_SMP_CALL_FUNC_SNGL), %set_softint |
|---|
| 963 | retry |
|---|
| 964 | |
|---|
| 965 | .globl xcall_receive_signal |
|---|
| 966 | xcall_receive_signal: |
|---|
| 967 | wr %g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint |
|---|
| 968 | retry |
|---|
| 969 | |
|---|
| 970 | .globl xcall_capture |
|---|
| 971 | xcall_capture: |
|---|
| 972 | wr %g0, (1 << PIL_SMP_CAPTURE), %set_softint |
|---|
| 973 | retry |
|---|
| 974 | |
|---|
| 975 | #ifdef CONFIG_KGDB |
|---|
| 976 | .globl xcall_kgdb_capture |
|---|
| 977 | xcall_kgdb_capture: |
|---|
| 978 | wr %g0, (1 << PIL_KGDB_CAPTURE), %set_softint |
|---|
| 979 | retry |
|---|
| 980 | #endif |
|---|
| 981 | |
|---|
| 982 | #endif /* CONFIG_SMP */ |
|---|
| 983 | |
|---|
| 984 | .globl cheetah_patch_cachetlbops |
|---|
| 985 | cheetah_patch_cachetlbops: |
|---|
| 986 | save %sp, -128, %sp |
|---|
| 987 | |
|---|
| 988 | sethi %hi(__flush_tlb_mm), %o0 |
|---|
| 989 | or %o0, %lo(__flush_tlb_mm), %o0 |
|---|
| 990 | sethi %hi(__cheetah_flush_tlb_mm), %o1 |
|---|
| 991 | or %o1, %lo(__cheetah_flush_tlb_mm), %o1 |
|---|
| 992 | call tlb_patch_one |
|---|
| 993 | mov 19, %o2 |
|---|
| 994 | |
|---|
| 995 | sethi %hi(__flush_tlb_page), %o0 |
|---|
| 996 | or %o0, %lo(__flush_tlb_page), %o0 |
|---|
| 997 | sethi %hi(__cheetah_flush_tlb_page), %o1 |
|---|
| 998 | or %o1, %lo(__cheetah_flush_tlb_page), %o1 |
|---|
| 999 | call tlb_patch_one |
|---|
| 1000 | mov 22, %o2 |
|---|
| 1001 | |
|---|
| 1002 | sethi %hi(__flush_tlb_pending), %o0 |
|---|
| 1003 | or %o0, %lo(__flush_tlb_pending), %o0 |
|---|
| 1004 | sethi %hi(__cheetah_flush_tlb_pending), %o1 |
|---|
| 1005 | or %o1, %lo(__cheetah_flush_tlb_pending), %o1 |
|---|
| 1006 | call tlb_patch_one |
|---|
| 1007 | mov 27, %o2 |
|---|
| 1008 | |
|---|
| 1009 | sethi %hi(__flush_tlb_kernel_range), %o0 |
|---|
| 1010 | or %o0, %lo(__flush_tlb_kernel_range), %o0 |
|---|
| 1011 | sethi %hi(__cheetah_flush_tlb_kernel_range), %o1 |
|---|
| 1012 | or %o1, %lo(__cheetah_flush_tlb_kernel_range), %o1 |
|---|
| 1013 | call tlb_patch_one |
|---|
| 1014 | mov 31, %o2 |
|---|
| 1015 | |
|---|
| 1016 | #ifdef DCACHE_ALIASING_POSSIBLE |
|---|
| 1017 | sethi %hi(__flush_dcache_page), %o0 |
|---|
| 1018 | or %o0, %lo(__flush_dcache_page), %o0 |
|---|
| 1019 | sethi %hi(__cheetah_flush_dcache_page), %o1 |
|---|
| 1020 | or %o1, %lo(__cheetah_flush_dcache_page), %o1 |
|---|
| 1021 | call tlb_patch_one |
|---|
| 1022 | mov 11, %o2 |
|---|
| 1023 | #endif /* DCACHE_ALIASING_POSSIBLE */ |
|---|
| 1024 | |
|---|
| 1025 | #ifdef CONFIG_SMP |
|---|
| 1026 | sethi %hi(xcall_flush_tlb_kernel_range), %o0 |
|---|
| 1027 | or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 |
|---|
| 1028 | sethi %hi(__cheetah_xcall_flush_tlb_kernel_range), %o1 |
|---|
| 1029 | or %o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1 |
|---|
| 1030 | call tlb_patch_one |
|---|
| 1031 | mov 44, %o2 |
|---|
| 1032 | #endif /* CONFIG_SMP */ |
|---|
| 1033 | |
|---|
| 1034 | ret |
|---|
| 1035 | restore |
|---|
| 1036 | |
|---|
| 1037 | .globl hypervisor_patch_cachetlbops |
|---|
| 1038 | hypervisor_patch_cachetlbops: |
|---|
| 1039 | save %sp, -128, %sp |
|---|
| 1040 | |
|---|
| 1041 | sethi %hi(__flush_tlb_mm), %o0 |
|---|
| 1042 | or %o0, %lo(__flush_tlb_mm), %o0 |
|---|
| 1043 | sethi %hi(__hypervisor_flush_tlb_mm), %o1 |
|---|
| 1044 | or %o1, %lo(__hypervisor_flush_tlb_mm), %o1 |
|---|
| 1045 | call tlb_patch_one |
|---|
| 1046 | mov 19, %o2 |
|---|
| 1047 | |
|---|
| 1048 | sethi %hi(__flush_tlb_page), %o0 |
|---|
| 1049 | or %o0, %lo(__flush_tlb_page), %o0 |
|---|
| 1050 | sethi %hi(__hypervisor_flush_tlb_page), %o1 |
|---|
| 1051 | or %o1, %lo(__hypervisor_flush_tlb_page), %o1 |
|---|
| 1052 | call tlb_patch_one |
|---|
| 1053 | mov 22, %o2 |
|---|
| 1054 | |
|---|
| 1055 | sethi %hi(__flush_tlb_pending), %o0 |
|---|
| 1056 | or %o0, %lo(__flush_tlb_pending), %o0 |
|---|
| 1057 | sethi %hi(__hypervisor_flush_tlb_pending), %o1 |
|---|
| 1058 | or %o1, %lo(__hypervisor_flush_tlb_pending), %o1 |
|---|
| 1059 | call tlb_patch_one |
|---|
| 1060 | mov 27, %o2 |
|---|
| 1061 | |
|---|
| 1062 | sethi %hi(__flush_tlb_kernel_range), %o0 |
|---|
| 1063 | or %o0, %lo(__flush_tlb_kernel_range), %o0 |
|---|
| 1064 | sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1 |
|---|
| 1065 | or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1 |
|---|
| 1066 | call tlb_patch_one |
|---|
| 1067 | mov 31, %o2 |
|---|
| 1068 | |
|---|
| 1069 | #ifdef DCACHE_ALIASING_POSSIBLE |
|---|
| 1070 | sethi %hi(__flush_dcache_page), %o0 |
|---|
| 1071 | or %o0, %lo(__flush_dcache_page), %o0 |
|---|
| 1072 | sethi %hi(__hypervisor_flush_dcache_page), %o1 |
|---|
| 1073 | or %o1, %lo(__hypervisor_flush_dcache_page), %o1 |
|---|
| 1074 | call tlb_patch_one |
|---|
| 1075 | mov 2, %o2 |
|---|
| 1076 | #endif /* DCACHE_ALIASING_POSSIBLE */ |
|---|
| 1077 | |
|---|
| 1078 | #ifdef CONFIG_SMP |
|---|
| 1079 | sethi %hi(xcall_flush_tlb_mm), %o0 |
|---|
| 1080 | or %o0, %lo(xcall_flush_tlb_mm), %o0 |
|---|
| 1081 | sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1 |
|---|
| 1082 | or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1 |
|---|
| 1083 | call tlb_patch_one |
|---|
| 1084 | mov 24, %o2 |
|---|
| 1085 | |
|---|
| 1086 | sethi %hi(xcall_flush_tlb_page), %o0 |
|---|
| 1087 | or %o0, %lo(xcall_flush_tlb_page), %o0 |
|---|
| 1088 | sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1 |
|---|
| 1089 | or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1 |
|---|
| 1090 | call tlb_patch_one |
|---|
| 1091 | mov 20, %o2 |
|---|
| 1092 | |
|---|
| 1093 | sethi %hi(xcall_flush_tlb_kernel_range), %o0 |
|---|
| 1094 | or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 |
|---|
| 1095 | sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1 |
|---|
| 1096 | or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1 |
|---|
| 1097 | call tlb_patch_one |
|---|
| 1098 | mov 44, %o2 |
|---|
| 1099 | #endif /* CONFIG_SMP */ |
|---|
| 1100 | |
|---|
| 1101 | ret |
|---|
| 1102 | restore |
|---|
| 1103 | |
|---|
