| 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
|---|
| 2 | /* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling. |
|---|
| 3 | * |
|---|
| 4 | * Copyright (C) 1995, 1997, 2005, 2008 David S. Miller <davem@davemloft.net> |
|---|
| 5 | * Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de) |
|---|
| 6 | * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx) |
|---|
| 7 | * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz) |
|---|
| 8 | */ |
|---|
| 9 | |
|---|
| 10 | #include <linux/pgtable.h> |
|---|
| 11 | #include <asm/head.h> |
|---|
| 12 | #include <asm/asi.h> |
|---|
| 13 | #include <asm/page.h> |
|---|
| 14 | #include <asm/tsb.h> |
|---|
| 15 | |
|---|
| 16 | .text |
|---|
| 17 | .align 32 |
|---|
| 18 | |
|---|
| 19 | kvmap_itlb: |
|---|
| 20 | /* g6: TAG TARGET */ |
|---|
| 21 | mov TLB_TAG_ACCESS, %g4 |
|---|
| 22 | ldxa [%g4] ASI_IMMU, %g4 |
|---|
| 23 | |
|---|
| 24 | /* The kernel executes in context zero, therefore we do not |
|---|
| 25 | * need to clear the context ID bits out of %g4 here. |
|---|
| 26 | */ |
|---|
| 27 | |
|---|
| 28 | /* sun4v_itlb_miss branches here with the missing virtual |
|---|
| 29 | * address already loaded into %g4 |
|---|
| 30 | */ |
|---|
| 31 | kvmap_itlb_4v: |
|---|
| 32 | |
|---|
| 33 | /* Catch kernel NULL pointer calls. */ |
|---|
| 34 | sethi %hi(PAGE_SIZE), %g5 |
|---|
| 35 | cmp %g4, %g5 |
|---|
| 36 | blu,pn %xcc, kvmap_itlb_longpath |
|---|
| 37 | nop |
|---|
| 38 | |
|---|
| 39 | KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load) |
|---|
| 40 | |
|---|
| 41 | kvmap_itlb_tsb_miss: |
|---|
| 42 | sethi %hi(LOW_OBP_ADDRESS), %g5 |
|---|
| 43 | cmp %g4, %g5 |
|---|
| 44 | blu,pn %xcc, kvmap_itlb_vmalloc_addr |
|---|
| 45 | mov 0x1, %g5 |
|---|
| 46 | sllx %g5, 32, %g5 |
|---|
| 47 | cmp %g4, %g5 |
|---|
| 48 | blu,pn %xcc, kvmap_itlb_obp |
|---|
| 49 | nop |
|---|
| 50 | |
|---|
| 51 | kvmap_itlb_vmalloc_addr: |
|---|
| 52 | KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath) |
|---|
| 53 | |
|---|
| 54 | TSB_LOCK_TAG(%g1, %g2, %g7) |
|---|
| 55 | TSB_WRITE(%g1, %g5, %g6) |
|---|
| 56 | |
|---|
| 57 | /* fallthrough to TLB load */ |
|---|
| 58 | |
|---|
| 59 | kvmap_itlb_load: |
|---|
| 60 | |
|---|
| 61 | 661: stxa %g5, [%g0] ASI_ITLB_DATA_IN |
|---|
| 62 | retry |
|---|
| 63 | .section .sun4v_2insn_patch, "ax" |
|---|
| 64 | .word 661b |
|---|
| 65 | nop |
|---|
| 66 | nop |
|---|
| 67 | .previous |
|---|
| 68 | |
|---|
| 69 | /* For sun4v the ASI_ITLB_DATA_IN store and the retry |
|---|
| 70 | * instruction get nop'd out and we get here to branch |
|---|
| 71 | * to the sun4v tlb load code. The registers are setup |
|---|
| 72 | * as follows: |
|---|
| 73 | * |
|---|
| 74 | * %g4: vaddr |
|---|
| 75 | * %g5: PTE |
|---|
| 76 | * %g6: TAG |
|---|
| 77 | * |
|---|
| 78 | * The sun4v TLB load wants the PTE in %g3 so we fix that |
|---|
| 79 | * up here. |
|---|
| 80 | */ |
|---|
| 81 | ba,pt %xcc, sun4v_itlb_load |
|---|
| 82 | mov %g5, %g3 |
|---|
| 83 | |
|---|
| 84 | kvmap_itlb_longpath: |
|---|
| 85 | |
|---|
| 86 | 661: rdpr %pstate, %g5 |
|---|
| 87 | wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate |
|---|
| 88 | .section .sun4v_2insn_patch, "ax" |
|---|
| 89 | .word 661b |
|---|
| 90 | SET_GL(1) |
|---|
| 91 | nop |
|---|
| 92 | .previous |
|---|
| 93 | |
|---|
| 94 | rdpr %tpc, %g5 |
|---|
| 95 | ba,pt %xcc, sparc64_realfault_common |
|---|
| 96 | mov FAULT_CODE_ITLB, %g4 |
|---|
| 97 | |
|---|
| 98 | kvmap_itlb_obp: |
|---|
| 99 | OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath) |
|---|
| 100 | |
|---|
| 101 | TSB_LOCK_TAG(%g1, %g2, %g7) |
|---|
| 102 | |
|---|
| 103 | TSB_WRITE(%g1, %g5, %g6) |
|---|
| 104 | |
|---|
| 105 | ba,pt %xcc, kvmap_itlb_load |
|---|
| 106 | nop |
|---|
| 107 | |
|---|
| 108 | kvmap_dtlb_obp: |
|---|
| 109 | OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath) |
|---|
| 110 | |
|---|
| 111 | TSB_LOCK_TAG(%g1, %g2, %g7) |
|---|
| 112 | |
|---|
| 113 | TSB_WRITE(%g1, %g5, %g6) |
|---|
| 114 | |
|---|
| 115 | ba,pt %xcc, kvmap_dtlb_load |
|---|
| 116 | nop |
|---|
| 117 | |
|---|
| 118 | kvmap_linear_early: |
|---|
| 119 | sethi %hi(kern_linear_pte_xor), %g7 |
|---|
| 120 | ldx [%g7 + %lo(kern_linear_pte_xor)], %g2 |
|---|
| 121 | ba,pt %xcc, kvmap_dtlb_tsb4m_load |
|---|
| 122 | xor %g2, %g4, %g5 |
|---|
| 123 | |
|---|
| 124 | .align 32 |
|---|
| 125 | kvmap_dtlb_tsb4m_load: |
|---|
| 126 | TSB_LOCK_TAG(%g1, %g2, %g7) |
|---|
| 127 | TSB_WRITE(%g1, %g5, %g6) |
|---|
| 128 | ba,pt %xcc, kvmap_dtlb_load |
|---|
| 129 | nop |
|---|
| 130 | |
|---|
| 131 | kvmap_dtlb: |
|---|
| 132 | /* %g6: TAG TARGET */ |
|---|
| 133 | mov TLB_TAG_ACCESS, %g4 |
|---|
| 134 | ldxa [%g4] ASI_DMMU, %g4 |
|---|
| 135 | |
|---|
| 136 | /* The kernel executes in context zero, therefore we do not |
|---|
| 137 | * need to clear the context ID bits out of %g4 here. |
|---|
| 138 | */ |
|---|
| 139 | |
|---|
| 140 | /* sun4v_dtlb_miss branches here with the missing virtual |
|---|
| 141 | * address already loaded into %g4 |
|---|
| 142 | */ |
|---|
| 143 | kvmap_dtlb_4v: |
|---|
| 144 | brgez,pn %g4, kvmap_dtlb_nonlinear |
|---|
| 145 | nop |
|---|
| 146 | |
|---|
| 147 | #ifdef CONFIG_DEBUG_PAGEALLOC |
|---|
| 148 | /* Index through the base page size TSB even for linear |
|---|
| 149 | * mappings when using page allocation debugging. |
|---|
| 150 | */ |
|---|
| 151 | KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load) |
|---|
| 152 | #else |
|---|
| 153 | /* Correct TAG_TARGET is already in %g6, check 4mb TSB. */ |
|---|
| 154 | KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load) |
|---|
| 155 | #endif |
|---|
| 156 | /* Linear mapping TSB lookup failed. Fallthrough to kernel |
|---|
| 157 | * page table based lookup. |
|---|
| 158 | */ |
|---|
| 159 | .globl kvmap_linear_patch |
|---|
| 160 | kvmap_linear_patch: |
|---|
| 161 | ba,a,pt %xcc, kvmap_linear_early |
|---|
| 162 | |
|---|
| 163 | kvmap_dtlb_vmalloc_addr: |
|---|
| 164 | KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath) |
|---|
| 165 | |
|---|
| 166 | TSB_LOCK_TAG(%g1, %g2, %g7) |
|---|
| 167 | TSB_WRITE(%g1, %g5, %g6) |
|---|
| 168 | |
|---|
| 169 | /* fallthrough to TLB load */ |
|---|
| 170 | |
|---|
| 171 | kvmap_dtlb_load: |
|---|
| 172 | |
|---|
| 173 | 661: stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB |
|---|
| 174 | retry |
|---|
| 175 | .section .sun4v_2insn_patch, "ax" |
|---|
| 176 | .word 661b |
|---|
| 177 | nop |
|---|
| 178 | nop |
|---|
| 179 | .previous |
|---|
| 180 | |
|---|
| 181 | /* For sun4v the ASI_DTLB_DATA_IN store and the retry |
|---|
| 182 | * instruction get nop'd out and we get here to branch |
|---|
| 183 | * to the sun4v tlb load code. The registers are setup |
|---|
| 184 | * as follows: |
|---|
| 185 | * |
|---|
| 186 | * %g4: vaddr |
|---|
| 187 | * %g5: PTE |
|---|
| 188 | * %g6: TAG |
|---|
| 189 | * |
|---|
| 190 | * The sun4v TLB load wants the PTE in %g3 so we fix that |
|---|
| 191 | * up here. |
|---|
| 192 | */ |
|---|
| 193 | ba,pt %xcc, sun4v_dtlb_load |
|---|
| 194 | mov %g5, %g3 |
|---|
| 195 | |
|---|
| 196 | #ifdef CONFIG_SPARSEMEM_VMEMMAP |
|---|
| 197 | kvmap_vmemmap: |
|---|
| 198 | KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath) |
|---|
| 199 | ba,a,pt %xcc, kvmap_dtlb_load |
|---|
| 200 | #endif |
|---|
| 201 | |
|---|
| 202 | kvmap_dtlb_nonlinear: |
|---|
| 203 | /* Catch kernel NULL pointer derefs. */ |
|---|
| 204 | sethi %hi(PAGE_SIZE), %g5 |
|---|
| 205 | cmp %g4, %g5 |
|---|
| 206 | bleu,pn %xcc, kvmap_dtlb_longpath |
|---|
| 207 | nop |
|---|
| 208 | |
|---|
| 209 | #ifdef CONFIG_SPARSEMEM_VMEMMAP |
|---|
| 210 | /* Do not use the TSB for vmemmap. */ |
|---|
| 211 | sethi %hi(VMEMMAP_BASE), %g5 |
|---|
| 212 | ldx [%g5 + %lo(VMEMMAP_BASE)], %g5 |
|---|
| 213 | cmp %g4,%g5 |
|---|
| 214 | bgeu,pn %xcc, kvmap_vmemmap |
|---|
| 215 | nop |
|---|
| 216 | #endif |
|---|
| 217 | |
|---|
| 218 | KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load) |
|---|
| 219 | |
|---|
| 220 | kvmap_dtlb_tsbmiss: |
|---|
| 221 | sethi %hi(MODULES_VADDR), %g5 |
|---|
| 222 | cmp %g4, %g5 |
|---|
| 223 | blu,pn %xcc, kvmap_dtlb_longpath |
|---|
| 224 | sethi %hi(VMALLOC_END), %g5 |
|---|
| 225 | ldx [%g5 + %lo(VMALLOC_END)], %g5 |
|---|
| 226 | cmp %g4, %g5 |
|---|
| 227 | bgeu,pn %xcc, kvmap_dtlb_longpath |
|---|
| 228 | nop |
|---|
| 229 | |
|---|
| 230 | kvmap_check_obp: |
|---|
| 231 | sethi %hi(LOW_OBP_ADDRESS), %g5 |
|---|
| 232 | cmp %g4, %g5 |
|---|
| 233 | blu,pn %xcc, kvmap_dtlb_vmalloc_addr |
|---|
| 234 | mov 0x1, %g5 |
|---|
| 235 | sllx %g5, 32, %g5 |
|---|
| 236 | cmp %g4, %g5 |
|---|
| 237 | blu,pn %xcc, kvmap_dtlb_obp |
|---|
| 238 | nop |
|---|
| 239 | ba,pt %xcc, kvmap_dtlb_vmalloc_addr |
|---|
| 240 | nop |
|---|
| 241 | |
|---|
| 242 | kvmap_dtlb_longpath: |
|---|
| 243 | |
|---|
| 244 | 661: rdpr %pstate, %g5 |
|---|
| 245 | wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate |
|---|
| 246 | .section .sun4v_2insn_patch, "ax" |
|---|
| 247 | .word 661b |
|---|
| 248 | SET_GL(1) |
|---|
| 249 | ldxa [%g0] ASI_SCRATCHPAD, %g5 |
|---|
| 250 | .previous |
|---|
| 251 | |
|---|
| 252 | rdpr %tl, %g3 |
|---|
| 253 | cmp %g3, 1 |
|---|
| 254 | |
|---|
| 255 | 661: mov TLB_TAG_ACCESS, %g4 |
|---|
| 256 | ldxa [%g4] ASI_DMMU, %g5 |
|---|
| 257 | .section .sun4v_2insn_patch, "ax" |
|---|
| 258 | .word 661b |
|---|
| 259 | ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5 |
|---|
| 260 | nop |
|---|
| 261 | .previous |
|---|
| 262 | |
|---|
| 263 | /* The kernel executes in context zero, therefore we do not |
|---|
| 264 | * need to clear the context ID bits out of %g5 here. |
|---|
| 265 | */ |
|---|
| 266 | |
|---|
| 267 | be,pt %xcc, sparc64_realfault_common |
|---|
| 268 | mov FAULT_CODE_DTLB, %g4 |
|---|
| 269 | ba,pt %xcc, winfix_trampoline |
|---|
| 270 | nop |
|---|
| 271 | |
|---|
