1// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
2// See https://llvm.org/LICENSE.txt for license information.
3// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
4
5// This patch implements the support routines for the SME ABI,
6// described here:
7// https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#sme-support-routines
8
9#include "../assembly.h"
10
11
12#if !defined(__APPLE__)
13#define TPIDR2_SYMBOL SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)
14#define TPIDR2_SYMBOL_OFFSET :lo12:SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)
15#else
16// MachO requires @page/@pageoff directives because the global is defined
17// in a different file. Otherwise this file may fail to build.
18#define TPIDR2_SYMBOL SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)@page
19#define TPIDR2_SYMBOL_OFFSET SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)@pageoff
20#endif
21
22.arch armv9-a+sme
23
24// Utility function which calls a system's abort() routine. Because the function
25// is streaming-compatible it should disable streaming-SVE mode before calling
26// abort(). Note that there is no need to preserve any state before the call,
27// because the function does not return.
28DEFINE_COMPILERRT_PRIVATE_FUNCTION(do_abort)
29 .cfi_startproc
30 .variant_pcs SYMBOL_NAME(do_abort)
31 BTI_C
32 stp x29, x30, [sp, #-32]!
33 cntd x0
34 // Store VG to a stack location that we describe with .cfi_offset
35 str x0, [sp, #16]
36 .cfi_def_cfa_offset 32
37 .cfi_offset w30, -24
38 .cfi_offset w29, -32
39 .cfi_offset 46, -16
40 bl __arm_sme_state
41 tbz x0, #0, 2f
421:
43 smstop sm
442:
45 // We can't make this into a tail-call because the unwinder would
46 // need to restore the value of VG.
47 bl SYMBOL_NAME(abort)
48 .cfi_endproc
49END_COMPILERRT_FUNCTION(do_abort)
50
51// __arm_sme_state fills the result registers based on a local
52// that is set as part of the compiler-rt startup code.
53// __aarch64_has_sme_and_tpidr2_el0
54DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_sme_state)
55 .variant_pcs __arm_sme_state
56 BTI_C
57 mov x0, xzr
58 mov x1, xzr
59
60 adrp x16, TPIDR2_SYMBOL
61 ldrb w16, [x16, TPIDR2_SYMBOL_OFFSET]
62 cbz w16, 1f
630:
64 orr x0, x0, #0xC000000000000000
65 mrs x16, SVCR
66 bfxil x0, x16, #0, #2
67 mrs x1, TPIDR2_EL0
681:
69 ret
70END_COMPILERRT_OUTLINE_FUNCTION(__arm_sme_state)
71
72DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_tpidr2_restore)
73 .variant_pcs __arm_tpidr2_restore
74 BTI_C
75 // If TPIDR2_EL0 is nonnull, the subroutine aborts in some platform-specific
76 // manner.
77 mrs x14, TPIDR2_EL0
78 cbnz x14, 2f
79
80 // If any of the reserved bytes in the first 16 bytes of BLK are nonzero,
81 // the subroutine [..] aborts in some platform-defined manner.
82 ldrh w14, [x0, #10]
83 cbnz w14, 2f
84 ldr w14, [x0, #12]
85 cbnz w14, 2f
86
87 // If BLK.za_save_buffer is NULL, the subroutine does nothing.
88 ldr x16, [x0]
89 cbz x16, 1f
90
91 // If BLK.num_za_save_slices is zero, the subroutine does nothing.
92 ldrh w14, [x0, #8]
93 cbz x14, 1f
94
95 mov x15, xzr
960:
97 ldr za[w15,0], [x16]
98 addsvl x16, x16, #1
99 add x15, x15, #1
100 cmp x14, x15
101 b.ne 0b
1021:
103 ret
1042:
105 b SYMBOL_NAME(do_abort)
106END_COMPILERRT_OUTLINE_FUNCTION(__arm_tpidr2_restore)
107
108DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_tpidr2_save)
109 .variant_pcs __arm_tpidr2_restore
110 BTI_C
111 // If the current thread does not have access to TPIDR2_EL0, the subroutine
112 // does nothing.
113 adrp x14, TPIDR2_SYMBOL
114 ldrb w14, [x14, TPIDR2_SYMBOL_OFFSET]
115 cbz w14, 1f
116
117 // If TPIDR2_EL0 is null, the subroutine does nothing.
118 mrs x16, TPIDR2_EL0
119 cbz x16, 1f
120
121 // If any of the reserved bytes in the first 16 bytes of the TPIDR2 block are
122 // nonzero, the subroutine [..] aborts in some platform-defined manner.
123 ldrh w14, [x16, #10]
124 cbnz w14, 2f
125 ldr w14, [x16, #12]
126 cbnz w14, 2f
127
128 // If num_za_save_slices is zero, the subroutine does nothing.
129 ldrh w14, [x16, #8]
130 cbz x14, 1f
131
132 // If za_save_buffer is NULL, the subroutine does nothing.
133 ldr x16, [x16]
134 cbz x16, 1f
135
136 mov x15, xzr
1370:
138 str za[w15,0], [x16]
139 addsvl x16, x16, #1
140 add x15, x15, #1
141 cmp x14, x15
142 b.ne 0b
1431:
144 ret
1452:
146 b SYMBOL_NAME(do_abort)
147END_COMPILERRT_OUTLINE_FUNCTION(__arm_tpidr2_save)
148
149DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_za_disable)
150 .variant_pcs __arm_tpidr2_restore
151 BTI_C
152 // If the current thread does not have access to SME, the subroutine does
153 // nothing.
154 adrp x14, TPIDR2_SYMBOL
155 ldrb w14, [x14, TPIDR2_SYMBOL_OFFSET]
156 cbz w14, 0f
157
158 // Otherwise, the subroutine behaves as if it did the following:
159 // * Call __arm_tpidr2_save.
160 stp x29, x30, [sp, #-16]!
161 .cfi_def_cfa_offset 16
162 mov x29, sp
163 .cfi_def_cfa w29, 16
164 .cfi_offset w30, -8
165 .cfi_offset w29, -16
166 bl __arm_tpidr2_save
167
168 // * Set TPIDR2_EL0 to null.
169 msr TPIDR2_EL0, xzr
170
171 // * Set PSTATE.ZA to 0.
172 smstop za
173
174 .cfi_def_cfa wsp, 16
175 ldp x29, x30, [sp], #16
176 .cfi_def_cfa_offset 0
177 .cfi_restore w30
178 .cfi_restore w29
1790:
180 ret
181END_COMPILERRT_OUTLINE_FUNCTION(__arm_za_disable)
182
183NO_EXEC_STACK_DIRECTIVE
184
185// GNU property note for BTI and PAC
186GNU_PROPERTY_BTI_PAC
187

source code of compiler-rt/lib/builtins/aarch64/sme-abi.S