1 | //===-- NativeRegisterContextLinux_arm64.cpp ------------------------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #if defined(__arm64__) || defined(__aarch64__) |
10 | |
11 | #include "NativeRegisterContextLinux_arm.h" |
12 | #include "NativeRegisterContextLinux_arm64.h" |
13 | |
14 | #include "lldb/Host/HostInfo.h" |
15 | #include "lldb/Host/common/NativeProcessProtocol.h" |
16 | #include "lldb/Host/linux/Ptrace.h" |
17 | #include "lldb/Utility/DataBufferHeap.h" |
18 | #include "lldb/Utility/Log.h" |
19 | #include "lldb/Utility/RegisterValue.h" |
20 | #include "lldb/Utility/Status.h" |
21 | |
22 | #include "Plugins/Process/Linux/NativeProcessLinux.h" |
23 | #include "Plugins/Process/Linux/Procfs.h" |
24 | #include "Plugins/Process/POSIX/ProcessPOSIXLog.h" |
25 | #include "Plugins/Process/Utility/MemoryTagManagerAArch64MTE.h" |
26 | #include "Plugins/Process/Utility/RegisterFlagsLinux_arm64.h" |
27 | #include "Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h" |
28 | |
29 | // System includes - They have to be included after framework includes because |
30 | // they define some macros which collide with variable names in other modules |
31 | #include <sys/uio.h> |
32 | // NT_PRSTATUS and NT_FPREGSET definition |
33 | #include <elf.h> |
34 | #include <mutex> |
35 | #include <optional> |
36 | |
37 | #ifndef NT_ARM_SVE |
38 | #define NT_ARM_SVE 0x405 /* ARM Scalable Vector Extension */ |
39 | #endif |
40 | |
41 | #ifndef NT_ARM_SSVE |
42 | #define NT_ARM_SSVE \ |
43 | 0x40b /* ARM Scalable Matrix Extension, Streaming SVE mode */ |
44 | #endif |
45 | |
46 | #ifndef NT_ARM_ZA |
47 | #define NT_ARM_ZA 0x40c /* ARM Scalable Matrix Extension, Array Storage */ |
48 | #endif |
49 | |
50 | #ifndef NT_ARM_ZT |
51 | #define NT_ARM_ZT \ |
52 | 0x40d /* ARM Scalable Matrix Extension 2, lookup table register */ |
53 | #endif |
54 | |
55 | #ifndef NT_ARM_PAC_MASK |
56 | #define NT_ARM_PAC_MASK 0x406 /* Pointer authentication code masks */ |
57 | #endif |
58 | |
59 | #ifndef NT_ARM_TAGGED_ADDR_CTRL |
60 | #define NT_ARM_TAGGED_ADDR_CTRL 0x409 /* Tagged address control register */ |
61 | #endif |
62 | |
63 | #define HWCAP_PACA (1 << 30) |
64 | |
65 | #define HWCAP2_MTE (1 << 18) |
66 | |
67 | using namespace lldb; |
68 | using namespace lldb_private; |
69 | using namespace lldb_private::process_linux; |
70 | |
71 | // A NativeRegisterContext is constructed per thread, but all threads' registers |
72 | // will contain the same fields. Therefore this mutex prevents each instance |
73 | // competing with the other, and subsequent instances from having to detect the |
74 | // fields all over again. |
75 | static std::mutex g_register_flags_mutex; |
76 | static LinuxArm64RegisterFlags g_register_flags; |
77 | |
78 | std::unique_ptr<NativeRegisterContextLinux> |
79 | NativeRegisterContextLinux::CreateHostNativeRegisterContextLinux( |
80 | const ArchSpec &target_arch, NativeThreadLinux &native_thread) { |
81 | switch (target_arch.GetMachine()) { |
82 | case llvm::Triple::arm: |
83 | return std::make_unique<NativeRegisterContextLinux_arm>(target_arch, |
84 | native_thread); |
85 | case llvm::Triple::aarch64: { |
86 | // Configure register sets supported by this AArch64 target. |
87 | // Read SVE header to check for SVE support. |
88 | struct sve::user_sve_header sve_header; |
89 | struct iovec ioVec; |
90 | ioVec.iov_base = &sve_header; |
91 | ioVec.iov_len = sizeof(sve_header); |
92 | unsigned int regset = NT_ARM_SVE; |
93 | |
94 | Flags opt_regsets; |
95 | if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET, |
96 | native_thread.GetID(), ®set, |
97 | &ioVec, sizeof(sve_header)) |
98 | .Success()) { |
99 | opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskSVE); |
100 | |
101 | // We may also have the Scalable Matrix Extension (SME) which adds a |
102 | // streaming SVE mode. |
103 | ioVec.iov_len = sizeof(sve_header); |
104 | regset = NT_ARM_SSVE; |
105 | if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET, |
106 | native_thread.GetID(), ®set, |
107 | &ioVec, sizeof(sve_header)) |
108 | .Success()) |
109 | opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskSSVE); |
110 | } |
111 | |
112 | sve::user_za_header za_header; |
113 | ioVec.iov_base = &za_header; |
114 | ioVec.iov_len = sizeof(za_header); |
115 | regset = NT_ARM_ZA; |
116 | if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET, |
117 | native_thread.GetID(), ®set, |
118 | &ioVec, sizeof(za_header)) |
119 | .Success()) |
120 | opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskZA); |
121 | |
122 | // SME's ZT0 is a 512 bit register. |
123 | std::array<uint8_t, 64> zt_reg; |
124 | ioVec.iov_base = zt_reg.data(); |
125 | ioVec.iov_len = zt_reg.size(); |
126 | regset = NT_ARM_ZT; |
127 | if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET, |
128 | native_thread.GetID(), ®set, |
129 | &ioVec, zt_reg.size()) |
130 | .Success()) |
131 | opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskZT); |
132 | |
133 | NativeProcessLinux &process = native_thread.GetProcess(); |
134 | |
135 | std::optional<uint64_t> auxv_at_hwcap = |
136 | process.GetAuxValue(AuxVector::AUXV_AT_HWCAP); |
137 | if (auxv_at_hwcap && (*auxv_at_hwcap & HWCAP_PACA)) |
138 | opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskPAuth); |
139 | |
140 | std::optional<uint64_t> auxv_at_hwcap2 = |
141 | process.GetAuxValue(AuxVector::AUXV_AT_HWCAP2); |
142 | if (auxv_at_hwcap2 && (*auxv_at_hwcap2 & HWCAP2_MTE)) |
143 | opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskMTE); |
144 | |
145 | opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskTLS); |
146 | |
147 | std::lock_guard<std::mutex> lock(g_register_flags_mutex); |
148 | if (!g_register_flags.HasDetected()) |
149 | g_register_flags.DetectFields(auxv_at_hwcap.value_or(0), |
150 | auxv_at_hwcap2.value_or(0)); |
151 | |
152 | auto register_info_up = |
153 | std::make_unique<RegisterInfoPOSIX_arm64>(target_arch, opt_regsets); |
154 | return std::make_unique<NativeRegisterContextLinux_arm64>( |
155 | target_arch, native_thread, std::move(register_info_up)); |
156 | } |
157 | default: |
158 | llvm_unreachable("have no register context for architecture" ); |
159 | } |
160 | } |
161 | |
162 | llvm::Expected<ArchSpec> |
163 | NativeRegisterContextLinux::DetermineArchitecture(lldb::tid_t tid) { |
164 | return DetermineArchitectureViaGPR( |
165 | tid, RegisterInfoPOSIX_arm64::GetGPRSizeStatic()); |
166 | } |
167 | |
168 | NativeRegisterContextLinux_arm64::NativeRegisterContextLinux_arm64( |
169 | const ArchSpec &target_arch, NativeThreadProtocol &native_thread, |
170 | std::unique_ptr<RegisterInfoPOSIX_arm64> register_info_up) |
171 | : NativeRegisterContextRegisterInfo(native_thread, |
172 | register_info_up.release()), |
173 | NativeRegisterContextLinux(native_thread) { |
174 | g_register_flags.UpdateRegisterInfo( |
175 | GetRegisterInfoInterface().GetRegisterInfo(), |
176 | GetRegisterInfoInterface().GetRegisterCount()); |
177 | |
178 | ::memset(&m_fpr, 0, sizeof(m_fpr)); |
179 | ::memset(&m_gpr_arm64, 0, sizeof(m_gpr_arm64)); |
180 | ::memset(&m_hwp_regs, 0, sizeof(m_hwp_regs)); |
181 | ::memset(&m_hbp_regs, 0, sizeof(m_hbp_regs)); |
182 | ::memset(&m_sve_header, 0, sizeof(m_sve_header)); |
183 | ::memset(&m_pac_mask, 0, sizeof(m_pac_mask)); |
184 | ::memset(&m_tls_regs, 0, sizeof(m_tls_regs)); |
185 | ::memset(&m_sme_pseudo_regs, 0, sizeof(m_sme_pseudo_regs)); |
186 | std::fill(m_zt_reg.begin(), m_zt_reg.end(), 0); |
187 | |
188 | m_mte_ctrl_reg = 0; |
189 | |
190 | // 16 is just a maximum value, query hardware for actual watchpoint count |
191 | m_max_hwp_supported = 16; |
192 | m_max_hbp_supported = 16; |
193 | |
194 | m_refresh_hwdebug_info = true; |
195 | |
196 | m_gpr_is_valid = false; |
197 | m_fpu_is_valid = false; |
198 | m_sve_buffer_is_valid = false; |
199 | m_sve_header_is_valid = false; |
200 | m_pac_mask_is_valid = false; |
201 | m_mte_ctrl_is_valid = false; |
202 | m_tls_is_valid = false; |
203 | m_zt_buffer_is_valid = false; |
204 | |
205 | // SME adds the tpidr2 register |
206 | m_tls_size = GetRegisterInfo().IsSSVEPresent() ? sizeof(m_tls_regs) |
207 | : sizeof(m_tls_regs.tpidr_reg); |
208 | |
209 | if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent()) |
210 | m_sve_state = SVEState::Unknown; |
211 | else |
212 | m_sve_state = SVEState::Disabled; |
213 | } |
214 | |
215 | RegisterInfoPOSIX_arm64 & |
216 | NativeRegisterContextLinux_arm64::GetRegisterInfo() const { |
217 | return static_cast<RegisterInfoPOSIX_arm64 &>(*m_register_info_interface_up); |
218 | } |
219 | |
220 | uint32_t NativeRegisterContextLinux_arm64::GetRegisterSetCount() const { |
221 | return GetRegisterInfo().GetRegisterSetCount(); |
222 | } |
223 | |
224 | const RegisterSet * |
225 | NativeRegisterContextLinux_arm64::GetRegisterSet(uint32_t set_index) const { |
226 | return GetRegisterInfo().GetRegisterSet(set_index); |
227 | } |
228 | |
229 | uint32_t NativeRegisterContextLinux_arm64::GetUserRegisterCount() const { |
230 | uint32_t count = 0; |
231 | for (uint32_t set_index = 0; set_index < GetRegisterSetCount(); ++set_index) |
232 | count += GetRegisterSet(set_index)->num_registers; |
233 | return count; |
234 | } |
235 | |
236 | Status |
237 | NativeRegisterContextLinux_arm64::ReadRegister(const RegisterInfo *reg_info, |
238 | RegisterValue ®_value) { |
239 | Status error; |
240 | |
241 | if (!reg_info) { |
242 | error.SetErrorString("reg_info NULL" ); |
243 | return error; |
244 | } |
245 | |
246 | const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB]; |
247 | |
248 | if (reg == LLDB_INVALID_REGNUM) |
249 | return Status("no lldb regnum for %s" , reg_info && reg_info->name |
250 | ? reg_info->name |
251 | : "<unknown register>" ); |
252 | |
253 | uint8_t *src; |
254 | uint32_t offset = LLDB_INVALID_INDEX32; |
255 | uint64_t sve_vg; |
256 | std::vector<uint8_t> sve_reg_non_live; |
257 | |
258 | if (IsGPR(reg)) { |
259 | error = ReadGPR(); |
260 | if (error.Fail()) |
261 | return error; |
262 | |
263 | offset = reg_info->byte_offset; |
264 | assert(offset < GetGPRSize()); |
265 | src = (uint8_t *)GetGPRBuffer() + offset; |
266 | |
267 | } else if (IsFPR(reg)) { |
268 | if (m_sve_state == SVEState::Disabled) { |
269 | // SVE is disabled take legacy route for FPU register access |
270 | error = ReadFPR(); |
271 | if (error.Fail()) |
272 | return error; |
273 | |
274 | offset = CalculateFprOffset(reg_info); |
275 | assert(offset < GetFPRSize()); |
276 | src = (uint8_t *)GetFPRBuffer() + offset; |
277 | } else { |
278 | // SVE or SSVE enabled, we will read and cache SVE ptrace data. |
279 | // In SIMD or Full mode, the data comes from the SVE regset. In streaming |
280 | // mode it comes from the streaming SVE regset. |
281 | error = ReadAllSVE(); |
282 | if (error.Fail()) |
283 | return error; |
284 | |
285 | // FPSR and FPCR will be located right after Z registers in |
286 | // SVEState::FPSIMD while in SVEState::Full or SVEState::Streaming they |
287 | // will be located at the end of register data after an alignment |
288 | // correction based on currently selected vector length. |
289 | uint32_t sve_reg_num = LLDB_INVALID_REGNUM; |
290 | if (reg == GetRegisterInfo().GetRegNumFPSR()) { |
291 | sve_reg_num = reg; |
292 | if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming) |
293 | offset = sve::PTraceFPSROffset(sve::vq_from_vl(m_sve_header.vl)); |
294 | else if (m_sve_state == SVEState::FPSIMD) |
295 | offset = sve::ptrace_fpsimd_offset + (32 * 16); |
296 | } else if (reg == GetRegisterInfo().GetRegNumFPCR()) { |
297 | sve_reg_num = reg; |
298 | if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming) |
299 | offset = sve::PTraceFPCROffset(sve::vq_from_vl(m_sve_header.vl)); |
300 | else if (m_sve_state == SVEState::FPSIMD) |
301 | offset = sve::ptrace_fpsimd_offset + (32 * 16) + 4; |
302 | } else { |
303 | // Extract SVE Z register value register number for this reg_info |
304 | if (reg_info->value_regs && |
305 | reg_info->value_regs[0] != LLDB_INVALID_REGNUM) |
306 | sve_reg_num = reg_info->value_regs[0]; |
307 | offset = CalculateSVEOffset(GetRegisterInfoAtIndex(sve_reg_num)); |
308 | } |
309 | |
310 | assert(offset < GetSVEBufferSize()); |
311 | src = (uint8_t *)GetSVEBuffer() + offset; |
312 | } |
313 | } else if (IsTLS(reg)) { |
314 | error = ReadTLS(); |
315 | if (error.Fail()) |
316 | return error; |
317 | |
318 | offset = reg_info->byte_offset - GetRegisterInfo().GetTLSOffset(); |
319 | assert(offset < GetTLSBufferSize()); |
320 | src = (uint8_t *)GetTLSBuffer() + offset; |
321 | } else if (IsSVE(reg)) { |
322 | if (m_sve_state == SVEState::Disabled || m_sve_state == SVEState::Unknown) |
323 | return Status("SVE disabled or not supported" ); |
324 | |
325 | if (GetRegisterInfo().IsSVERegVG(reg)) { |
326 | sve_vg = GetSVERegVG(); |
327 | src = (uint8_t *)&sve_vg; |
328 | } else { |
329 | // SVE enabled, we will read and cache SVE ptrace data |
330 | error = ReadAllSVE(); |
331 | if (error.Fail()) |
332 | return error; |
333 | |
334 | if (m_sve_state == SVEState::FPSIMD) { |
335 | // In FPSIMD state SVE payload mirrors legacy fpsimd struct and so |
336 | // just copy 16 bytes of v register to the start of z register. All |
337 | // other SVE register will be set to zero. |
338 | sve_reg_non_live.resize(reg_info->byte_size, 0); |
339 | src = sve_reg_non_live.data(); |
340 | |
341 | if (GetRegisterInfo().IsSVEZReg(reg)) { |
342 | offset = CalculateSVEOffset(reg_info); |
343 | assert(offset < GetSVEBufferSize()); |
344 | ::memcpy(sve_reg_non_live.data(), (uint8_t *)GetSVEBuffer() + offset, |
345 | 16); |
346 | } |
347 | } else { |
348 | offset = CalculateSVEOffset(reg_info); |
349 | assert(offset < GetSVEBufferSize()); |
350 | src = (uint8_t *)GetSVEBuffer() + offset; |
351 | } |
352 | } |
353 | } else if (IsPAuth(reg)) { |
354 | error = ReadPAuthMask(); |
355 | if (error.Fail()) |
356 | return error; |
357 | |
358 | offset = reg_info->byte_offset - GetRegisterInfo().GetPAuthOffset(); |
359 | assert(offset < GetPACMaskSize()); |
360 | src = (uint8_t *)GetPACMask() + offset; |
361 | } else if (IsMTE(reg)) { |
362 | error = ReadMTEControl(); |
363 | if (error.Fail()) |
364 | return error; |
365 | |
366 | offset = reg_info->byte_offset - GetRegisterInfo().GetMTEOffset(); |
367 | assert(offset < GetMTEControlSize()); |
368 | src = (uint8_t *)GetMTEControl() + offset; |
369 | } else if (IsSME(reg)) { |
370 | if (GetRegisterInfo().IsSMERegZA(reg)) { |
371 | error = ReadZAHeader(); |
372 | if (error.Fail()) |
373 | return error; |
374 | |
375 | // If there is only a header and no registers, ZA is inactive. Read as 0 |
376 | // in this case. |
377 | if (m_za_header.size == sizeof(m_za_header)) { |
378 | // This will get reconfigured/reset later, so we are safe to use it. |
379 | // ZA is a square of VL * VL and the ptrace buffer also includes the |
380 | // header itself. |
381 | m_za_ptrace_payload.resize(((m_za_header.vl) * (m_za_header.vl)) + |
382 | GetZAHeaderSize()); |
383 | std::fill(m_za_ptrace_payload.begin(), m_za_ptrace_payload.end(), 0); |
384 | } else { |
385 | // ZA is active, read the real register. |
386 | error = ReadZA(); |
387 | if (error.Fail()) |
388 | return error; |
389 | } |
390 | |
391 | // ZA is part of the SME set but uses a seperate member buffer for |
392 | // storage. Therefore its effective byte offset is always 0 even if it |
393 | // isn't 0 within the SME register set. |
394 | src = (uint8_t *)GetZABuffer() + GetZAHeaderSize(); |
395 | } else if (GetRegisterInfo().IsSMERegZT(reg)) { |
396 | // Unlike ZA, the kernel will return register data for ZT0 when ZA is not |
397 | // enabled. This data will be all 0s so we don't have to invent anything |
398 | // like we did for ZA. |
399 | error = ReadZT(); |
400 | if (error.Fail()) |
401 | return error; |
402 | |
403 | src = (uint8_t *)GetZTBuffer(); |
404 | } else { |
405 | error = ReadSMESVG(); |
406 | if (error.Fail()) |
407 | return error; |
408 | |
409 | // This is a psuedo so it never fails. |
410 | ReadSMEControl(); |
411 | |
412 | offset = reg_info->byte_offset - GetRegisterInfo().GetSMEOffset(); |
413 | assert(offset < GetSMEPseudoBufferSize()); |
414 | src = (uint8_t *)GetSMEPseudoBuffer() + offset; |
415 | } |
416 | } else |
417 | return Status("failed - register wasn't recognized to be a GPR or an FPR, " |
418 | "write strategy unknown" ); |
419 | |
420 | reg_value.SetFromMemoryData(*reg_info, src, reg_info->byte_size, |
421 | eByteOrderLittle, error); |
422 | |
423 | return error; |
424 | } |
425 | |
426 | Status NativeRegisterContextLinux_arm64::WriteRegister( |
427 | const RegisterInfo *reg_info, const RegisterValue ®_value) { |
428 | Status error; |
429 | |
430 | if (!reg_info) |
431 | return Status("reg_info NULL" ); |
432 | |
433 | const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB]; |
434 | |
435 | if (reg == LLDB_INVALID_REGNUM) |
436 | return Status("no lldb regnum for %s" , reg_info && reg_info->name |
437 | ? reg_info->name |
438 | : "<unknown register>" ); |
439 | |
440 | uint8_t *dst; |
441 | uint32_t offset = LLDB_INVALID_INDEX32; |
442 | std::vector<uint8_t> sve_reg_non_live; |
443 | |
444 | if (IsGPR(reg)) { |
445 | error = ReadGPR(); |
446 | if (error.Fail()) |
447 | return error; |
448 | |
449 | assert(reg_info->byte_offset < GetGPRSize()); |
450 | dst = (uint8_t *)GetGPRBuffer() + reg_info->byte_offset; |
451 | ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size); |
452 | |
453 | return WriteGPR(); |
454 | } else if (IsFPR(reg)) { |
455 | if (m_sve_state == SVEState::Disabled) { |
456 | // SVE is disabled take legacy route for FPU register access |
457 | error = ReadFPR(); |
458 | if (error.Fail()) |
459 | return error; |
460 | |
461 | offset = CalculateFprOffset(reg_info); |
462 | assert(offset < GetFPRSize()); |
463 | dst = (uint8_t *)GetFPRBuffer() + offset; |
464 | ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size); |
465 | |
466 | return WriteFPR(); |
467 | } else { |
468 | // SVE enabled, we will read and cache SVE ptrace data. |
469 | error = ReadAllSVE(); |
470 | if (error.Fail()) |
471 | return error; |
472 | |
473 | // FPSR and FPCR will be located right after Z registers in |
474 | // SVEState::FPSIMD while in SVEState::Full or SVEState::Streaming they |
475 | // will be located at the end of register data after an alignment |
476 | // correction based on currently selected vector length. |
477 | uint32_t sve_reg_num = LLDB_INVALID_REGNUM; |
478 | if (reg == GetRegisterInfo().GetRegNumFPSR()) { |
479 | sve_reg_num = reg; |
480 | if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming) |
481 | offset = sve::PTraceFPSROffset(sve::vq_from_vl(m_sve_header.vl)); |
482 | else if (m_sve_state == SVEState::FPSIMD) |
483 | offset = sve::ptrace_fpsimd_offset + (32 * 16); |
484 | } else if (reg == GetRegisterInfo().GetRegNumFPCR()) { |
485 | sve_reg_num = reg; |
486 | if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming) |
487 | offset = sve::PTraceFPCROffset(sve::vq_from_vl(m_sve_header.vl)); |
488 | else if (m_sve_state == SVEState::FPSIMD) |
489 | offset = sve::ptrace_fpsimd_offset + (32 * 16) + 4; |
490 | } else { |
491 | // Extract SVE Z register value register number for this reg_info |
492 | if (reg_info->value_regs && |
493 | reg_info->value_regs[0] != LLDB_INVALID_REGNUM) |
494 | sve_reg_num = reg_info->value_regs[0]; |
495 | offset = CalculateSVEOffset(GetRegisterInfoAtIndex(sve_reg_num)); |
496 | } |
497 | |
498 | assert(offset < GetSVEBufferSize()); |
499 | dst = (uint8_t *)GetSVEBuffer() + offset; |
500 | ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size); |
501 | return WriteAllSVE(); |
502 | } |
503 | } else if (IsSVE(reg)) { |
504 | if (m_sve_state == SVEState::Disabled || m_sve_state == SVEState::Unknown) |
505 | return Status("SVE disabled or not supported" ); |
506 | else { |
507 | // Target has SVE enabled, we will read and cache SVE ptrace data |
508 | error = ReadAllSVE(); |
509 | if (error.Fail()) |
510 | return error; |
511 | |
512 | if (GetRegisterInfo().IsSVERegVG(reg)) { |
513 | uint64_t vg_value = reg_value.GetAsUInt64(); |
514 | |
515 | if (sve::vl_valid(vg_value * 8)) { |
516 | if (m_sve_header_is_valid && vg_value == GetSVERegVG()) |
517 | return error; |
518 | |
519 | SetSVERegVG(vg_value); |
520 | |
521 | error = WriteSVEHeader(); |
522 | if (error.Success()) { |
523 | // Changing VG during streaming mode also changes the size of ZA. |
524 | if (m_sve_state == SVEState::Streaming) |
525 | m_za_header_is_valid = false; |
526 | ConfigureRegisterContext(); |
527 | } |
528 | |
529 | if (m_sve_header_is_valid && vg_value == GetSVERegVG()) |
530 | return error; |
531 | } |
532 | |
533 | return Status("SVE vector length update failed." ); |
534 | } |
535 | |
536 | // If target supports SVE but currently in FPSIMD mode. |
537 | if (m_sve_state == SVEState::FPSIMD) { |
538 | // Here we will check if writing this SVE register enables |
539 | // SVEState::Full |
540 | bool set_sve_state_full = false; |
541 | const uint8_t *reg_bytes = (const uint8_t *)reg_value.GetBytes(); |
542 | if (GetRegisterInfo().IsSVEZReg(reg)) { |
543 | for (uint32_t i = 16; i < reg_info->byte_size; i++) { |
544 | if (reg_bytes[i]) { |
545 | set_sve_state_full = true; |
546 | break; |
547 | } |
548 | } |
549 | } else if (GetRegisterInfo().IsSVEPReg(reg) || |
550 | reg == GetRegisterInfo().GetRegNumSVEFFR()) { |
551 | for (uint32_t i = 0; i < reg_info->byte_size; i++) { |
552 | if (reg_bytes[i]) { |
553 | set_sve_state_full = true; |
554 | break; |
555 | } |
556 | } |
557 | } |
558 | |
559 | if (!set_sve_state_full && GetRegisterInfo().IsSVEZReg(reg)) { |
560 | // We are writing a Z register which is zero beyond 16 bytes so copy |
561 | // first 16 bytes only as SVE payload mirrors legacy fpsimd structure |
562 | offset = CalculateSVEOffset(reg_info); |
563 | assert(offset < GetSVEBufferSize()); |
564 | dst = (uint8_t *)GetSVEBuffer() + offset; |
565 | ::memcpy(dst, reg_value.GetBytes(), 16); |
566 | |
567 | return WriteAllSVE(); |
568 | } else |
569 | return Status("SVE state change operation not supported" ); |
570 | } else { |
571 | offset = CalculateSVEOffset(reg_info); |
572 | assert(offset < GetSVEBufferSize()); |
573 | dst = (uint8_t *)GetSVEBuffer() + offset; |
574 | ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size); |
575 | return WriteAllSVE(); |
576 | } |
577 | } |
578 | } else if (IsMTE(reg)) { |
579 | error = ReadMTEControl(); |
580 | if (error.Fail()) |
581 | return error; |
582 | |
583 | offset = reg_info->byte_offset - GetRegisterInfo().GetMTEOffset(); |
584 | assert(offset < GetMTEControlSize()); |
585 | dst = (uint8_t *)GetMTEControl() + offset; |
586 | ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size); |
587 | |
588 | return WriteMTEControl(); |
589 | } else if (IsTLS(reg)) { |
590 | error = ReadTLS(); |
591 | if (error.Fail()) |
592 | return error; |
593 | |
594 | offset = reg_info->byte_offset - GetRegisterInfo().GetTLSOffset(); |
595 | assert(offset < GetTLSBufferSize()); |
596 | dst = (uint8_t *)GetTLSBuffer() + offset; |
597 | ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size); |
598 | |
599 | return WriteTLS(); |
600 | } else if (IsSME(reg)) { |
601 | if (GetRegisterInfo().IsSMERegZA(reg)) { |
602 | error = ReadZA(); |
603 | if (error.Fail()) |
604 | return error; |
605 | |
606 | // ZA is part of the SME set but not stored with the other SME registers. |
607 | // So its byte offset is effectively always 0. |
608 | dst = (uint8_t *)GetZABuffer() + GetZAHeaderSize(); |
609 | ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size); |
610 | |
611 | // While this is writing a header that contains a vector length, the only |
612 | // way to change that is via the vg register. So here we assume the length |
613 | // will always be the current length and no reconfigure is needed. |
614 | return WriteZA(); |
615 | } else if (GetRegisterInfo().IsSMERegZT(reg)) { |
616 | error = ReadZT(); |
617 | if (error.Fail()) |
618 | return error; |
619 | |
620 | dst = (uint8_t *)GetZTBuffer(); |
621 | ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size); |
622 | |
623 | return WriteZT(); |
624 | } else |
625 | return Status("Writing to SVG or SVCR is not supported." ); |
626 | } |
627 | |
628 | return Status("Failed to write register value" ); |
629 | } |
630 | |
631 | enum RegisterSetType : uint32_t { |
632 | GPR, |
633 | SVE, // Used for SVE and SSVE. |
634 | FPR, // When there is no SVE, or SVE in FPSIMD mode. |
635 | // Pointer authentication registers are read only, so not included here. |
636 | MTE, |
637 | TLS, |
638 | SME, // ZA only, because SVCR and SVG are pseudo registers. |
639 | SME2, // ZT only. |
640 | }; |
641 | |
642 | static uint8_t *AddRegisterSetType(uint8_t *dst, |
643 | RegisterSetType register_set_type) { |
644 | *(reinterpret_cast<uint32_t *>(dst)) = register_set_type; |
645 | return dst + sizeof(uint32_t); |
646 | } |
647 | |
648 | static uint8_t *AddSavedRegistersData(uint8_t *dst, void *src, size_t size) { |
649 | ::memcpy(dst, src, size); |
650 | return dst + size; |
651 | } |
652 | |
653 | static uint8_t *AddSavedRegisters(uint8_t *dst, |
654 | enum RegisterSetType register_set_type, |
655 | void *src, size_t size) { |
656 | dst = AddRegisterSetType(dst, register_set_type); |
657 | return AddSavedRegistersData(dst, src, size); |
658 | } |
659 | |
660 | Status |
661 | NativeRegisterContextLinux_arm64::CacheAllRegisters(uint32_t &cached_size) { |
662 | Status error; |
663 | cached_size = sizeof(RegisterSetType) + GetGPRBufferSize(); |
664 | error = ReadGPR(); |
665 | if (error.Fail()) |
666 | return error; |
667 | |
668 | if (GetRegisterInfo().IsZAPresent()) { |
669 | error = ReadZAHeader(); |
670 | if (error.Fail()) |
671 | return error; |
672 | // Use header size here because the buffer may contain fake data when ZA is |
673 | // disabled. We do not want to write this fake data (all 0s) because this |
674 | // would tell the kernel that we want ZA to become active. Which is the |
675 | // opposite of what we want in the case where it is currently inactive. |
676 | cached_size += sizeof(RegisterSetType) + m_za_header.size; |
677 | // For the same reason, we need to force it to be re-read so that it will |
678 | // always contain the real header. |
679 | m_za_buffer_is_valid = false; |
680 | error = ReadZA(); |
681 | if (error.Fail()) |
682 | return error; |
683 | |
684 | // We will only be restoring ZT data if ZA is active. As writing to an |
685 | // inactive ZT enables ZA, which may not be desireable. |
686 | if ( |
687 | // If we have ZT0, or in other words, if we have SME2. |
688 | GetRegisterInfo().IsZTPresent() && |
689 | // And ZA is active, which means that ZT0 is also active. |
690 | m_za_header.size > sizeof(m_za_header)) { |
691 | cached_size += sizeof(RegisterSetType) + GetZTBufferSize(); |
692 | // The kernel handles an inactive ZT0 for us, and it will read as 0s if |
693 | // inactive (unlike ZA where we fake that behaviour). |
694 | error = ReadZT(); |
695 | if (error.Fail()) |
696 | return error; |
697 | } |
698 | } |
699 | |
700 | // If SVE is enabled we need not copy FPR separately. |
701 | if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent()) { |
702 | // Store mode and register data. |
703 | cached_size += |
704 | sizeof(RegisterSetType) + sizeof(m_sve_state) + GetSVEBufferSize(); |
705 | error = ReadAllSVE(); |
706 | } else { |
707 | cached_size += sizeof(RegisterSetType) + GetFPRSize(); |
708 | error = ReadFPR(); |
709 | } |
710 | if (error.Fail()) |
711 | return error; |
712 | |
713 | if (GetRegisterInfo().IsMTEPresent()) { |
714 | cached_size += sizeof(RegisterSetType) + GetMTEControlSize(); |
715 | error = ReadMTEControl(); |
716 | if (error.Fail()) |
717 | return error; |
718 | } |
719 | |
720 | // tpidr is always present but tpidr2 depends on SME. |
721 | cached_size += sizeof(RegisterSetType) + GetTLSBufferSize(); |
722 | error = ReadTLS(); |
723 | |
724 | return error; |
725 | } |
726 | |
727 | Status NativeRegisterContextLinux_arm64::ReadAllRegisterValues( |
728 | lldb::WritableDataBufferSP &data_sp) { |
729 | // AArch64 register data must contain GPRs and either FPR or SVE registers. |
730 | // SVE registers can be non-streaming (aka SVE) or streaming (aka SSVE). |
731 | // Finally an optional MTE register. Pointer Authentication (PAC) registers |
732 | // are read-only and will be skipped. |
733 | |
734 | // In order to create register data checkpoint we first read all register |
735 | // values if not done already and calculate total size of register set data. |
736 | // We store all register values in data_sp by copying full PTrace data that |
737 | // corresponds to register sets enabled by current register context. |
738 | |
739 | uint32_t reg_data_byte_size = 0; |
740 | Status error = CacheAllRegisters(reg_data_byte_size); |
741 | if (error.Fail()) |
742 | return error; |
743 | |
744 | data_sp.reset(new DataBufferHeap(reg_data_byte_size, 0)); |
745 | uint8_t *dst = data_sp->GetBytes(); |
746 | |
747 | dst = AddSavedRegisters(dst, RegisterSetType::GPR, GetGPRBuffer(), |
748 | GetGPRBufferSize()); |
749 | |
750 | // Streaming SVE and the ZA register both use the streaming vector length. |
751 | // When you change this, the kernel will invalidate parts of the process |
752 | // state. Therefore we need a specific order of restoration for each mode, if |
753 | // we also have ZA to restore. |
754 | // |
755 | // Streaming mode enabled, ZA enabled: |
756 | // * Write streaming registers. This sets SVCR.SM and clears SVCR.ZA. |
757 | // * Write ZA, this set SVCR.ZA. The register data we provide is written to |
758 | // ZA. |
759 | // * Result is SVCR.SM and SVCR.ZA set, with the expected data in both |
760 | // register sets. |
761 | // |
762 | // Streaming mode disabled, ZA enabled: |
763 | // * Write ZA. This sets SVCR.ZA, and the ZA content. In the majority of cases |
764 | // the streaming vector length is changing, so the thread is converted into |
765 | // an FPSIMD thread if it is not already one. This also clears SVCR.SM. |
766 | // * Write SVE registers, which also clears SVCR.SM but most importantly, puts |
767 | // us into full SVE mode instead of FPSIMD mode (where the registers are |
768 | // actually the 128 bit Neon registers). |
769 | // * Result is we have SVCR.SM = 0, SVCR.ZA = 1 and the expected register |
770 | // state. |
771 | // |
772 | // Restoring in different orders leads to things like the SVE registers being |
773 | // truncated due to the FPSIMD mode and ZA being disabled or filled with 0s |
774 | // (disabled and 0s looks the same from inside lldb since we fake the value |
775 | // when it's disabled). |
776 | // |
777 | // For more information on this, look up the uses of the relevant NT_ARM_ |
778 | // constants and the functions vec_set_vector_length, sve_set_common and |
779 | // za_set in the Linux Kernel. |
780 | |
781 | if ((m_sve_state != SVEState::Streaming) && GetRegisterInfo().IsZAPresent()) { |
782 | // Use the header size not the buffer size, as we may be using the buffer |
783 | // for fake data, which we do not want to write out. |
784 | assert(m_za_header.size <= GetZABufferSize()); |
785 | dst = AddSavedRegisters(dst, RegisterSetType::SME, GetZABuffer(), |
786 | m_za_header.size); |
787 | } |
788 | |
789 | if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent()) { |
790 | dst = AddRegisterSetType(dst, RegisterSetType::SVE); |
791 | *(reinterpret_cast<SVEState *>(dst)) = m_sve_state; |
792 | dst += sizeof(m_sve_state); |
793 | dst = AddSavedRegistersData(dst, GetSVEBuffer(), GetSVEBufferSize()); |
794 | } else { |
795 | dst = AddSavedRegisters(dst, RegisterSetType::FPR, GetFPRBuffer(), |
796 | GetFPRSize()); |
797 | } |
798 | |
799 | if ((m_sve_state == SVEState::Streaming) && GetRegisterInfo().IsZAPresent()) { |
800 | assert(m_za_header.size <= GetZABufferSize()); |
801 | dst = AddSavedRegisters(dst, RegisterSetType::SME, GetZABuffer(), |
802 | m_za_header.size); |
803 | } |
804 | |
805 | // If ZT0 is present and we are going to be restoring an active ZA (which |
806 | // implies an active ZT0), then restore ZT0 after ZA has been set. This |
807 | // prevents us enabling ZA accidentally after the restore of ZA disabled it. |
808 | // If we leave ZA/ZT0 inactive and read ZT0, the kernel returns 0s. Therefore |
809 | // there's nothing for us to restore if ZA was originally inactive. |
810 | if ( |
811 | // If we have SME2 and therefore ZT0. |
812 | GetRegisterInfo().IsZTPresent() && |
813 | // And ZA is enabled. |
814 | m_za_header.size > sizeof(m_za_header)) |
815 | dst = AddSavedRegisters(dst, RegisterSetType::SME2, GetZTBuffer(), |
816 | GetZTBufferSize()); |
817 | |
818 | if (GetRegisterInfo().IsMTEPresent()) { |
819 | dst = AddSavedRegisters(dst, RegisterSetType::MTE, GetMTEControl(), |
820 | GetMTEControlSize()); |
821 | } |
822 | |
823 | dst = AddSavedRegisters(dst, RegisterSetType::TLS, GetTLSBuffer(), |
824 | GetTLSBufferSize()); |
825 | |
826 | return error; |
827 | } |
828 | |
829 | static Status RestoreRegisters(void *buffer, const uint8_t **src, size_t len, |
830 | bool &is_valid, std::function<Status()> writer) { |
831 | ::memcpy(buffer, *src, len); |
832 | is_valid = true; |
833 | *src += len; |
834 | return writer(); |
835 | } |
836 | |
837 | Status NativeRegisterContextLinux_arm64::WriteAllRegisterValues( |
838 | const lldb::DataBufferSP &data_sp) { |
839 | // AArch64 register data must contain GPRs, either FPR or SVE registers |
840 | // (which can be streaming or non-streaming) and optional MTE register. |
841 | // Pointer Authentication (PAC) registers are read-only and will be skipped. |
842 | |
843 | // We store all register values in data_sp by copying full PTrace data that |
844 | // corresponds to register sets enabled by current register context. In order |
845 | // to restore from register data checkpoint we will first restore GPRs, based |
846 | // on size of remaining register data either SVE or FPRs should be restored |
847 | // next. SVE is not enabled if we have register data size less than or equal |
848 | // to size of GPR + FPR + MTE. |
849 | |
850 | Status error; |
851 | if (!data_sp) { |
852 | error.SetErrorStringWithFormat( |
853 | "NativeRegisterContextLinux_arm64::%s invalid data_sp provided" , |
854 | __FUNCTION__); |
855 | return error; |
856 | } |
857 | |
858 | const uint8_t *src = data_sp->GetBytes(); |
859 | if (src == nullptr) { |
860 | error.SetErrorStringWithFormat("NativeRegisterContextLinux_arm64::%s " |
861 | "DataBuffer::GetBytes() returned a null " |
862 | "pointer" , |
863 | __FUNCTION__); |
864 | return error; |
865 | } |
866 | |
867 | uint64_t reg_data_min_size = |
868 | GetGPRBufferSize() + GetFPRSize() + 2 * (sizeof(RegisterSetType)); |
869 | if (data_sp->GetByteSize() < reg_data_min_size) { |
870 | error.SetErrorStringWithFormat( |
871 | "NativeRegisterContextLinux_arm64::%s data_sp contained insufficient " |
872 | "register data bytes, expected at least %" PRIu64 ", actual %" PRIu64, |
873 | __FUNCTION__, reg_data_min_size, data_sp->GetByteSize()); |
874 | return error; |
875 | } |
876 | |
877 | const uint8_t *end = src + data_sp->GetByteSize(); |
878 | while (src < end) { |
879 | const RegisterSetType kind = |
880 | *reinterpret_cast<const RegisterSetType *>(src); |
881 | src += sizeof(RegisterSetType); |
882 | |
883 | switch (kind) { |
884 | case RegisterSetType::GPR: |
885 | error = RestoreRegisters( |
886 | GetGPRBuffer(), &src, GetGPRBufferSize(), m_gpr_is_valid, |
887 | std::bind(&NativeRegisterContextLinux_arm64::WriteGPR, this)); |
888 | break; |
889 | case RegisterSetType::SVE: |
890 | // Restore to the correct mode, streaming or not. |
891 | m_sve_state = static_cast<SVEState>(*src); |
892 | src += sizeof(m_sve_state); |
893 | |
894 | // First write SVE header. We do not use RestoreRegisters because we do |
895 | // not want src to be modified yet. |
896 | ::memcpy(GetSVEHeader(), src, GetSVEHeaderSize()); |
897 | if (!sve::vl_valid(m_sve_header.vl)) { |
898 | m_sve_header_is_valid = false; |
899 | error.SetErrorStringWithFormat("NativeRegisterContextLinux_arm64::%s " |
900 | "Invalid SVE header in data_sp" , |
901 | __FUNCTION__); |
902 | return error; |
903 | } |
904 | m_sve_header_is_valid = true; |
905 | error = WriteSVEHeader(); |
906 | if (error.Fail()) |
907 | return error; |
908 | |
909 | // SVE header has been written configure SVE vector length if needed. |
910 | // This could change ZA data too, but that will be restored again later |
911 | // anyway. |
912 | ConfigureRegisterContext(); |
913 | |
914 | // Write header and register data, incrementing src this time. |
915 | error = RestoreRegisters( |
916 | GetSVEBuffer(), &src, GetSVEBufferSize(), m_sve_buffer_is_valid, |
917 | std::bind(&NativeRegisterContextLinux_arm64::WriteAllSVE, this)); |
918 | break; |
919 | case RegisterSetType::FPR: |
920 | error = RestoreRegisters( |
921 | GetFPRBuffer(), &src, GetFPRSize(), m_fpu_is_valid, |
922 | std::bind(&NativeRegisterContextLinux_arm64::WriteFPR, this)); |
923 | break; |
924 | case RegisterSetType::MTE: |
925 | error = RestoreRegisters( |
926 | GetMTEControl(), &src, GetMTEControlSize(), m_mte_ctrl_is_valid, |
927 | std::bind(&NativeRegisterContextLinux_arm64::WriteMTEControl, this)); |
928 | break; |
929 | case RegisterSetType::TLS: |
930 | error = RestoreRegisters( |
931 | GetTLSBuffer(), &src, GetTLSBufferSize(), m_tls_is_valid, |
932 | std::bind(&NativeRegisterContextLinux_arm64::WriteTLS, this)); |
933 | break; |
934 | case RegisterSetType::SME: |
935 | // To enable or disable ZA you write the regset with or without register |
936 | // data. The kernel detects this by looking at the ioVec's length, not the |
937 | // ZA header size you pass in. Therefore we must write header and register |
938 | // data (if present) in one go every time. Read the header only first just |
939 | // to get the size. |
940 | ::memcpy(GetZAHeader(), src, GetZAHeaderSize()); |
941 | // Read the header and register data. Can't use the buffer size here, it |
942 | // may be incorrect due to being filled with dummy data previously. Resize |
943 | // this so WriteZA uses the correct size. |
944 | m_za_ptrace_payload.resize(m_za_header.size); |
945 | ::memcpy(GetZABuffer(), src, GetZABufferSize()); |
946 | m_za_buffer_is_valid = true; |
947 | |
948 | error = WriteZA(); |
949 | if (error.Fail()) |
950 | return error; |
951 | |
952 | // Update size of ZA, which resizes the ptrace payload potentially |
953 | // trashing our copy of the data we just wrote. |
954 | ConfigureRegisterContext(); |
955 | |
956 | // ZA buffer now has proper size, read back the data we wrote above, from |
957 | // ptrace. |
958 | error = ReadZA(); |
959 | src += GetZABufferSize(); |
960 | break; |
961 | case RegisterSetType::SME2: |
962 | // Doing this would activate an inactive ZA, however we will only get here |
963 | // if the state we are restoring had an active ZA. Restoring ZT0 will |
964 | // always come after restoring ZA. |
965 | error = RestoreRegisters( |
966 | GetZTBuffer(), &src, GetZTBufferSize(), m_zt_buffer_is_valid, |
967 | std::bind(&NativeRegisterContextLinux_arm64::WriteZT, this)); |
968 | break; |
969 | } |
970 | |
971 | if (error.Fail()) |
972 | return error; |
973 | } |
974 | |
975 | return error; |
976 | } |
977 | |
978 | bool NativeRegisterContextLinux_arm64::IsGPR(unsigned reg) const { |
979 | if (GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg) == |
980 | RegisterInfoPOSIX_arm64::GPRegSet) |
981 | return true; |
982 | return false; |
983 | } |
984 | |
985 | bool NativeRegisterContextLinux_arm64::IsFPR(unsigned reg) const { |
986 | if (GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg) == |
987 | RegisterInfoPOSIX_arm64::FPRegSet) |
988 | return true; |
989 | return false; |
990 | } |
991 | |
992 | bool NativeRegisterContextLinux_arm64::IsSVE(unsigned reg) const { |
993 | return GetRegisterInfo().IsSVEReg(reg); |
994 | } |
995 | |
996 | bool NativeRegisterContextLinux_arm64::IsSME(unsigned reg) const { |
997 | return GetRegisterInfo().IsSMEReg(reg); |
998 | } |
999 | |
1000 | bool NativeRegisterContextLinux_arm64::IsPAuth(unsigned reg) const { |
1001 | return GetRegisterInfo().IsPAuthReg(reg); |
1002 | } |
1003 | |
1004 | bool NativeRegisterContextLinux_arm64::IsMTE(unsigned reg) const { |
1005 | return GetRegisterInfo().IsMTEReg(reg); |
1006 | } |
1007 | |
1008 | bool NativeRegisterContextLinux_arm64::IsTLS(unsigned reg) const { |
1009 | return GetRegisterInfo().IsTLSReg(reg); |
1010 | } |
1011 | |
1012 | llvm::Error NativeRegisterContextLinux_arm64::ReadHardwareDebugInfo() { |
1013 | if (!m_refresh_hwdebug_info) { |
1014 | return llvm::Error::success(); |
1015 | } |
1016 | |
1017 | ::pid_t tid = m_thread.GetID(); |
1018 | |
1019 | int regset = NT_ARM_HW_WATCH; |
1020 | struct iovec ioVec; |
1021 | struct user_hwdebug_state dreg_state; |
1022 | Status error; |
1023 | |
1024 | ioVec.iov_base = &dreg_state; |
1025 | ioVec.iov_len = sizeof(dreg_state); |
1026 | error = NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET, tid, ®set, |
1027 | &ioVec, ioVec.iov_len); |
1028 | |
1029 | if (error.Fail()) |
1030 | return error.ToError(); |
1031 | |
1032 | m_max_hwp_supported = dreg_state.dbg_info & 0xff; |
1033 | |
1034 | regset = NT_ARM_HW_BREAK; |
1035 | error = NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET, tid, ®set, |
1036 | &ioVec, ioVec.iov_len); |
1037 | |
1038 | if (error.Fail()) |
1039 | return error.ToError(); |
1040 | |
1041 | m_max_hbp_supported = dreg_state.dbg_info & 0xff; |
1042 | m_refresh_hwdebug_info = false; |
1043 | |
1044 | return llvm::Error::success(); |
1045 | } |
1046 | |
1047 | llvm::Error |
1048 | NativeRegisterContextLinux_arm64::WriteHardwareDebugRegs(DREGType hwbType) { |
1049 | struct iovec ioVec; |
1050 | struct user_hwdebug_state dreg_state; |
1051 | int regset; |
1052 | |
1053 | memset(&dreg_state, 0, sizeof(dreg_state)); |
1054 | ioVec.iov_base = &dreg_state; |
1055 | |
1056 | switch (hwbType) { |
1057 | case eDREGTypeWATCH: |
1058 | regset = NT_ARM_HW_WATCH; |
1059 | ioVec.iov_len = sizeof(dreg_state.dbg_info) + sizeof(dreg_state.pad) + |
1060 | (sizeof(dreg_state.dbg_regs[0]) * m_max_hwp_supported); |
1061 | |
1062 | for (uint32_t i = 0; i < m_max_hwp_supported; i++) { |
1063 | dreg_state.dbg_regs[i].addr = m_hwp_regs[i].address; |
1064 | dreg_state.dbg_regs[i].ctrl = m_hwp_regs[i].control; |
1065 | } |
1066 | break; |
1067 | case eDREGTypeBREAK: |
1068 | regset = NT_ARM_HW_BREAK; |
1069 | ioVec.iov_len = sizeof(dreg_state.dbg_info) + sizeof(dreg_state.pad) + |
1070 | (sizeof(dreg_state.dbg_regs[0]) * m_max_hbp_supported); |
1071 | |
1072 | for (uint32_t i = 0; i < m_max_hbp_supported; i++) { |
1073 | dreg_state.dbg_regs[i].addr = m_hbp_regs[i].address; |
1074 | dreg_state.dbg_regs[i].ctrl = m_hbp_regs[i].control; |
1075 | } |
1076 | break; |
1077 | } |
1078 | |
1079 | return NativeProcessLinux::PtraceWrapper(PTRACE_SETREGSET, m_thread.GetID(), |
1080 | ®set, &ioVec, ioVec.iov_len) |
1081 | .ToError(); |
1082 | } |
1083 | |
1084 | Status NativeRegisterContextLinux_arm64::ReadGPR() { |
1085 | Status error; |
1086 | |
1087 | if (m_gpr_is_valid) |
1088 | return error; |
1089 | |
1090 | struct iovec ioVec; |
1091 | ioVec.iov_base = GetGPRBuffer(); |
1092 | ioVec.iov_len = GetGPRBufferSize(); |
1093 | |
1094 | error = ReadRegisterSet(&ioVec, GetGPRBufferSize(), NT_PRSTATUS); |
1095 | |
1096 | if (error.Success()) |
1097 | m_gpr_is_valid = true; |
1098 | |
1099 | return error; |
1100 | } |
1101 | |
1102 | Status NativeRegisterContextLinux_arm64::WriteGPR() { |
1103 | Status error = ReadGPR(); |
1104 | if (error.Fail()) |
1105 | return error; |
1106 | |
1107 | struct iovec ioVec; |
1108 | ioVec.iov_base = GetGPRBuffer(); |
1109 | ioVec.iov_len = GetGPRBufferSize(); |
1110 | |
1111 | m_gpr_is_valid = false; |
1112 | |
1113 | return WriteRegisterSet(&ioVec, GetGPRBufferSize(), NT_PRSTATUS); |
1114 | } |
1115 | |
1116 | Status NativeRegisterContextLinux_arm64::ReadFPR() { |
1117 | Status error; |
1118 | |
1119 | if (m_fpu_is_valid) |
1120 | return error; |
1121 | |
1122 | struct iovec ioVec; |
1123 | ioVec.iov_base = GetFPRBuffer(); |
1124 | ioVec.iov_len = GetFPRSize(); |
1125 | |
1126 | error = ReadRegisterSet(&ioVec, GetFPRSize(), NT_FPREGSET); |
1127 | |
1128 | if (error.Success()) |
1129 | m_fpu_is_valid = true; |
1130 | |
1131 | return error; |
1132 | } |
1133 | |
1134 | Status NativeRegisterContextLinux_arm64::WriteFPR() { |
1135 | Status error = ReadFPR(); |
1136 | if (error.Fail()) |
1137 | return error; |
1138 | |
1139 | struct iovec ioVec; |
1140 | ioVec.iov_base = GetFPRBuffer(); |
1141 | ioVec.iov_len = GetFPRSize(); |
1142 | |
1143 | m_fpu_is_valid = false; |
1144 | |
1145 | return WriteRegisterSet(&ioVec, GetFPRSize(), NT_FPREGSET); |
1146 | } |
1147 | |
1148 | void NativeRegisterContextLinux_arm64::InvalidateAllRegisters() { |
1149 | m_gpr_is_valid = false; |
1150 | m_fpu_is_valid = false; |
1151 | m_sve_buffer_is_valid = false; |
1152 | m_sve_header_is_valid = false; |
1153 | m_za_buffer_is_valid = false; |
1154 | m_za_header_is_valid = false; |
1155 | m_pac_mask_is_valid = false; |
1156 | m_mte_ctrl_is_valid = false; |
1157 | m_tls_is_valid = false; |
1158 | m_zt_buffer_is_valid = false; |
1159 | |
1160 | // Update SVE and ZA registers in case there is change in configuration. |
1161 | ConfigureRegisterContext(); |
1162 | } |
1163 | |
1164 | unsigned NativeRegisterContextLinux_arm64::GetSVERegSet() { |
1165 | return m_sve_state == SVEState::Streaming ? NT_ARM_SSVE : NT_ARM_SVE; |
1166 | } |
1167 | |
1168 | Status NativeRegisterContextLinux_arm64::ReadSVEHeader() { |
1169 | Status error; |
1170 | |
1171 | if (m_sve_header_is_valid) |
1172 | return error; |
1173 | |
1174 | struct iovec ioVec; |
1175 | ioVec.iov_base = GetSVEHeader(); |
1176 | ioVec.iov_len = GetSVEHeaderSize(); |
1177 | |
1178 | error = ReadRegisterSet(&ioVec, GetSVEHeaderSize(), GetSVERegSet()); |
1179 | |
1180 | if (error.Success()) |
1181 | m_sve_header_is_valid = true; |
1182 | |
1183 | return error; |
1184 | } |
1185 | |
1186 | Status NativeRegisterContextLinux_arm64::ReadPAuthMask() { |
1187 | Status error; |
1188 | |
1189 | if (m_pac_mask_is_valid) |
1190 | return error; |
1191 | |
1192 | struct iovec ioVec; |
1193 | ioVec.iov_base = GetPACMask(); |
1194 | ioVec.iov_len = GetPACMaskSize(); |
1195 | |
1196 | error = ReadRegisterSet(&ioVec, GetPACMaskSize(), NT_ARM_PAC_MASK); |
1197 | |
1198 | if (error.Success()) |
1199 | m_pac_mask_is_valid = true; |
1200 | |
1201 | return error; |
1202 | } |
1203 | |
1204 | Status NativeRegisterContextLinux_arm64::WriteSVEHeader() { |
1205 | Status error; |
1206 | |
1207 | error = ReadSVEHeader(); |
1208 | if (error.Fail()) |
1209 | return error; |
1210 | |
1211 | struct iovec ioVec; |
1212 | ioVec.iov_base = GetSVEHeader(); |
1213 | ioVec.iov_len = GetSVEHeaderSize(); |
1214 | |
1215 | m_sve_buffer_is_valid = false; |
1216 | m_sve_header_is_valid = false; |
1217 | m_fpu_is_valid = false; |
1218 | |
1219 | return WriteRegisterSet(&ioVec, GetSVEHeaderSize(), GetSVERegSet()); |
1220 | } |
1221 | |
1222 | Status NativeRegisterContextLinux_arm64::ReadAllSVE() { |
1223 | Status error; |
1224 | if (m_sve_buffer_is_valid) |
1225 | return error; |
1226 | |
1227 | struct iovec ioVec; |
1228 | ioVec.iov_base = GetSVEBuffer(); |
1229 | ioVec.iov_len = GetSVEBufferSize(); |
1230 | |
1231 | error = ReadRegisterSet(&ioVec, GetSVEBufferSize(), GetSVERegSet()); |
1232 | |
1233 | if (error.Success()) |
1234 | m_sve_buffer_is_valid = true; |
1235 | |
1236 | return error; |
1237 | } |
1238 | |
1239 | Status NativeRegisterContextLinux_arm64::WriteAllSVE() { |
1240 | Status error; |
1241 | |
1242 | error = ReadAllSVE(); |
1243 | if (error.Fail()) |
1244 | return error; |
1245 | |
1246 | struct iovec ioVec; |
1247 | |
1248 | ioVec.iov_base = GetSVEBuffer(); |
1249 | ioVec.iov_len = GetSVEBufferSize(); |
1250 | |
1251 | m_sve_buffer_is_valid = false; |
1252 | m_sve_header_is_valid = false; |
1253 | m_fpu_is_valid = false; |
1254 | |
1255 | return WriteRegisterSet(&ioVec, GetSVEBufferSize(), GetSVERegSet()); |
1256 | } |
1257 | |
1258 | Status NativeRegisterContextLinux_arm64::ReadSMEControl() { |
1259 | // The real register is SVCR and is accessible from EL0. However we don't want |
1260 | // to have to JIT code into the target process so we'll just recreate it using |
1261 | // what we know from ptrace. |
1262 | |
1263 | // Bit 0 indicates whether streaming mode is active. |
1264 | m_sme_pseudo_regs.ctrl_reg = m_sve_state == SVEState::Streaming; |
1265 | |
1266 | // Bit 1 indicates whether the array storage is active. |
1267 | // It is active if we can read the header and the size field tells us that |
1268 | // there is register data following it. |
1269 | Status error = ReadZAHeader(); |
1270 | if (error.Success() && (m_za_header.size > sizeof(m_za_header))) |
1271 | m_sme_pseudo_regs.ctrl_reg |= 2; |
1272 | |
1273 | return error; |
1274 | } |
1275 | |
1276 | Status NativeRegisterContextLinux_arm64::ReadMTEControl() { |
1277 | Status error; |
1278 | |
1279 | if (m_mte_ctrl_is_valid) |
1280 | return error; |
1281 | |
1282 | struct iovec ioVec; |
1283 | ioVec.iov_base = GetMTEControl(); |
1284 | ioVec.iov_len = GetMTEControlSize(); |
1285 | |
1286 | error = ReadRegisterSet(&ioVec, GetMTEControlSize(), NT_ARM_TAGGED_ADDR_CTRL); |
1287 | |
1288 | if (error.Success()) |
1289 | m_mte_ctrl_is_valid = true; |
1290 | |
1291 | return error; |
1292 | } |
1293 | |
1294 | Status NativeRegisterContextLinux_arm64::WriteMTEControl() { |
1295 | Status error; |
1296 | |
1297 | error = ReadMTEControl(); |
1298 | if (error.Fail()) |
1299 | return error; |
1300 | |
1301 | struct iovec ioVec; |
1302 | ioVec.iov_base = GetMTEControl(); |
1303 | ioVec.iov_len = GetMTEControlSize(); |
1304 | |
1305 | m_mte_ctrl_is_valid = false; |
1306 | |
1307 | return WriteRegisterSet(&ioVec, GetMTEControlSize(), NT_ARM_TAGGED_ADDR_CTRL); |
1308 | } |
1309 | |
1310 | Status NativeRegisterContextLinux_arm64::ReadTLS() { |
1311 | Status error; |
1312 | |
1313 | if (m_tls_is_valid) |
1314 | return error; |
1315 | |
1316 | struct iovec ioVec; |
1317 | ioVec.iov_base = GetTLSBuffer(); |
1318 | ioVec.iov_len = GetTLSBufferSize(); |
1319 | |
1320 | error = ReadRegisterSet(&ioVec, GetTLSBufferSize(), NT_ARM_TLS); |
1321 | |
1322 | if (error.Success()) |
1323 | m_tls_is_valid = true; |
1324 | |
1325 | return error; |
1326 | } |
1327 | |
1328 | Status NativeRegisterContextLinux_arm64::WriteTLS() { |
1329 | Status error; |
1330 | |
1331 | error = ReadTLS(); |
1332 | if (error.Fail()) |
1333 | return error; |
1334 | |
1335 | struct iovec ioVec; |
1336 | ioVec.iov_base = GetTLSBuffer(); |
1337 | ioVec.iov_len = GetTLSBufferSize(); |
1338 | |
1339 | m_tls_is_valid = false; |
1340 | |
1341 | return WriteRegisterSet(&ioVec, GetTLSBufferSize(), NT_ARM_TLS); |
1342 | } |
1343 | |
1344 | Status NativeRegisterContextLinux_arm64::ReadZAHeader() { |
1345 | Status error; |
1346 | |
1347 | if (m_za_header_is_valid) |
1348 | return error; |
1349 | |
1350 | struct iovec ioVec; |
1351 | ioVec.iov_base = GetZAHeader(); |
1352 | ioVec.iov_len = GetZAHeaderSize(); |
1353 | |
1354 | error = ReadRegisterSet(&ioVec, GetZAHeaderSize(), NT_ARM_ZA); |
1355 | |
1356 | if (error.Success()) |
1357 | m_za_header_is_valid = true; |
1358 | |
1359 | return error; |
1360 | } |
1361 | |
1362 | Status NativeRegisterContextLinux_arm64::ReadZA() { |
1363 | Status error; |
1364 | |
1365 | if (m_za_buffer_is_valid) |
1366 | return error; |
1367 | |
1368 | struct iovec ioVec; |
1369 | ioVec.iov_base = GetZABuffer(); |
1370 | ioVec.iov_len = GetZABufferSize(); |
1371 | |
1372 | error = ReadRegisterSet(&ioVec, GetZABufferSize(), NT_ARM_ZA); |
1373 | |
1374 | if (error.Success()) |
1375 | m_za_buffer_is_valid = true; |
1376 | |
1377 | return error; |
1378 | } |
1379 | |
1380 | Status NativeRegisterContextLinux_arm64::WriteZA() { |
1381 | // Note that because the ZA ptrace payload contains the header also, this |
1382 | // method will write both. This is done because writing only the header |
1383 | // will disable ZA, even if .size in the header is correct for an enabled ZA. |
1384 | Status error; |
1385 | |
1386 | error = ReadZA(); |
1387 | if (error.Fail()) |
1388 | return error; |
1389 | |
1390 | struct iovec ioVec; |
1391 | ioVec.iov_base = GetZABuffer(); |
1392 | ioVec.iov_len = GetZABufferSize(); |
1393 | |
1394 | m_za_buffer_is_valid = false; |
1395 | m_za_header_is_valid = false; |
1396 | // Writing to ZA may enable ZA, which means ZT0 may change too. |
1397 | m_zt_buffer_is_valid = false; |
1398 | |
1399 | return WriteRegisterSet(&ioVec, GetZABufferSize(), NT_ARM_ZA); |
1400 | } |
1401 | |
1402 | Status NativeRegisterContextLinux_arm64::ReadZT() { |
1403 | Status error; |
1404 | |
1405 | if (m_zt_buffer_is_valid) |
1406 | return error; |
1407 | |
1408 | struct iovec ioVec; |
1409 | ioVec.iov_base = GetZTBuffer(); |
1410 | ioVec.iov_len = GetZTBufferSize(); |
1411 | |
1412 | error = ReadRegisterSet(&ioVec, GetZTBufferSize(), NT_ARM_ZT); |
1413 | m_zt_buffer_is_valid = error.Success(); |
1414 | |
1415 | return error; |
1416 | } |
1417 | |
1418 | Status NativeRegisterContextLinux_arm64::WriteZT() { |
1419 | Status error; |
1420 | |
1421 | error = ReadZT(); |
1422 | if (error.Fail()) |
1423 | return error; |
1424 | |
1425 | struct iovec ioVec; |
1426 | ioVec.iov_base = GetZTBuffer(); |
1427 | ioVec.iov_len = GetZTBufferSize(); |
1428 | |
1429 | m_zt_buffer_is_valid = false; |
1430 | // Writing to an inactive ZT0 will enable ZA as well, which invalidates our |
1431 | // current copy of it. |
1432 | m_za_buffer_is_valid = false; |
1433 | m_za_header_is_valid = false; |
1434 | |
1435 | return WriteRegisterSet(&ioVec, GetZTBufferSize(), NT_ARM_ZT); |
1436 | } |
1437 | |
1438 | void NativeRegisterContextLinux_arm64::ConfigureRegisterContext() { |
1439 | // ConfigureRegisterContext gets called from InvalidateAllRegisters |
1440 | // on every stop and configures SVE vector length and whether we are in |
1441 | // streaming SVE mode. |
1442 | // If m_sve_state is set to SVEState::Disabled on first stop, code below will |
1443 | // be deemed non operational for the lifetime of current process. |
1444 | if (!m_sve_header_is_valid && m_sve_state != SVEState::Disabled) { |
1445 | // If we have SVE we may also have the SVE streaming mode that SME added. |
1446 | // We can read the header of either mode, but only the active mode will |
1447 | // have valid register data. |
1448 | |
1449 | // Check whether SME is present and the streaming SVE mode is active. |
1450 | m_sve_header_is_valid = false; |
1451 | m_sve_buffer_is_valid = false; |
1452 | m_sve_state = SVEState::Streaming; |
1453 | Status error = ReadSVEHeader(); |
1454 | |
1455 | // Streaming mode is active if the header has the SVE active flag set. |
1456 | if (!(error.Success() && ((m_sve_header.flags & sve::ptrace_regs_mask) == |
1457 | sve::ptrace_regs_sve))) { |
1458 | // Non-streaming might be active instead. |
1459 | m_sve_header_is_valid = false; |
1460 | m_sve_buffer_is_valid = false; |
1461 | m_sve_state = SVEState::Full; |
1462 | error = ReadSVEHeader(); |
1463 | if (error.Success()) { |
1464 | // If SVE is enabled thread can switch between SVEState::FPSIMD and |
1465 | // SVEState::Full on every stop. |
1466 | if ((m_sve_header.flags & sve::ptrace_regs_mask) == |
1467 | sve::ptrace_regs_fpsimd) |
1468 | m_sve_state = SVEState::FPSIMD; |
1469 | // Else we are in SVEState::Full. |
1470 | } else { |
1471 | m_sve_state = SVEState::Disabled; |
1472 | } |
1473 | } |
1474 | |
1475 | if (m_sve_state == SVEState::Full || m_sve_state == SVEState::FPSIMD || |
1476 | m_sve_state == SVEState::Streaming) { |
1477 | // On every stop we configure SVE vector length by calling |
1478 | // ConfigureVectorLengthSVE regardless of current SVEState of this thread. |
1479 | uint32_t vq = RegisterInfoPOSIX_arm64::eVectorQuadwordAArch64SVE; |
1480 | if (sve::vl_valid(m_sve_header.vl)) |
1481 | vq = sve::vq_from_vl(m_sve_header.vl); |
1482 | |
1483 | GetRegisterInfo().ConfigureVectorLengthSVE(vq); |
1484 | m_sve_ptrace_payload.resize(sve::PTraceSize(vq, sve::ptrace_regs_sve)); |
1485 | } |
1486 | } |
1487 | |
1488 | if (!m_za_header_is_valid) { |
1489 | Status error = ReadZAHeader(); |
1490 | if (error.Success()) { |
1491 | uint32_t vq = RegisterInfoPOSIX_arm64::eVectorQuadwordAArch64SVE; |
1492 | if (sve::vl_valid(m_za_header.vl)) |
1493 | vq = sve::vq_from_vl(m_za_header.vl); |
1494 | |
1495 | GetRegisterInfo().ConfigureVectorLengthZA(vq); |
1496 | m_za_ptrace_payload.resize(m_za_header.size); |
1497 | m_za_buffer_is_valid = false; |
1498 | } |
1499 | } |
1500 | } |
1501 | |
1502 | uint32_t NativeRegisterContextLinux_arm64::CalculateFprOffset( |
1503 | const RegisterInfo *reg_info) const { |
1504 | return reg_info->byte_offset - GetGPRSize(); |
1505 | } |
1506 | |
1507 | uint32_t NativeRegisterContextLinux_arm64::CalculateSVEOffset( |
1508 | const RegisterInfo *reg_info) const { |
1509 | // Start of Z0 data is after GPRs plus 8 bytes of vg register |
1510 | uint32_t sve_reg_offset = LLDB_INVALID_INDEX32; |
1511 | if (m_sve_state == SVEState::FPSIMD) { |
1512 | const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB]; |
1513 | sve_reg_offset = sve::ptrace_fpsimd_offset + |
1514 | (reg - GetRegisterInfo().GetRegNumSVEZ0()) * 16; |
1515 | // Between non-streaming and streaming mode, the layout is identical. |
1516 | } else if (m_sve_state == SVEState::Full || |
1517 | m_sve_state == SVEState::Streaming) { |
1518 | uint32_t sve_z0_offset = GetGPRSize() + 16; |
1519 | sve_reg_offset = |
1520 | sve::SigRegsOffset() + reg_info->byte_offset - sve_z0_offset; |
1521 | } |
1522 | return sve_reg_offset; |
1523 | } |
1524 | |
1525 | Status NativeRegisterContextLinux_arm64::ReadSMESVG() { |
1526 | // This register is the streaming vector length, so we will get it from |
1527 | // NT_ARM_ZA regardless of the current streaming mode. |
1528 | Status error = ReadZAHeader(); |
1529 | if (error.Success()) |
1530 | m_sme_pseudo_regs.svg_reg = m_za_header.vl / 8; |
1531 | |
1532 | return error; |
1533 | } |
1534 | |
1535 | std::vector<uint32_t> NativeRegisterContextLinux_arm64::GetExpeditedRegisters( |
1536 | ExpeditedRegs expType) const { |
1537 | std::vector<uint32_t> expedited_reg_nums = |
1538 | NativeRegisterContext::GetExpeditedRegisters(expType); |
1539 | // SVE, non-streaming vector length. |
1540 | if (m_sve_state == SVEState::FPSIMD || m_sve_state == SVEState::Full) |
1541 | expedited_reg_nums.push_back(GetRegisterInfo().GetRegNumSVEVG()); |
1542 | // SME, streaming vector length. This is used by the ZA register which is |
1543 | // present even when streaming mode is not enabled. |
1544 | if (GetRegisterInfo().IsSSVEPresent()) |
1545 | expedited_reg_nums.push_back(GetRegisterInfo().GetRegNumSMESVG()); |
1546 | |
1547 | return expedited_reg_nums; |
1548 | } |
1549 | |
1550 | llvm::Expected<NativeRegisterContextLinux::MemoryTaggingDetails> |
1551 | NativeRegisterContextLinux_arm64::GetMemoryTaggingDetails(int32_t type) { |
1552 | if (type == MemoryTagManagerAArch64MTE::eMTE_allocation) { |
1553 | return MemoryTaggingDetails{std::make_unique<MemoryTagManagerAArch64MTE>(), |
1554 | PTRACE_PEEKMTETAGS, PTRACE_POKEMTETAGS}; |
1555 | } |
1556 | |
1557 | return llvm::createStringError(llvm::inconvertibleErrorCode(), |
1558 | "Unknown AArch64 memory tag type %d" , type); |
1559 | } |
1560 | |
1561 | lldb::addr_t NativeRegisterContextLinux_arm64::FixWatchpointHitAddress( |
1562 | lldb::addr_t hit_addr) { |
1563 | // Linux configures user-space virtual addresses with top byte ignored. |
1564 | // We set default value of mask such that top byte is masked out. |
1565 | lldb::addr_t mask = ~((1ULL << 56) - 1); |
1566 | |
1567 | // Try to read pointer authentication data_mask register and calculate a |
1568 | // consolidated data address mask after ignoring the top byte. |
1569 | if (ReadPAuthMask().Success()) |
1570 | mask |= m_pac_mask.data_mask; |
1571 | |
1572 | return hit_addr & ~mask; |
1573 | ; |
1574 | } |
1575 | |
1576 | #endif // defined (__arm64__) || defined (__aarch64__) |
1577 | |