1//===-- NativeRegisterContextLinux_arm64.cpp ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#if defined(__arm64__) || defined(__aarch64__)
10
11#include "NativeRegisterContextLinux_arm.h"
12#include "NativeRegisterContextLinux_arm64.h"
13
14#include "lldb/Host/HostInfo.h"
15#include "lldb/Host/common/NativeProcessProtocol.h"
16#include "lldb/Host/linux/Ptrace.h"
17#include "lldb/Utility/DataBufferHeap.h"
18#include "lldb/Utility/Log.h"
19#include "lldb/Utility/RegisterValue.h"
20#include "lldb/Utility/Status.h"
21
22#include "Plugins/Process/Linux/NativeProcessLinux.h"
23#include "Plugins/Process/Linux/Procfs.h"
24#include "Plugins/Process/POSIX/ProcessPOSIXLog.h"
25#include "Plugins/Process/Utility/MemoryTagManagerAArch64MTE.h"
26#include "Plugins/Process/Utility/RegisterFlagsDetector_arm64.h"
27#include "Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h"
28
29// System includes - They have to be included after framework includes because
30// they define some macros which collide with variable names in other modules
31#include <sys/uio.h>
32// NT_PRSTATUS and NT_FPREGSET definition
33#include <elf.h>
34#include <mutex>
35#include <optional>
36
37#ifndef NT_ARM_SVE
38#define NT_ARM_SVE 0x405 /* ARM Scalable Vector Extension */
39#endif
40
41#ifndef NT_ARM_SSVE
42#define NT_ARM_SSVE \
43 0x40b /* ARM Scalable Matrix Extension, Streaming SVE mode */
44#endif
45
46#ifndef NT_ARM_ZA
47#define NT_ARM_ZA 0x40c /* ARM Scalable Matrix Extension, Array Storage */
48#endif
49
50#ifndef NT_ARM_ZT
51#define NT_ARM_ZT \
52 0x40d /* ARM Scalable Matrix Extension 2, lookup table register */
53#endif
54
55#ifndef NT_ARM_PAC_MASK
56#define NT_ARM_PAC_MASK 0x406 /* Pointer authentication code masks */
57#endif
58
59#ifndef NT_ARM_TAGGED_ADDR_CTRL
60#define NT_ARM_TAGGED_ADDR_CTRL 0x409 /* Tagged address control register */
61#endif
62
63#ifndef NT_ARM_FPMR
64#define NT_ARM_FPMR 0x40e /* Floating point mode register */
65#endif
66
67#ifndef NT_ARM_GCS
68#define NT_ARM_GCS 0x410 /* Guarded Control Stack control registers */
69#endif
70
71#define HWCAP_PACA (1 << 30)
72
73#define HWCAP_GCS (1UL << 32)
74
75#define HWCAP2_MTE (1 << 18)
76
77#define HWCAP2_FPMR (1UL << 48)
78
79using namespace lldb;
80using namespace lldb_private;
81using namespace lldb_private::process_linux;
82
83// A NativeRegisterContext is constructed per thread, but all threads' registers
84// will contain the same fields. Therefore this mutex prevents each instance
85// competing with the other, and subsequent instances from having to detect the
86// fields all over again.
87static std::mutex g_register_flags_detector_mutex;
88static Arm64RegisterFlagsDetector g_register_flags_detector;
89
90std::unique_ptr<NativeRegisterContextLinux>
91NativeRegisterContextLinux::CreateHostNativeRegisterContextLinux(
92 const ArchSpec &target_arch, NativeThreadLinux &native_thread) {
93 switch (target_arch.GetMachine()) {
94 case llvm::Triple::arm:
95 return std::make_unique<NativeRegisterContextLinux_arm>(target_arch,
96 native_thread);
97 case llvm::Triple::aarch64: {
98 // Configure register sets supported by this AArch64 target.
99 // Read SVE header to check for SVE support.
100 struct sve::user_sve_header sve_header;
101 struct iovec ioVec;
102 ioVec.iov_base = &sve_header;
103 ioVec.iov_len = sizeof(sve_header);
104 unsigned int regset = NT_ARM_SVE;
105
106 Flags opt_regsets;
107 if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
108 native_thread.GetID(), &regset,
109 &ioVec, sizeof(sve_header))
110 .Success()) {
111 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskSVE);
112
113 // We may also have the Scalable Matrix Extension (SME) which adds a
114 // streaming SVE mode.
115 ioVec.iov_len = sizeof(sve_header);
116 regset = NT_ARM_SSVE;
117 if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
118 native_thread.GetID(), &regset,
119 &ioVec, sizeof(sve_header))
120 .Success())
121 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskSSVE);
122 }
123
124 sve::user_za_header za_header;
125 ioVec.iov_base = &za_header;
126 ioVec.iov_len = sizeof(za_header);
127 regset = NT_ARM_ZA;
128 if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
129 native_thread.GetID(), &regset,
130 &ioVec, sizeof(za_header))
131 .Success())
132 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskZA);
133
134 // SME's ZT0 is a 512 bit register.
135 std::array<uint8_t, 64> zt_reg;
136 ioVec.iov_base = zt_reg.data();
137 ioVec.iov_len = zt_reg.size();
138 regset = NT_ARM_ZT;
139 if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
140 native_thread.GetID(), &regset,
141 &ioVec, zt_reg.size())
142 .Success())
143 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskZT);
144
145 NativeProcessLinux &process = native_thread.GetProcess();
146
147 std::optional<uint64_t> auxv_at_hwcap =
148 process.GetAuxValue(AuxVector::AUXV_AT_HWCAP);
149 if (auxv_at_hwcap && (*auxv_at_hwcap & HWCAP_PACA))
150 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskPAuth);
151
152 std::optional<uint64_t> auxv_at_hwcap2 =
153 process.GetAuxValue(AuxVector::AUXV_AT_HWCAP2);
154 if (auxv_at_hwcap2) {
155 if (*auxv_at_hwcap2 & HWCAP2_MTE)
156 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskMTE);
157 if (*auxv_at_hwcap2 & HWCAP2_FPMR)
158 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskFPMR);
159 if (*auxv_at_hwcap & HWCAP_GCS)
160 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskGCS);
161 }
162
163 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskTLS);
164
165 std::optional<uint64_t> auxv_at_hwcap3 =
166 process.GetAuxValue(AuxVector::AUXV_AT_HWCAP3);
167 std::lock_guard<std::mutex> lock(g_register_flags_detector_mutex);
168 if (!g_register_flags_detector.HasDetected())
169 g_register_flags_detector.DetectFields(auxv_at_hwcap.value_or(0),
170 auxv_at_hwcap2.value_or(0),
171 auxv_at_hwcap3.value_or(0));
172
173 auto register_info_up =
174 std::make_unique<RegisterInfoPOSIX_arm64>(target_arch, opt_regsets);
175 return std::make_unique<NativeRegisterContextLinux_arm64>(
176 target_arch, native_thread, std::move(register_info_up));
177 }
178 default:
179 llvm_unreachable("have no register context for architecture");
180 }
181}
182
183llvm::Expected<ArchSpec>
184NativeRegisterContextLinux::DetermineArchitecture(lldb::tid_t tid) {
185 return DetermineArchitectureViaGPR(
186 tid, RegisterInfoPOSIX_arm64::GetGPRSizeStatic());
187}
188
189NativeRegisterContextLinux_arm64::NativeRegisterContextLinux_arm64(
190 const ArchSpec &target_arch, NativeThreadProtocol &native_thread,
191 std::unique_ptr<RegisterInfoPOSIX_arm64> register_info_up)
192 : NativeRegisterContextRegisterInfo(native_thread,
193 register_info_up.release()),
194 NativeRegisterContextLinux(native_thread) {
195 g_register_flags_detector.UpdateRegisterInfo(
196 GetRegisterInfoInterface().GetRegisterInfo(),
197 GetRegisterInfoInterface().GetRegisterCount());
198
199 ::memset(&m_fpr, 0, sizeof(m_fpr));
200 ::memset(&m_gpr_arm64, 0, sizeof(m_gpr_arm64));
201 ::memset(&m_hwp_regs, 0, sizeof(m_hwp_regs));
202 ::memset(&m_hbp_regs, 0, sizeof(m_hbp_regs));
203 ::memset(&m_sve_header, 0, sizeof(m_sve_header));
204 ::memset(&m_pac_mask, 0, sizeof(m_pac_mask));
205 ::memset(&m_tls_regs, 0, sizeof(m_tls_regs));
206 ::memset(&m_sme_pseudo_regs, 0, sizeof(m_sme_pseudo_regs));
207 ::memset(&m_gcs_regs, 0, sizeof(m_gcs_regs));
208 std::fill(m_zt_reg.begin(), m_zt_reg.end(), 0);
209
210 m_mte_ctrl_reg = 0;
211 m_fpmr_reg = 0;
212
213 // 16 is just a maximum value, query hardware for actual watchpoint count
214 m_max_hwp_supported = 16;
215 m_max_hbp_supported = 16;
216
217 m_refresh_hwdebug_info = true;
218
219 m_gpr_is_valid = false;
220 m_fpu_is_valid = false;
221 m_sve_buffer_is_valid = false;
222 m_sve_header_is_valid = false;
223 m_pac_mask_is_valid = false;
224 m_mte_ctrl_is_valid = false;
225 m_tls_is_valid = false;
226 m_zt_buffer_is_valid = false;
227 m_fpmr_is_valid = false;
228 m_gcs_is_valid = false;
229
230 // SME adds the tpidr2 register
231 m_tls_size = GetRegisterInfo().IsSSVEPresent() ? sizeof(m_tls_regs)
232 : sizeof(m_tls_regs.tpidr_reg);
233
234 if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent())
235 m_sve_state = SVEState::Unknown;
236 else
237 m_sve_state = SVEState::Disabled;
238}
239
240RegisterInfoPOSIX_arm64 &
241NativeRegisterContextLinux_arm64::GetRegisterInfo() const {
242 return static_cast<RegisterInfoPOSIX_arm64 &>(*m_register_info_interface_up);
243}
244
245uint32_t NativeRegisterContextLinux_arm64::GetRegisterSetCount() const {
246 return GetRegisterInfo().GetRegisterSetCount();
247}
248
249const RegisterSet *
250NativeRegisterContextLinux_arm64::GetRegisterSet(uint32_t set_index) const {
251 return GetRegisterInfo().GetRegisterSet(set_index);
252}
253
254uint32_t NativeRegisterContextLinux_arm64::GetUserRegisterCount() const {
255 uint32_t count = 0;
256 for (uint32_t set_index = 0; set_index < GetRegisterSetCount(); ++set_index)
257 count += GetRegisterSet(set_index)->num_registers;
258 return count;
259}
260
261Status
262NativeRegisterContextLinux_arm64::ReadRegister(const RegisterInfo *reg_info,
263 RegisterValue &reg_value) {
264 Status error;
265
266 if (!reg_info) {
267 error = Status::FromErrorString("reg_info NULL");
268 return error;
269 }
270
271 const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];
272
273 if (reg == LLDB_INVALID_REGNUM)
274 return Status::FromErrorStringWithFormat(
275 "no lldb regnum for %s",
276 reg_info && reg_info->name ? reg_info->name : "<unknown register>");
277
278 uint8_t *src;
279 uint32_t offset = LLDB_INVALID_INDEX32;
280 uint64_t sve_vg;
281 std::vector<uint8_t> sve_reg_non_live;
282
283 if (IsGPR(reg)) {
284 error = ReadGPR();
285 if (error.Fail())
286 return error;
287
288 offset = reg_info->byte_offset;
289 assert(offset < GetGPRSize());
290 src = (uint8_t *)GetGPRBuffer() + offset;
291
292 } else if (IsFPR(reg)) {
293 if (m_sve_state == SVEState::Disabled) {
294 // SVE is disabled take legacy route for FPU register access
295 error = ReadFPR();
296 if (error.Fail())
297 return error;
298
299 offset = CalculateFprOffset(reg_info);
300 assert(offset < GetFPRSize());
301 src = (uint8_t *)GetFPRBuffer() + offset;
302 } else {
303 // SVE or SSVE enabled, we will read and cache SVE ptrace data.
304 // In SIMD or Full mode, the data comes from the SVE regset. In streaming
305 // mode it comes from the streaming SVE regset.
306 error = ReadAllSVE();
307 if (error.Fail())
308 return error;
309
310 // FPSR and FPCR will be located right after Z registers in
311 // SVEState::FPSIMD while in SVEState::Full or SVEState::Streaming they
312 // will be located at the end of register data after an alignment
313 // correction based on currently selected vector length.
314 uint32_t sve_reg_num = LLDB_INVALID_REGNUM;
315 if (reg == GetRegisterInfo().GetRegNumFPSR()) {
316 sve_reg_num = reg;
317 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
318 offset = sve::PTraceFPSROffset(sve::vq_from_vl(m_sve_header.vl));
319 else if (m_sve_state == SVEState::FPSIMD)
320 offset = sve::ptrace_fpsimd_offset + (32 * 16);
321 } else if (reg == GetRegisterInfo().GetRegNumFPCR()) {
322 sve_reg_num = reg;
323 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
324 offset = sve::PTraceFPCROffset(sve::vq_from_vl(m_sve_header.vl));
325 else if (m_sve_state == SVEState::FPSIMD)
326 offset = sve::ptrace_fpsimd_offset + (32 * 16) + 4;
327 } else {
328 // Extract SVE Z register value register number for this reg_info
329 if (reg_info->value_regs &&
330 reg_info->value_regs[0] != LLDB_INVALID_REGNUM)
331 sve_reg_num = reg_info->value_regs[0];
332 offset = CalculateSVEOffset(GetRegisterInfoAtIndex(sve_reg_num));
333 }
334
335 assert(offset < GetSVEBufferSize());
336 src = (uint8_t *)GetSVEBuffer() + offset;
337 }
338 } else if (IsTLS(reg)) {
339 error = ReadTLS();
340 if (error.Fail())
341 return error;
342
343 offset = reg_info->byte_offset - GetRegisterInfo().GetTLSOffset();
344 assert(offset < GetTLSBufferSize());
345 src = (uint8_t *)GetTLSBuffer() + offset;
346 } else if (IsSVE(reg)) {
347 if (m_sve_state == SVEState::Disabled || m_sve_state == SVEState::Unknown)
348 return Status::FromErrorString("SVE disabled or not supported");
349
350 if (GetRegisterInfo().IsSVERegVG(reg)) {
351 sve_vg = GetSVERegVG();
352 src = (uint8_t *)&sve_vg;
353 } else {
354 // SVE enabled, we will read and cache SVE ptrace data
355 error = ReadAllSVE();
356 if (error.Fail())
357 return error;
358
359 if (m_sve_state == SVEState::FPSIMD) {
360 // In FPSIMD state SVE payload mirrors legacy fpsimd struct and so
361 // just copy 16 bytes of v register to the start of z register. All
362 // other SVE register will be set to zero.
363 sve_reg_non_live.resize(reg_info->byte_size, 0);
364 src = sve_reg_non_live.data();
365
366 if (GetRegisterInfo().IsSVEZReg(reg)) {
367 offset = CalculateSVEOffset(reg_info);
368 assert(offset < GetSVEBufferSize());
369 ::memcpy(sve_reg_non_live.data(), (uint8_t *)GetSVEBuffer() + offset,
370 16);
371 }
372 } else {
373 offset = CalculateSVEOffset(reg_info);
374 assert(offset < GetSVEBufferSize());
375 src = (uint8_t *)GetSVEBuffer() + offset;
376 }
377 }
378 } else if (IsPAuth(reg)) {
379 error = ReadPAuthMask();
380 if (error.Fail())
381 return error;
382
383 offset = reg_info->byte_offset - GetRegisterInfo().GetPAuthOffset();
384 assert(offset < GetPACMaskSize());
385 src = (uint8_t *)GetPACMask() + offset;
386 } else if (IsMTE(reg)) {
387 error = ReadMTEControl();
388 if (error.Fail())
389 return error;
390
391 offset = reg_info->byte_offset - GetRegisterInfo().GetMTEOffset();
392 assert(offset < GetMTEControlSize());
393 src = (uint8_t *)GetMTEControl() + offset;
394 } else if (IsSME(reg)) {
395 if (GetRegisterInfo().IsSMERegZA(reg)) {
396 error = ReadZAHeader();
397 if (error.Fail())
398 return error;
399
400 // If there is only a header and no registers, ZA is inactive. Read as 0
401 // in this case.
402 if (m_za_header.size == sizeof(m_za_header)) {
403 // This will get reconfigured/reset later, so we are safe to use it.
404 // ZA is a square of VL * VL and the ptrace buffer also includes the
405 // header itself.
406 m_za_ptrace_payload.resize(((m_za_header.vl) * (m_za_header.vl)) +
407 GetZAHeaderSize());
408 std::fill(m_za_ptrace_payload.begin(), m_za_ptrace_payload.end(), 0);
409 } else {
410 // ZA is active, read the real register.
411 error = ReadZA();
412 if (error.Fail())
413 return error;
414 }
415
416 // ZA is part of the SME set but uses a separate member buffer for
417 // storage. Therefore its effective byte offset is always 0 even if it
418 // isn't 0 within the SME register set.
419 src = (uint8_t *)GetZABuffer() + GetZAHeaderSize();
420 } else if (GetRegisterInfo().IsSMERegZT(reg)) {
421 // Unlike ZA, the kernel will return register data for ZT0 when ZA is not
422 // enabled. This data will be all 0s so we don't have to invent anything
423 // like we did for ZA.
424 error = ReadZT();
425 if (error.Fail())
426 return error;
427
428 src = (uint8_t *)GetZTBuffer();
429 } else {
430 error = ReadSMESVG();
431 if (error.Fail())
432 return error;
433
434 // This is a psuedo so it never fails.
435 ReadSMEControl();
436
437 offset = reg_info->byte_offset - GetRegisterInfo().GetSMEOffset();
438 assert(offset < GetSMEPseudoBufferSize());
439 src = (uint8_t *)GetSMEPseudoBuffer() + offset;
440 }
441 } else if (IsFPMR(reg)) {
442 error = ReadFPMR();
443 if (error.Fail())
444 return error;
445
446 offset = reg_info->byte_offset - GetRegisterInfo().GetFPMROffset();
447 assert(offset < GetFPMRBufferSize());
448 src = (uint8_t *)GetFPMRBuffer() + offset;
449 } else if (IsGCS(reg)) {
450 error = ReadGCS();
451 if (error.Fail())
452 return error;
453
454 offset = reg_info->byte_offset - GetRegisterInfo().GetGCSOffset();
455 assert(offset < GetGCSBufferSize());
456 src = (uint8_t *)GetGCSBuffer() + offset;
457 } else
458 return Status::FromErrorString(
459 "failed - register wasn't recognized to be a GPR or an FPR, "
460 "write strategy unknown");
461
462 reg_value.SetFromMemoryData(*reg_info, src, reg_info->byte_size,
463 eByteOrderLittle, error);
464
465 return error;
466}
467
468Status NativeRegisterContextLinux_arm64::WriteRegister(
469 const RegisterInfo *reg_info, const RegisterValue &reg_value) {
470 Status error;
471
472 if (!reg_info)
473 return Status::FromErrorString("reg_info NULL");
474
475 const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];
476
477 if (reg == LLDB_INVALID_REGNUM)
478 return Status::FromErrorStringWithFormat(
479 "no lldb regnum for %s",
480 reg_info && reg_info->name ? reg_info->name : "<unknown register>");
481
482 uint8_t *dst;
483 uint32_t offset = LLDB_INVALID_INDEX32;
484 std::vector<uint8_t> sve_reg_non_live;
485
486 if (IsGPR(reg)) {
487 error = ReadGPR();
488 if (error.Fail())
489 return error;
490
491 assert(reg_info->byte_offset < GetGPRSize());
492 dst = (uint8_t *)GetGPRBuffer() + reg_info->byte_offset;
493 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
494
495 return WriteGPR();
496 } else if (IsFPR(reg)) {
497 if (m_sve_state == SVEState::Disabled) {
498 // SVE is disabled take legacy route for FPU register access
499 error = ReadFPR();
500 if (error.Fail())
501 return error;
502
503 offset = CalculateFprOffset(reg_info);
504 assert(offset < GetFPRSize());
505 dst = (uint8_t *)GetFPRBuffer() + offset;
506 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
507
508 return WriteFPR();
509 } else {
510 // SVE enabled, we will read and cache SVE ptrace data.
511 error = ReadAllSVE();
512 if (error.Fail())
513 return error;
514
515 // FPSR and FPCR will be located right after Z registers in
516 // SVEState::FPSIMD while in SVEState::Full or SVEState::Streaming they
517 // will be located at the end of register data after an alignment
518 // correction based on currently selected vector length.
519 uint32_t sve_reg_num = LLDB_INVALID_REGNUM;
520 if (reg == GetRegisterInfo().GetRegNumFPSR()) {
521 sve_reg_num = reg;
522 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
523 offset = sve::PTraceFPSROffset(sve::vq_from_vl(m_sve_header.vl));
524 else if (m_sve_state == SVEState::FPSIMD)
525 offset = sve::ptrace_fpsimd_offset + (32 * 16);
526 } else if (reg == GetRegisterInfo().GetRegNumFPCR()) {
527 sve_reg_num = reg;
528 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
529 offset = sve::PTraceFPCROffset(sve::vq_from_vl(m_sve_header.vl));
530 else if (m_sve_state == SVEState::FPSIMD)
531 offset = sve::ptrace_fpsimd_offset + (32 * 16) + 4;
532 } else {
533 // Extract SVE Z register value register number for this reg_info
534 if (reg_info->value_regs &&
535 reg_info->value_regs[0] != LLDB_INVALID_REGNUM)
536 sve_reg_num = reg_info->value_regs[0];
537 offset = CalculateSVEOffset(GetRegisterInfoAtIndex(sve_reg_num));
538 }
539
540 assert(offset < GetSVEBufferSize());
541 dst = (uint8_t *)GetSVEBuffer() + offset;
542 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
543 return WriteAllSVE();
544 }
545 } else if (IsSVE(reg)) {
546 if (m_sve_state == SVEState::Disabled || m_sve_state == SVEState::Unknown)
547 return Status::FromErrorString("SVE disabled or not supported");
548 else {
549 // Target has SVE enabled, we will read and cache SVE ptrace data
550 error = ReadAllSVE();
551 if (error.Fail())
552 return error;
553
554 if (GetRegisterInfo().IsSVERegVG(reg)) {
555 uint64_t vg_value = reg_value.GetAsUInt64();
556
557 if (sve::vl_valid(vg_value * 8)) {
558 if (m_sve_header_is_valid && vg_value == GetSVERegVG())
559 return error;
560
561 SetSVERegVG(vg_value);
562
563 error = WriteSVEHeader();
564 if (error.Success()) {
565 // Changing VG during streaming mode also changes the size of ZA.
566 if (m_sve_state == SVEState::Streaming)
567 m_za_header_is_valid = false;
568 ConfigureRegisterContext();
569 }
570
571 if (m_sve_header_is_valid && vg_value == GetSVERegVG())
572 return error;
573 }
574
575 return Status::FromErrorString("SVE vector length update failed.");
576 }
577
578 // If target supports SVE but currently in FPSIMD mode.
579 if (m_sve_state == SVEState::FPSIMD) {
580 // Here we will check if writing this SVE register enables
581 // SVEState::Full
582 bool set_sve_state_full = false;
583 const uint8_t *reg_bytes = (const uint8_t *)reg_value.GetBytes();
584 if (GetRegisterInfo().IsSVEZReg(reg)) {
585 for (uint32_t i = 16; i < reg_info->byte_size; i++) {
586 if (reg_bytes[i]) {
587 set_sve_state_full = true;
588 break;
589 }
590 }
591 } else if (GetRegisterInfo().IsSVEPReg(reg) ||
592 reg == GetRegisterInfo().GetRegNumSVEFFR()) {
593 for (uint32_t i = 0; i < reg_info->byte_size; i++) {
594 if (reg_bytes[i]) {
595 set_sve_state_full = true;
596 break;
597 }
598 }
599 }
600
601 if (!set_sve_state_full && GetRegisterInfo().IsSVEZReg(reg)) {
602 // We are writing a Z register which is zero beyond 16 bytes so copy
603 // first 16 bytes only as SVE payload mirrors legacy fpsimd structure
604 offset = CalculateSVEOffset(reg_info);
605 assert(offset < GetSVEBufferSize());
606 dst = (uint8_t *)GetSVEBuffer() + offset;
607 ::memcpy(dst, reg_value.GetBytes(), 16);
608
609 return WriteAllSVE();
610 } else
611 return Status::FromErrorString(
612 "SVE state change operation not supported");
613 } else {
614 offset = CalculateSVEOffset(reg_info);
615 assert(offset < GetSVEBufferSize());
616 dst = (uint8_t *)GetSVEBuffer() + offset;
617 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
618 return WriteAllSVE();
619 }
620 }
621 } else if (IsMTE(reg)) {
622 error = ReadMTEControl();
623 if (error.Fail())
624 return error;
625
626 offset = reg_info->byte_offset - GetRegisterInfo().GetMTEOffset();
627 assert(offset < GetMTEControlSize());
628 dst = (uint8_t *)GetMTEControl() + offset;
629 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
630
631 return WriteMTEControl();
632 } else if (IsTLS(reg)) {
633 error = ReadTLS();
634 if (error.Fail())
635 return error;
636
637 offset = reg_info->byte_offset - GetRegisterInfo().GetTLSOffset();
638 assert(offset < GetTLSBufferSize());
639 dst = (uint8_t *)GetTLSBuffer() + offset;
640 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
641
642 return WriteTLS();
643 } else if (IsSME(reg)) {
644 if (GetRegisterInfo().IsSMERegZA(reg)) {
645 error = ReadZA();
646 if (error.Fail())
647 return error;
648
649 // ZA is part of the SME set but not stored with the other SME registers.
650 // So its byte offset is effectively always 0.
651 dst = (uint8_t *)GetZABuffer() + GetZAHeaderSize();
652 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
653
654 // While this is writing a header that contains a vector length, the only
655 // way to change that is via the vg register. So here we assume the length
656 // will always be the current length and no reconfigure is needed.
657 return WriteZA();
658 } else if (GetRegisterInfo().IsSMERegZT(reg)) {
659 error = ReadZT();
660 if (error.Fail())
661 return error;
662
663 dst = (uint8_t *)GetZTBuffer();
664 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
665
666 return WriteZT();
667 } else
668 return Status::FromErrorString(
669 "Writing to SVG or SVCR is not supported.");
670 } else if (IsFPMR(reg)) {
671 error = ReadFPMR();
672 if (error.Fail())
673 return error;
674
675 offset = reg_info->byte_offset - GetRegisterInfo().GetFPMROffset();
676 assert(offset < GetFPMRBufferSize());
677 dst = (uint8_t *)GetFPMRBuffer() + offset;
678 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
679
680 return WriteFPMR();
681 } else if (IsGCS(reg)) {
682 error = ReadGCS();
683 if (error.Fail())
684 return error;
685
686 offset = reg_info->byte_offset - GetRegisterInfo().GetGCSOffset();
687 assert(offset < GetGCSBufferSize());
688 dst = (uint8_t *)GetGCSBuffer() + offset;
689 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
690
691 return WriteGCS();
692 }
693
694 return Status::FromErrorString("Failed to write register value");
695}
696
697enum RegisterSetType : uint32_t {
698 GPR,
699 SVE, // Used for SVE and SSVE.
700 FPR, // When there is no SVE, or SVE in FPSIMD mode.
701 // Pointer authentication registers are read only, so not included here.
702 MTE,
703 TLS,
704 SME, // ZA only, because SVCR and SVG are pseudo registers.
705 SME2, // ZT only.
706 FPMR,
707 GCS, // Guarded Control Stack registers.
708};
709
710static uint8_t *AddRegisterSetType(uint8_t *dst,
711 RegisterSetType register_set_type) {
712 *(reinterpret_cast<uint32_t *>(dst)) = register_set_type;
713 return dst + sizeof(uint32_t);
714}
715
716static uint8_t *AddSavedRegistersData(uint8_t *dst, void *src, size_t size) {
717 ::memcpy(dst, src, size);
718 return dst + size;
719}
720
721static uint8_t *AddSavedRegisters(uint8_t *dst,
722 enum RegisterSetType register_set_type,
723 void *src, size_t size) {
724 dst = AddRegisterSetType(dst, register_set_type);
725 return AddSavedRegistersData(dst, src, size);
726}
727
728Status
729NativeRegisterContextLinux_arm64::CacheAllRegisters(uint32_t &cached_size) {
730 Status error;
731 cached_size = sizeof(RegisterSetType) + GetGPRBufferSize();
732 error = ReadGPR();
733 if (error.Fail())
734 return error;
735
736 if (GetRegisterInfo().IsZAPresent()) {
737 error = ReadZAHeader();
738 if (error.Fail())
739 return error;
740 // Use header size here because the buffer may contain fake data when ZA is
741 // disabled. We do not want to write this fake data (all 0s) because this
742 // would tell the kernel that we want ZA to become active. Which is the
743 // opposite of what we want in the case where it is currently inactive.
744 cached_size += sizeof(RegisterSetType) + m_za_header.size;
745 // For the same reason, we need to force it to be re-read so that it will
746 // always contain the real header.
747 m_za_buffer_is_valid = false;
748 error = ReadZA();
749 if (error.Fail())
750 return error;
751
752 // We will only be restoring ZT data if ZA is active. As writing to an
753 // inactive ZT enables ZA, which may not be desireable.
754 if (
755 // If we have ZT0, or in other words, if we have SME2.
756 GetRegisterInfo().IsZTPresent() &&
757 // And ZA is active, which means that ZT0 is also active.
758 m_za_header.size > sizeof(m_za_header)) {
759 cached_size += sizeof(RegisterSetType) + GetZTBufferSize();
760 // The kernel handles an inactive ZT0 for us, and it will read as 0s if
761 // inactive (unlike ZA where we fake that behaviour).
762 error = ReadZT();
763 if (error.Fail())
764 return error;
765 }
766 }
767
768 // If SVE is enabled we need not copy FPR separately.
769 if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent()) {
770 // Store mode and register data.
771 cached_size +=
772 sizeof(RegisterSetType) + sizeof(m_sve_state) + GetSVEBufferSize();
773 error = ReadAllSVE();
774 } else {
775 cached_size += sizeof(RegisterSetType) + GetFPRSize();
776 error = ReadFPR();
777 }
778 if (error.Fail())
779 return error;
780
781 if (GetRegisterInfo().IsMTEPresent()) {
782 cached_size += sizeof(RegisterSetType) + GetMTEControlSize();
783 error = ReadMTEControl();
784 if (error.Fail())
785 return error;
786 }
787
788 if (GetRegisterInfo().IsFPMRPresent()) {
789 cached_size += sizeof(RegisterSetType) + GetFPMRBufferSize();
790 error = ReadFPMR();
791 if (error.Fail())
792 return error;
793 }
794
795 if (GetRegisterInfo().IsGCSPresent()) {
796 cached_size += sizeof(RegisterSetType) + GetGCSBufferSize();
797 error = ReadGCS();
798 if (error.Fail())
799 return error;
800 }
801
802 // tpidr is always present but tpidr2 depends on SME.
803 cached_size += sizeof(RegisterSetType) + GetTLSBufferSize();
804 error = ReadTLS();
805
806 return error;
807}
808
809Status NativeRegisterContextLinux_arm64::ReadAllRegisterValues(
810 lldb::WritableDataBufferSP &data_sp) {
811 // AArch64 register data must contain GPRs and either FPR or SVE registers.
812 // SVE registers can be non-streaming (aka SVE) or streaming (aka SSVE).
813 // Finally an optional MTE register. Pointer Authentication (PAC) registers
814 // are read-only and will be skipped.
815
816 // In order to create register data checkpoint we first read all register
817 // values if not done already and calculate total size of register set data.
818 // We store all register values in data_sp by copying full PTrace data that
819 // corresponds to register sets enabled by current register context.
820
821 uint32_t reg_data_byte_size = 0;
822 Status error = CacheAllRegisters(reg_data_byte_size);
823 if (error.Fail())
824 return error;
825
826 data_sp.reset(new DataBufferHeap(reg_data_byte_size, 0));
827 uint8_t *dst = data_sp->GetBytes();
828
829 dst = AddSavedRegisters(dst, RegisterSetType::GPR, GetGPRBuffer(),
830 GetGPRBufferSize());
831
832 // Streaming SVE and the ZA register both use the streaming vector length.
833 // When you change this, the kernel will invalidate parts of the process
834 // state. Therefore we need a specific order of restoration for each mode, if
835 // we also have ZA to restore.
836 //
837 // Streaming mode enabled, ZA enabled:
838 // * Write streaming registers. This sets SVCR.SM and clears SVCR.ZA.
839 // * Write ZA, this set SVCR.ZA. The register data we provide is written to
840 // ZA.
841 // * Result is SVCR.SM and SVCR.ZA set, with the expected data in both
842 // register sets.
843 //
844 // Streaming mode disabled, ZA enabled:
845 // * Write ZA. This sets SVCR.ZA, and the ZA content. In the majority of cases
846 // the streaming vector length is changing, so the thread is converted into
847 // an FPSIMD thread if it is not already one. This also clears SVCR.SM.
848 // * Write SVE registers, which also clears SVCR.SM but most importantly, puts
849 // us into full SVE mode instead of FPSIMD mode (where the registers are
850 // actually the 128 bit Neon registers).
851 // * Result is we have SVCR.SM = 0, SVCR.ZA = 1 and the expected register
852 // state.
853 //
854 // Restoring in different orders leads to things like the SVE registers being
855 // truncated due to the FPSIMD mode and ZA being disabled or filled with 0s
856 // (disabled and 0s looks the same from inside lldb since we fake the value
857 // when it's disabled).
858 //
859 // For more information on this, look up the uses of the relevant NT_ARM_
860 // constants and the functions vec_set_vector_length, sve_set_common and
861 // za_set in the Linux Kernel.
862
863 if ((m_sve_state != SVEState::Streaming) && GetRegisterInfo().IsZAPresent()) {
864 // Use the header size not the buffer size, as we may be using the buffer
865 // for fake data, which we do not want to write out.
866 assert(m_za_header.size <= GetZABufferSize());
867 dst = AddSavedRegisters(dst, RegisterSetType::SME, GetZABuffer(),
868 m_za_header.size);
869 }
870
871 if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent()) {
872 dst = AddRegisterSetType(dst, RegisterSetType::SVE);
873 *(reinterpret_cast<SVEState *>(dst)) = m_sve_state;
874 dst += sizeof(m_sve_state);
875 dst = AddSavedRegistersData(dst, GetSVEBuffer(), GetSVEBufferSize());
876 } else {
877 dst = AddSavedRegisters(dst, RegisterSetType::FPR, GetFPRBuffer(),
878 GetFPRSize());
879 }
880
881 if ((m_sve_state == SVEState::Streaming) && GetRegisterInfo().IsZAPresent()) {
882 assert(m_za_header.size <= GetZABufferSize());
883 dst = AddSavedRegisters(dst, RegisterSetType::SME, GetZABuffer(),
884 m_za_header.size);
885 }
886
887 // If ZT0 is present and we are going to be restoring an active ZA (which
888 // implies an active ZT0), then restore ZT0 after ZA has been set. This
889 // prevents us enabling ZA accidentally after the restore of ZA disabled it.
890 // If we leave ZA/ZT0 inactive and read ZT0, the kernel returns 0s. Therefore
891 // there's nothing for us to restore if ZA was originally inactive.
892 if (
893 // If we have SME2 and therefore ZT0.
894 GetRegisterInfo().IsZTPresent() &&
895 // And ZA is enabled.
896 m_za_header.size > sizeof(m_za_header))
897 dst = AddSavedRegisters(dst, RegisterSetType::SME2, GetZTBuffer(),
898 GetZTBufferSize());
899
900 if (GetRegisterInfo().IsMTEPresent()) {
901 dst = AddSavedRegisters(dst, RegisterSetType::MTE, GetMTEControl(),
902 GetMTEControlSize());
903 }
904
905 if (GetRegisterInfo().IsFPMRPresent()) {
906 dst = AddSavedRegisters(dst, RegisterSetType::FPMR, GetFPMRBuffer(),
907 GetFPMRBufferSize());
908 }
909
910 if (GetRegisterInfo().IsGCSPresent()) {
911 dst = AddSavedRegisters(dst, RegisterSetType::GCS, GetGCSBuffer(),
912 GetGCSBufferSize());
913 }
914
915 dst = AddSavedRegisters(dst, RegisterSetType::TLS, GetTLSBuffer(),
916 GetTLSBufferSize());
917
918 return error;
919}
920
921static Status RestoreRegisters(void *buffer, const uint8_t **src, size_t len,
922 bool &is_valid, std::function<Status()> writer) {
923 ::memcpy(buffer, *src, len);
924 is_valid = true;
925 *src += len;
926 return writer();
927}
928
929Status NativeRegisterContextLinux_arm64::WriteAllRegisterValues(
930 const lldb::DataBufferSP &data_sp) {
931 // AArch64 register data must contain GPRs, either FPR or SVE registers
932 // (which can be streaming or non-streaming) and optional MTE register.
933 // Pointer Authentication (PAC) registers are read-only and will be skipped.
934
935 // We store all register values in data_sp by copying full PTrace data that
936 // corresponds to register sets enabled by current register context. In order
937 // to restore from register data checkpoint we will first restore GPRs, based
938 // on size of remaining register data either SVE or FPRs should be restored
939 // next. SVE is not enabled if we have register data size less than or equal
940 // to size of GPR + FPR + MTE.
941
942 Status error;
943 if (!data_sp) {
944 error = Status::FromErrorStringWithFormat(
945 "NativeRegisterContextLinux_arm64::%s invalid data_sp provided",
946 __FUNCTION__);
947 return error;
948 }
949
950 const uint8_t *src = data_sp->GetBytes();
951 if (src == nullptr) {
952 error = Status::FromErrorStringWithFormat(
953 "NativeRegisterContextLinux_arm64::%s "
954 "DataBuffer::GetBytes() returned a null "
955 "pointer",
956 __FUNCTION__);
957 return error;
958 }
959
960 uint64_t reg_data_min_size =
961 GetGPRBufferSize() + GetFPRSize() + 2 * (sizeof(RegisterSetType));
962 if (data_sp->GetByteSize() < reg_data_min_size) {
963 error = Status::FromErrorStringWithFormat(
964 "NativeRegisterContextLinux_arm64::%s data_sp contained insufficient "
965 "register data bytes, expected at least %" PRIu64 ", actual %" PRIu64,
966 __FUNCTION__, reg_data_min_size, data_sp->GetByteSize());
967 return error;
968 }
969
970 const uint8_t *end = src + data_sp->GetByteSize();
971 while (src < end) {
972 const RegisterSetType kind =
973 *reinterpret_cast<const RegisterSetType *>(src);
974 src += sizeof(RegisterSetType);
975
976 switch (kind) {
977 case RegisterSetType::GPR:
978 error = RestoreRegisters(
979 GetGPRBuffer(), &src, GetGPRBufferSize(), m_gpr_is_valid,
980 std::bind(&NativeRegisterContextLinux_arm64::WriteGPR, this));
981 break;
982 case RegisterSetType::SVE:
983 // Restore to the correct mode, streaming or not.
984 m_sve_state = static_cast<SVEState>(*src);
985 src += sizeof(m_sve_state);
986
987 // First write SVE header. We do not use RestoreRegisters because we do
988 // not want src to be modified yet.
989 ::memcpy(GetSVEHeader(), src, GetSVEHeaderSize());
990 if (!sve::vl_valid(m_sve_header.vl)) {
991 m_sve_header_is_valid = false;
992 error = Status::FromErrorStringWithFormat(
993 "NativeRegisterContextLinux_arm64::%s "
994 "Invalid SVE header in data_sp",
995 __FUNCTION__);
996 return error;
997 }
998 m_sve_header_is_valid = true;
999 error = WriteSVEHeader();
1000 if (error.Fail())
1001 return error;
1002
1003 // SVE header has been written configure SVE vector length if needed.
1004 // This could change ZA data too, but that will be restored again later
1005 // anyway.
1006 ConfigureRegisterContext();
1007
1008 // Write header and register data, incrementing src this time.
1009 error = RestoreRegisters(
1010 GetSVEBuffer(), &src, GetSVEBufferSize(), m_sve_buffer_is_valid,
1011 std::bind(&NativeRegisterContextLinux_arm64::WriteAllSVE, this));
1012 break;
1013 case RegisterSetType::FPR:
1014 error = RestoreRegisters(
1015 GetFPRBuffer(), &src, GetFPRSize(), m_fpu_is_valid,
1016 std::bind(&NativeRegisterContextLinux_arm64::WriteFPR, this));
1017 break;
1018 case RegisterSetType::MTE:
1019 error = RestoreRegisters(
1020 GetMTEControl(), &src, GetMTEControlSize(), m_mte_ctrl_is_valid,
1021 std::bind(&NativeRegisterContextLinux_arm64::WriteMTEControl, this));
1022 break;
1023 case RegisterSetType::TLS:
1024 error = RestoreRegisters(
1025 GetTLSBuffer(), &src, GetTLSBufferSize(), m_tls_is_valid,
1026 std::bind(&NativeRegisterContextLinux_arm64::WriteTLS, this));
1027 break;
1028 case RegisterSetType::SME:
1029 // To enable or disable ZA you write the regset with or without register
1030 // data. The kernel detects this by looking at the ioVec's length, not the
1031 // ZA header size you pass in. Therefore we must write header and register
1032 // data (if present) in one go every time. Read the header only first just
1033 // to get the size.
1034 ::memcpy(GetZAHeader(), src, GetZAHeaderSize());
1035 // Read the header and register data. Can't use the buffer size here, it
1036 // may be incorrect due to being filled with dummy data previously. Resize
1037 // this so WriteZA uses the correct size.
1038 m_za_ptrace_payload.resize(m_za_header.size);
1039 ::memcpy(GetZABuffer(), src, GetZABufferSize());
1040 m_za_buffer_is_valid = true;
1041
1042 error = WriteZA();
1043 if (error.Fail())
1044 return error;
1045
1046 // Update size of ZA, which resizes the ptrace payload potentially
1047 // trashing our copy of the data we just wrote.
1048 ConfigureRegisterContext();
1049
1050 // ZA buffer now has proper size, read back the data we wrote above, from
1051 // ptrace.
1052 error = ReadZA();
1053 src += GetZABufferSize();
1054 break;
1055 case RegisterSetType::SME2:
1056 // Doing this would activate an inactive ZA, however we will only get here
1057 // if the state we are restoring had an active ZA. Restoring ZT0 will
1058 // always come after restoring ZA.
1059 error = RestoreRegisters(
1060 GetZTBuffer(), &src, GetZTBufferSize(), m_zt_buffer_is_valid,
1061 std::bind(&NativeRegisterContextLinux_arm64::WriteZT, this));
1062 break;
1063 case RegisterSetType::FPMR:
1064 error = RestoreRegisters(
1065 GetFPMRBuffer(), &src, GetFPMRBufferSize(), m_fpmr_is_valid,
1066 std::bind(&NativeRegisterContextLinux_arm64::WriteFPMR, this));
1067 break;
1068 case RegisterSetType::GCS:
1069 // It is not permitted to enable GCS via ptrace. We can disable it, but
1070 // to keep things simple we will not revert any change to the
1071 // PR_SHADOW_STACK_ENABLE bit. Instead patch in the current enable bit
1072 // into the registers we are about to restore.
1073 m_gcs_is_valid = false;
1074 error = ReadGCS();
1075 if (error.Fail())
1076 return error;
1077
1078 uint64_t enable_bit = m_gcs_regs.features_enabled & 1UL;
1079 gcs_regs new_gcs_regs = *reinterpret_cast<const gcs_regs *>(src);
1080 new_gcs_regs.features_enabled =
1081 (new_gcs_regs.features_enabled & ~1UL) | enable_bit;
1082
1083 const uint8_t *new_gcs_src =
1084 reinterpret_cast<const uint8_t *>(&new_gcs_regs);
1085 error = RestoreRegisters(
1086 GetGCSBuffer(), &new_gcs_src, GetGCSBufferSize(), m_gcs_is_valid,
1087 std::bind(&NativeRegisterContextLinux_arm64::WriteGCS, this));
1088 src += GetGCSBufferSize();
1089
1090 break;
1091 }
1092
1093 if (error.Fail())
1094 return error;
1095 }
1096
1097 return error;
1098}
1099
1100bool NativeRegisterContextLinux_arm64::IsGPR(unsigned reg) const {
1101 if (GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg) ==
1102 RegisterInfoPOSIX_arm64::GPRegSet)
1103 return true;
1104 return false;
1105}
1106
1107bool NativeRegisterContextLinux_arm64::IsFPR(unsigned reg) const {
1108 if (GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg) ==
1109 RegisterInfoPOSIX_arm64::FPRegSet)
1110 return true;
1111 return false;
1112}
1113
1114bool NativeRegisterContextLinux_arm64::IsSVE(unsigned reg) const {
1115 return GetRegisterInfo().IsSVEReg(reg);
1116}
1117
1118bool NativeRegisterContextLinux_arm64::IsSME(unsigned reg) const {
1119 return GetRegisterInfo().IsSMEReg(reg);
1120}
1121
1122bool NativeRegisterContextLinux_arm64::IsPAuth(unsigned reg) const {
1123 return GetRegisterInfo().IsPAuthReg(reg);
1124}
1125
1126bool NativeRegisterContextLinux_arm64::IsMTE(unsigned reg) const {
1127 return GetRegisterInfo().IsMTEReg(reg);
1128}
1129
1130bool NativeRegisterContextLinux_arm64::IsTLS(unsigned reg) const {
1131 return GetRegisterInfo().IsTLSReg(reg);
1132}
1133
1134bool NativeRegisterContextLinux_arm64::IsFPMR(unsigned reg) const {
1135 return GetRegisterInfo().IsFPMRReg(reg);
1136}
1137
1138bool NativeRegisterContextLinux_arm64::IsGCS(unsigned reg) const {
1139 return GetRegisterInfo().IsGCSReg(reg);
1140}
1141
1142llvm::Error NativeRegisterContextLinux_arm64::ReadHardwareDebugInfo() {
1143 if (!m_refresh_hwdebug_info) {
1144 return llvm::Error::success();
1145 }
1146
1147 ::pid_t tid = m_thread.GetID();
1148
1149 int regset = NT_ARM_HW_WATCH;
1150 struct iovec ioVec;
1151 struct user_hwdebug_state dreg_state;
1152 Status error;
1153
1154 ioVec.iov_base = &dreg_state;
1155 ioVec.iov_len = sizeof(dreg_state);
1156 error = NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET, tid, &regset,
1157 &ioVec, ioVec.iov_len);
1158
1159 if (error.Fail())
1160 return error.ToError();
1161
1162 m_max_hwp_supported = dreg_state.dbg_info & 0xff;
1163
1164 regset = NT_ARM_HW_BREAK;
1165 error = NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET, tid, &regset,
1166 &ioVec, ioVec.iov_len);
1167
1168 if (error.Fail())
1169 return error.ToError();
1170
1171 m_max_hbp_supported = dreg_state.dbg_info & 0xff;
1172 m_refresh_hwdebug_info = false;
1173
1174 return llvm::Error::success();
1175}
1176
1177llvm::Error
1178NativeRegisterContextLinux_arm64::WriteHardwareDebugRegs(DREGType hwbType) {
1179 struct iovec ioVec;
1180 struct user_hwdebug_state dreg_state;
1181 int regset;
1182
1183 memset(&dreg_state, 0, sizeof(dreg_state));
1184 ioVec.iov_base = &dreg_state;
1185
1186 switch (hwbType) {
1187 case eDREGTypeWATCH:
1188 regset = NT_ARM_HW_WATCH;
1189 ioVec.iov_len = sizeof(dreg_state.dbg_info) + sizeof(dreg_state.pad) +
1190 (sizeof(dreg_state.dbg_regs[0]) * m_max_hwp_supported);
1191
1192 for (uint32_t i = 0; i < m_max_hwp_supported; i++) {
1193 dreg_state.dbg_regs[i].addr = m_hwp_regs[i].address;
1194 dreg_state.dbg_regs[i].ctrl = m_hwp_regs[i].control;
1195 }
1196 break;
1197 case eDREGTypeBREAK:
1198 regset = NT_ARM_HW_BREAK;
1199 ioVec.iov_len = sizeof(dreg_state.dbg_info) + sizeof(dreg_state.pad) +
1200 (sizeof(dreg_state.dbg_regs[0]) * m_max_hbp_supported);
1201
1202 for (uint32_t i = 0; i < m_max_hbp_supported; i++) {
1203 dreg_state.dbg_regs[i].addr = m_hbp_regs[i].address;
1204 dreg_state.dbg_regs[i].ctrl = m_hbp_regs[i].control;
1205 }
1206 break;
1207 }
1208
1209 return NativeProcessLinux::PtraceWrapper(PTRACE_SETREGSET, m_thread.GetID(),
1210 &regset, &ioVec, ioVec.iov_len)
1211 .ToError();
1212}
1213
1214Status NativeRegisterContextLinux_arm64::ReadGPR() {
1215 Status error;
1216
1217 if (m_gpr_is_valid)
1218 return error;
1219
1220 struct iovec ioVec;
1221 ioVec.iov_base = GetGPRBuffer();
1222 ioVec.iov_len = GetGPRBufferSize();
1223
1224 error = ReadRegisterSet(&ioVec, GetGPRBufferSize(), NT_PRSTATUS);
1225
1226 if (error.Success())
1227 m_gpr_is_valid = true;
1228
1229 return error;
1230}
1231
1232Status NativeRegisterContextLinux_arm64::WriteGPR() {
1233 Status error = ReadGPR();
1234 if (error.Fail())
1235 return error;
1236
1237 struct iovec ioVec;
1238 ioVec.iov_base = GetGPRBuffer();
1239 ioVec.iov_len = GetGPRBufferSize();
1240
1241 m_gpr_is_valid = false;
1242
1243 return WriteRegisterSet(&ioVec, GetGPRBufferSize(), NT_PRSTATUS);
1244}
1245
1246Status NativeRegisterContextLinux_arm64::ReadFPR() {
1247 Status error;
1248
1249 if (m_fpu_is_valid)
1250 return error;
1251
1252 struct iovec ioVec;
1253 ioVec.iov_base = GetFPRBuffer();
1254 ioVec.iov_len = GetFPRSize();
1255
1256 error = ReadRegisterSet(&ioVec, GetFPRSize(), NT_FPREGSET);
1257
1258 if (error.Success())
1259 m_fpu_is_valid = true;
1260
1261 return error;
1262}
1263
1264Status NativeRegisterContextLinux_arm64::WriteFPR() {
1265 Status error = ReadFPR();
1266 if (error.Fail())
1267 return error;
1268
1269 struct iovec ioVec;
1270 ioVec.iov_base = GetFPRBuffer();
1271 ioVec.iov_len = GetFPRSize();
1272
1273 m_fpu_is_valid = false;
1274
1275 return WriteRegisterSet(&ioVec, GetFPRSize(), NT_FPREGSET);
1276}
1277
1278void NativeRegisterContextLinux_arm64::InvalidateAllRegisters() {
1279 m_gpr_is_valid = false;
1280 m_fpu_is_valid = false;
1281 m_sve_buffer_is_valid = false;
1282 m_sve_header_is_valid = false;
1283 m_za_buffer_is_valid = false;
1284 m_za_header_is_valid = false;
1285 m_pac_mask_is_valid = false;
1286 m_mte_ctrl_is_valid = false;
1287 m_tls_is_valid = false;
1288 m_zt_buffer_is_valid = false;
1289 m_fpmr_is_valid = false;
1290 m_gcs_is_valid = false;
1291
1292 // Update SVE and ZA registers in case there is change in configuration.
1293 ConfigureRegisterContext();
1294}
1295
1296unsigned NativeRegisterContextLinux_arm64::GetSVERegSet() {
1297 return m_sve_state == SVEState::Streaming ? NT_ARM_SSVE : NT_ARM_SVE;
1298}
1299
1300Status NativeRegisterContextLinux_arm64::ReadSVEHeader() {
1301 Status error;
1302
1303 if (m_sve_header_is_valid)
1304 return error;
1305
1306 struct iovec ioVec;
1307 ioVec.iov_base = GetSVEHeader();
1308 ioVec.iov_len = GetSVEHeaderSize();
1309
1310 error = ReadRegisterSet(&ioVec, GetSVEHeaderSize(), GetSVERegSet());
1311
1312 if (error.Success())
1313 m_sve_header_is_valid = true;
1314
1315 return error;
1316}
1317
1318Status NativeRegisterContextLinux_arm64::ReadPAuthMask() {
1319 Status error;
1320
1321 if (m_pac_mask_is_valid)
1322 return error;
1323
1324 struct iovec ioVec;
1325 ioVec.iov_base = GetPACMask();
1326 ioVec.iov_len = GetPACMaskSize();
1327
1328 error = ReadRegisterSet(&ioVec, GetPACMaskSize(), NT_ARM_PAC_MASK);
1329
1330 if (error.Success())
1331 m_pac_mask_is_valid = true;
1332
1333 return error;
1334}
1335
1336Status NativeRegisterContextLinux_arm64::WriteSVEHeader() {
1337 Status error;
1338
1339 error = ReadSVEHeader();
1340 if (error.Fail())
1341 return error;
1342
1343 struct iovec ioVec;
1344 ioVec.iov_base = GetSVEHeader();
1345 ioVec.iov_len = GetSVEHeaderSize();
1346
1347 m_sve_buffer_is_valid = false;
1348 m_sve_header_is_valid = false;
1349 m_fpu_is_valid = false;
1350
1351 return WriteRegisterSet(&ioVec, GetSVEHeaderSize(), GetSVERegSet());
1352}
1353
1354Status NativeRegisterContextLinux_arm64::ReadAllSVE() {
1355 Status error;
1356 if (m_sve_buffer_is_valid)
1357 return error;
1358
1359 struct iovec ioVec;
1360 ioVec.iov_base = GetSVEBuffer();
1361 ioVec.iov_len = GetSVEBufferSize();
1362
1363 error = ReadRegisterSet(&ioVec, GetSVEBufferSize(), GetSVERegSet());
1364
1365 if (error.Success())
1366 m_sve_buffer_is_valid = true;
1367
1368 return error;
1369}
1370
1371Status NativeRegisterContextLinux_arm64::WriteAllSVE() {
1372 Status error;
1373
1374 error = ReadAllSVE();
1375 if (error.Fail())
1376 return error;
1377
1378 struct iovec ioVec;
1379
1380 ioVec.iov_base = GetSVEBuffer();
1381 ioVec.iov_len = GetSVEBufferSize();
1382
1383 m_sve_buffer_is_valid = false;
1384 m_sve_header_is_valid = false;
1385 m_fpu_is_valid = false;
1386
1387 return WriteRegisterSet(&ioVec, GetSVEBufferSize(), GetSVERegSet());
1388}
1389
1390Status NativeRegisterContextLinux_arm64::ReadSMEControl() {
1391 // The real register is SVCR and is accessible from EL0. However we don't want
1392 // to have to JIT code into the target process so we'll just recreate it using
1393 // what we know from ptrace.
1394
1395 // Bit 0 indicates whether streaming mode is active.
1396 m_sme_pseudo_regs.ctrl_reg = m_sve_state == SVEState::Streaming;
1397
1398 // Bit 1 indicates whether the array storage is active.
1399 // It is active if we can read the header and the size field tells us that
1400 // there is register data following it.
1401 Status error = ReadZAHeader();
1402 if (error.Success() && (m_za_header.size > sizeof(m_za_header)))
1403 m_sme_pseudo_regs.ctrl_reg |= 2;
1404
1405 return error;
1406}
1407
1408Status NativeRegisterContextLinux_arm64::ReadMTEControl() {
1409 Status error;
1410
1411 if (m_mte_ctrl_is_valid)
1412 return error;
1413
1414 struct iovec ioVec;
1415 ioVec.iov_base = GetMTEControl();
1416 ioVec.iov_len = GetMTEControlSize();
1417
1418 error = ReadRegisterSet(&ioVec, GetMTEControlSize(), NT_ARM_TAGGED_ADDR_CTRL);
1419
1420 if (error.Success())
1421 m_mte_ctrl_is_valid = true;
1422
1423 return error;
1424}
1425
1426Status NativeRegisterContextLinux_arm64::WriteMTEControl() {
1427 Status error;
1428
1429 error = ReadMTEControl();
1430 if (error.Fail())
1431 return error;
1432
1433 struct iovec ioVec;
1434 ioVec.iov_base = GetMTEControl();
1435 ioVec.iov_len = GetMTEControlSize();
1436
1437 m_mte_ctrl_is_valid = false;
1438
1439 return WriteRegisterSet(&ioVec, GetMTEControlSize(), NT_ARM_TAGGED_ADDR_CTRL);
1440}
1441
1442Status NativeRegisterContextLinux_arm64::ReadTLS() {
1443 Status error;
1444
1445 if (m_tls_is_valid)
1446 return error;
1447
1448 struct iovec ioVec;
1449 ioVec.iov_base = GetTLSBuffer();
1450 ioVec.iov_len = GetTLSBufferSize();
1451
1452 error = ReadRegisterSet(&ioVec, GetTLSBufferSize(), NT_ARM_TLS);
1453
1454 if (error.Success())
1455 m_tls_is_valid = true;
1456
1457 return error;
1458}
1459
1460Status NativeRegisterContextLinux_arm64::WriteTLS() {
1461 Status error;
1462
1463 error = ReadTLS();
1464 if (error.Fail())
1465 return error;
1466
1467 struct iovec ioVec;
1468 ioVec.iov_base = GetTLSBuffer();
1469 ioVec.iov_len = GetTLSBufferSize();
1470
1471 m_tls_is_valid = false;
1472
1473 return WriteRegisterSet(&ioVec, GetTLSBufferSize(), NT_ARM_TLS);
1474}
1475
1476Status NativeRegisterContextLinux_arm64::ReadGCS() {
1477 Status error;
1478
1479 if (m_gcs_is_valid)
1480 return error;
1481
1482 struct iovec ioVec;
1483 ioVec.iov_base = GetGCSBuffer();
1484 ioVec.iov_len = GetGCSBufferSize();
1485
1486 error = ReadRegisterSet(&ioVec, GetGCSBufferSize(), NT_ARM_GCS);
1487
1488 if (error.Success())
1489 m_gcs_is_valid = true;
1490
1491 return error;
1492}
1493
1494Status NativeRegisterContextLinux_arm64::WriteGCS() {
1495 Status error;
1496
1497 error = ReadGCS();
1498 if (error.Fail())
1499 return error;
1500
1501 struct iovec ioVec;
1502 ioVec.iov_base = GetGCSBuffer();
1503 ioVec.iov_len = GetGCSBufferSize();
1504
1505 m_gcs_is_valid = false;
1506
1507 return WriteRegisterSet(&ioVec, GetGCSBufferSize(), NT_ARM_GCS);
1508}
1509
1510Status NativeRegisterContextLinux_arm64::ReadZAHeader() {
1511 Status error;
1512
1513 if (m_za_header_is_valid)
1514 return error;
1515
1516 struct iovec ioVec;
1517 ioVec.iov_base = GetZAHeader();
1518 ioVec.iov_len = GetZAHeaderSize();
1519
1520 error = ReadRegisterSet(&ioVec, GetZAHeaderSize(), NT_ARM_ZA);
1521
1522 if (error.Success())
1523 m_za_header_is_valid = true;
1524
1525 return error;
1526}
1527
1528Status NativeRegisterContextLinux_arm64::ReadZA() {
1529 Status error;
1530
1531 if (m_za_buffer_is_valid)
1532 return error;
1533
1534 struct iovec ioVec;
1535 ioVec.iov_base = GetZABuffer();
1536 ioVec.iov_len = GetZABufferSize();
1537
1538 error = ReadRegisterSet(&ioVec, GetZABufferSize(), NT_ARM_ZA);
1539
1540 if (error.Success())
1541 m_za_buffer_is_valid = true;
1542
1543 return error;
1544}
1545
1546Status NativeRegisterContextLinux_arm64::WriteZA() {
1547 // Note that because the ZA ptrace payload contains the header also, this
1548 // method will write both. This is done because writing only the header
1549 // will disable ZA, even if .size in the header is correct for an enabled ZA.
1550 Status error;
1551
1552 error = ReadZA();
1553 if (error.Fail())
1554 return error;
1555
1556 struct iovec ioVec;
1557 ioVec.iov_base = GetZABuffer();
1558 ioVec.iov_len = GetZABufferSize();
1559
1560 m_za_buffer_is_valid = false;
1561 m_za_header_is_valid = false;
1562 // Writing to ZA may enable ZA, which means ZT0 may change too.
1563 m_zt_buffer_is_valid = false;
1564
1565 return WriteRegisterSet(&ioVec, GetZABufferSize(), NT_ARM_ZA);
1566}
1567
1568Status NativeRegisterContextLinux_arm64::ReadZT() {
1569 Status error;
1570
1571 if (m_zt_buffer_is_valid)
1572 return error;
1573
1574 struct iovec ioVec;
1575 ioVec.iov_base = GetZTBuffer();
1576 ioVec.iov_len = GetZTBufferSize();
1577
1578 error = ReadRegisterSet(&ioVec, GetZTBufferSize(), NT_ARM_ZT);
1579 m_zt_buffer_is_valid = error.Success();
1580
1581 return error;
1582}
1583
1584Status NativeRegisterContextLinux_arm64::WriteZT() {
1585 Status error;
1586
1587 error = ReadZT();
1588 if (error.Fail())
1589 return error;
1590
1591 struct iovec ioVec;
1592 ioVec.iov_base = GetZTBuffer();
1593 ioVec.iov_len = GetZTBufferSize();
1594
1595 m_zt_buffer_is_valid = false;
1596 // Writing to an inactive ZT0 will enable ZA as well, which invalidates our
1597 // current copy of it.
1598 m_za_buffer_is_valid = false;
1599 m_za_header_is_valid = false;
1600
1601 return WriteRegisterSet(&ioVec, GetZTBufferSize(), NT_ARM_ZT);
1602}
1603
1604Status NativeRegisterContextLinux_arm64::ReadFPMR() {
1605 Status error;
1606
1607 if (m_fpmr_is_valid)
1608 return error;
1609
1610 struct iovec ioVec;
1611 ioVec.iov_base = GetFPMRBuffer();
1612 ioVec.iov_len = GetFPMRBufferSize();
1613
1614 error = ReadRegisterSet(&ioVec, GetFPMRBufferSize(), NT_ARM_FPMR);
1615
1616 if (error.Success())
1617 m_fpmr_is_valid = true;
1618
1619 return error;
1620}
1621
1622Status NativeRegisterContextLinux_arm64::WriteFPMR() {
1623 Status error;
1624
1625 error = ReadFPMR();
1626 if (error.Fail())
1627 return error;
1628
1629 struct iovec ioVec;
1630 ioVec.iov_base = GetFPMRBuffer();
1631 ioVec.iov_len = GetFPMRBufferSize();
1632
1633 m_fpmr_is_valid = false;
1634
1635 return WriteRegisterSet(&ioVec, GetFPMRBufferSize(), NT_ARM_FPMR);
1636}
1637
1638void NativeRegisterContextLinux_arm64::ConfigureRegisterContext() {
1639 // ConfigureRegisterContext gets called from InvalidateAllRegisters
1640 // on every stop and configures SVE vector length and whether we are in
1641 // streaming SVE mode.
1642 // If m_sve_state is set to SVEState::Disabled on first stop, code below will
1643 // be deemed non operational for the lifetime of current process.
1644 if (!m_sve_header_is_valid && m_sve_state != SVEState::Disabled) {
1645 // If we have SVE we may also have the SVE streaming mode that SME added.
1646 // We can read the header of either mode, but only the active mode will
1647 // have valid register data.
1648
1649 // Check whether SME is present and the streaming SVE mode is active.
1650 m_sve_header_is_valid = false;
1651 m_sve_buffer_is_valid = false;
1652 m_sve_state = SVEState::Streaming;
1653 Status error = ReadSVEHeader();
1654
1655 // Streaming mode is active if the header has the SVE active flag set.
1656 if (!(error.Success() && ((m_sve_header.flags & sve::ptrace_regs_mask) ==
1657 sve::ptrace_regs_sve))) {
1658 // Non-streaming might be active instead.
1659 m_sve_header_is_valid = false;
1660 m_sve_buffer_is_valid = false;
1661 m_sve_state = SVEState::Full;
1662 error = ReadSVEHeader();
1663 if (error.Success()) {
1664 // If SVE is enabled thread can switch between SVEState::FPSIMD and
1665 // SVEState::Full on every stop.
1666 if ((m_sve_header.flags & sve::ptrace_regs_mask) ==
1667 sve::ptrace_regs_fpsimd)
1668 m_sve_state = SVEState::FPSIMD;
1669 // Else we are in SVEState::Full.
1670 } else {
1671 m_sve_state = SVEState::Disabled;
1672 }
1673 }
1674
1675 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::FPSIMD ||
1676 m_sve_state == SVEState::Streaming) {
1677 // On every stop we configure SVE vector length by calling
1678 // ConfigureVectorLengthSVE regardless of current SVEState of this thread.
1679 uint32_t vq = RegisterInfoPOSIX_arm64::eVectorQuadwordAArch64SVE;
1680 if (sve::vl_valid(m_sve_header.vl))
1681 vq = sve::vq_from_vl(m_sve_header.vl);
1682
1683 GetRegisterInfo().ConfigureVectorLengthSVE(vq);
1684 m_sve_ptrace_payload.resize(sve::PTraceSize(vq, sve::ptrace_regs_sve));
1685 }
1686 }
1687
1688 if (!m_za_header_is_valid) {
1689 Status error = ReadZAHeader();
1690 if (error.Success()) {
1691 uint32_t vq = RegisterInfoPOSIX_arm64::eVectorQuadwordAArch64SVE;
1692 if (sve::vl_valid(m_za_header.vl))
1693 vq = sve::vq_from_vl(m_za_header.vl);
1694
1695 GetRegisterInfo().ConfigureVectorLengthZA(vq);
1696 m_za_ptrace_payload.resize(m_za_header.size);
1697 m_za_buffer_is_valid = false;
1698 }
1699 }
1700}
1701
1702uint32_t NativeRegisterContextLinux_arm64::CalculateFprOffset(
1703 const RegisterInfo *reg_info) const {
1704 return reg_info->byte_offset - GetGPRSize();
1705}
1706
1707uint32_t NativeRegisterContextLinux_arm64::CalculateSVEOffset(
1708 const RegisterInfo *reg_info) const {
1709 // Start of Z0 data is after GPRs plus 8 bytes of vg register
1710 uint32_t sve_reg_offset = LLDB_INVALID_INDEX32;
1711 if (m_sve_state == SVEState::FPSIMD) {
1712 const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];
1713 sve_reg_offset = sve::ptrace_fpsimd_offset +
1714 (reg - GetRegisterInfo().GetRegNumSVEZ0()) * 16;
1715 // Between non-streaming and streaming mode, the layout is identical.
1716 } else if (m_sve_state == SVEState::Full ||
1717 m_sve_state == SVEState::Streaming) {
1718 uint32_t sve_z0_offset = GetGPRSize() + 16;
1719 sve_reg_offset =
1720 sve::SigRegsOffset() + reg_info->byte_offset - sve_z0_offset;
1721 }
1722 return sve_reg_offset;
1723}
1724
1725Status NativeRegisterContextLinux_arm64::ReadSMESVG() {
1726 // This register is the streaming vector length, so we will get it from
1727 // NT_ARM_ZA regardless of the current streaming mode.
1728 Status error = ReadZAHeader();
1729 if (error.Success())
1730 m_sme_pseudo_regs.svg_reg = m_za_header.vl / 8;
1731
1732 return error;
1733}
1734
1735std::vector<uint32_t> NativeRegisterContextLinux_arm64::GetExpeditedRegisters(
1736 ExpeditedRegs expType) const {
1737 std::vector<uint32_t> expedited_reg_nums =
1738 NativeRegisterContext::GetExpeditedRegisters(expType);
1739 // SVE, non-streaming vector length.
1740 if (m_sve_state == SVEState::FPSIMD || m_sve_state == SVEState::Full)
1741 expedited_reg_nums.push_back(GetRegisterInfo().GetRegNumSVEVG());
1742 // SME, streaming vector length. This is used by the ZA register which is
1743 // present even when streaming mode is not enabled.
1744 if (GetRegisterInfo().IsSSVEPresent())
1745 expedited_reg_nums.push_back(GetRegisterInfo().GetRegNumSMESVG());
1746
1747 return expedited_reg_nums;
1748}
1749
1750llvm::Expected<NativeRegisterContextLinux::MemoryTaggingDetails>
1751NativeRegisterContextLinux_arm64::GetMemoryTaggingDetails(int32_t type) {
1752 if (type == MemoryTagManagerAArch64MTE::eMTE_allocation) {
1753 return MemoryTaggingDetails{std::make_unique<MemoryTagManagerAArch64MTE>(),
1754 PTRACE_PEEKMTETAGS, PTRACE_POKEMTETAGS};
1755 }
1756
1757 return llvm::createStringError(llvm::inconvertibleErrorCode(),
1758 "Unknown AArch64 memory tag type %d", type);
1759}
1760
1761lldb::addr_t NativeRegisterContextLinux_arm64::FixWatchpointHitAddress(
1762 lldb::addr_t hit_addr) {
1763 // Linux configures user-space virtual addresses with top byte ignored.
1764 // We set default value of mask such that top byte is masked out.
1765 lldb::addr_t mask = ~((1ULL << 56) - 1);
1766
1767 // Try to read pointer authentication data_mask register and calculate a
1768 // consolidated data address mask after ignoring the top byte.
1769 if (ReadPAuthMask().Success())
1770 mask |= m_pac_mask.data_mask;
1771
1772 return hit_addr & ~mask;
1773 ;
1774}
1775
1776#endif // defined (__arm64__) || defined (__aarch64__)
1777

source code of lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp