1//===-- NativeRegisterContextLinux_arm64.cpp ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#if defined(__arm64__) || defined(__aarch64__)
10
11#include "NativeRegisterContextLinux_arm.h"
12#include "NativeRegisterContextLinux_arm64.h"
13
14#include "lldb/Host/HostInfo.h"
15#include "lldb/Host/common/NativeProcessProtocol.h"
16#include "lldb/Host/linux/Ptrace.h"
17#include "lldb/Utility/DataBufferHeap.h"
18#include "lldb/Utility/Log.h"
19#include "lldb/Utility/RegisterValue.h"
20#include "lldb/Utility/Status.h"
21
22#include "Plugins/Process/Linux/NativeProcessLinux.h"
23#include "Plugins/Process/Linux/Procfs.h"
24#include "Plugins/Process/POSIX/ProcessPOSIXLog.h"
25#include "Plugins/Process/Utility/MemoryTagManagerAArch64MTE.h"
26#include "Plugins/Process/Utility/RegisterFlagsDetector_arm64.h"
27#include "Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h"
28
29// System includes - They have to be included after framework includes because
30// they define some macros which collide with variable names in other modules
31#include <sys/uio.h>
32// NT_PRSTATUS and NT_FPREGSET definition
33#include <elf.h>
34#include <mutex>
35#include <optional>
36
37#ifndef NT_ARM_SVE
38#define NT_ARM_SVE 0x405 /* ARM Scalable Vector Extension */
39#endif
40
41#ifndef NT_ARM_SSVE
42#define NT_ARM_SSVE \
43 0x40b /* ARM Scalable Matrix Extension, Streaming SVE mode */
44#endif
45
46#ifndef NT_ARM_ZA
47#define NT_ARM_ZA 0x40c /* ARM Scalable Matrix Extension, Array Storage */
48#endif
49
50#ifndef NT_ARM_ZT
51#define NT_ARM_ZT \
52 0x40d /* ARM Scalable Matrix Extension 2, lookup table register */
53#endif
54
55#ifndef NT_ARM_PAC_MASK
56#define NT_ARM_PAC_MASK 0x406 /* Pointer authentication code masks */
57#endif
58
59#ifndef NT_ARM_TAGGED_ADDR_CTRL
60#define NT_ARM_TAGGED_ADDR_CTRL 0x409 /* Tagged address control register */
61#endif
62
63#ifndef NT_ARM_FPMR
64#define NT_ARM_FPMR 0x40e /* Floating point mode register */
65#endif
66
67#ifndef NT_ARM_GCS
68#define NT_ARM_GCS 0x410 /* Guarded Control Stack control registers */
69#endif
70
71#define HWCAP_PACA (1 << 30)
72
73#define HWCAP_GCS (1UL << 32)
74
75#define HWCAP2_MTE (1 << 18)
76
77#define HWCAP2_FPMR (1UL << 48)
78
79using namespace lldb;
80using namespace lldb_private;
81using namespace lldb_private::process_linux;
82
83// A NativeRegisterContext is constructed per thread, but all threads' registers
84// will contain the same fields. Therefore this mutex prevents each instance
85// competing with the other, and subsequent instances from having to detect the
86// fields all over again.
87static std::mutex g_register_flags_detector_mutex;
88static Arm64RegisterFlagsDetector g_register_flags_detector;
89
90std::unique_ptr<NativeRegisterContextLinux>
91NativeRegisterContextLinux::CreateHostNativeRegisterContextLinux(
92 const ArchSpec &target_arch, NativeThreadLinux &native_thread) {
93 switch (target_arch.GetMachine()) {
94 case llvm::Triple::arm:
95 return std::make_unique<NativeRegisterContextLinux_arm>(target_arch,
96 native_thread);
97 case llvm::Triple::aarch64: {
98 // Configure register sets supported by this AArch64 target.
99 // Read SVE header to check for SVE support.
100 struct sve::user_sve_header sve_header;
101 struct iovec ioVec;
102 ioVec.iov_base = &sve_header;
103 ioVec.iov_len = sizeof(sve_header);
104 unsigned int regset = NT_ARM_SVE;
105
106 Flags opt_regsets;
107 if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
108 native_thread.GetID(), &regset,
109 &ioVec, sizeof(sve_header))
110 .Success()) {
111 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskSVE);
112
113 // We may also have the Scalable Matrix Extension (SME) which adds a
114 // streaming SVE mode.
115 ioVec.iov_len = sizeof(sve_header);
116 regset = NT_ARM_SSVE;
117 if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
118 native_thread.GetID(), &regset,
119 &ioVec, sizeof(sve_header))
120 .Success())
121 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskSSVE);
122 }
123
124 sve::user_za_header za_header;
125 ioVec.iov_base = &za_header;
126 ioVec.iov_len = sizeof(za_header);
127 regset = NT_ARM_ZA;
128 if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
129 native_thread.GetID(), &regset,
130 &ioVec, sizeof(za_header))
131 .Success())
132 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskZA);
133
134 // SME's ZT0 is a 512 bit register.
135 std::array<uint8_t, 64> zt_reg;
136 ioVec.iov_base = zt_reg.data();
137 ioVec.iov_len = zt_reg.size();
138 regset = NT_ARM_ZT;
139 if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
140 native_thread.GetID(), &regset,
141 &ioVec, zt_reg.size())
142 .Success())
143 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskZT);
144
145 NativeProcessLinux &process = native_thread.GetProcess();
146
147 std::optional<uint64_t> auxv_at_hwcap =
148 process.GetAuxValue(AuxVector::AUXV_AT_HWCAP);
149 if (auxv_at_hwcap && (*auxv_at_hwcap & HWCAP_PACA))
150 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskPAuth);
151
152 std::optional<uint64_t> auxv_at_hwcap2 =
153 process.GetAuxValue(AuxVector::AUXV_AT_HWCAP2);
154 if (auxv_at_hwcap2) {
155 if (*auxv_at_hwcap2 & HWCAP2_MTE)
156 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskMTE);
157 if (*auxv_at_hwcap2 & HWCAP2_FPMR)
158 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskFPMR);
159 if (*auxv_at_hwcap & HWCAP_GCS)
160 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskGCS);
161 }
162
163 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskTLS);
164
165 std::lock_guard<std::mutex> lock(g_register_flags_detector_mutex);
166 if (!g_register_flags_detector.HasDetected())
167 g_register_flags_detector.DetectFields(auxv_at_hwcap.value_or(0),
168 auxv_at_hwcap2.value_or(0));
169
170 auto register_info_up =
171 std::make_unique<RegisterInfoPOSIX_arm64>(target_arch, opt_regsets);
172 return std::make_unique<NativeRegisterContextLinux_arm64>(
173 target_arch, native_thread, std::move(register_info_up));
174 }
175 default:
176 llvm_unreachable("have no register context for architecture");
177 }
178}
179
180llvm::Expected<ArchSpec>
181NativeRegisterContextLinux::DetermineArchitecture(lldb::tid_t tid) {
182 return DetermineArchitectureViaGPR(
183 tid, RegisterInfoPOSIX_arm64::GetGPRSizeStatic());
184}
185
186NativeRegisterContextLinux_arm64::NativeRegisterContextLinux_arm64(
187 const ArchSpec &target_arch, NativeThreadProtocol &native_thread,
188 std::unique_ptr<RegisterInfoPOSIX_arm64> register_info_up)
189 : NativeRegisterContextRegisterInfo(native_thread,
190 register_info_up.release()),
191 NativeRegisterContextLinux(native_thread) {
192 g_register_flags_detector.UpdateRegisterInfo(
193 GetRegisterInfoInterface().GetRegisterInfo(),
194 GetRegisterInfoInterface().GetRegisterCount());
195
196 ::memset(&m_fpr, 0, sizeof(m_fpr));
197 ::memset(&m_gpr_arm64, 0, sizeof(m_gpr_arm64));
198 ::memset(&m_hwp_regs, 0, sizeof(m_hwp_regs));
199 ::memset(&m_hbp_regs, 0, sizeof(m_hbp_regs));
200 ::memset(&m_sve_header, 0, sizeof(m_sve_header));
201 ::memset(&m_pac_mask, 0, sizeof(m_pac_mask));
202 ::memset(&m_tls_regs, 0, sizeof(m_tls_regs));
203 ::memset(&m_sme_pseudo_regs, 0, sizeof(m_sme_pseudo_regs));
204 ::memset(&m_gcs_regs, 0, sizeof(m_gcs_regs));
205 std::fill(m_zt_reg.begin(), m_zt_reg.end(), 0);
206
207 m_mte_ctrl_reg = 0;
208 m_fpmr_reg = 0;
209
210 // 16 is just a maximum value, query hardware for actual watchpoint count
211 m_max_hwp_supported = 16;
212 m_max_hbp_supported = 16;
213
214 m_refresh_hwdebug_info = true;
215
216 m_gpr_is_valid = false;
217 m_fpu_is_valid = false;
218 m_sve_buffer_is_valid = false;
219 m_sve_header_is_valid = false;
220 m_pac_mask_is_valid = false;
221 m_mte_ctrl_is_valid = false;
222 m_tls_is_valid = false;
223 m_zt_buffer_is_valid = false;
224 m_fpmr_is_valid = false;
225 m_gcs_is_valid = false;
226
227 // SME adds the tpidr2 register
228 m_tls_size = GetRegisterInfo().IsSSVEPresent() ? sizeof(m_tls_regs)
229 : sizeof(m_tls_regs.tpidr_reg);
230
231 if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent())
232 m_sve_state = SVEState::Unknown;
233 else
234 m_sve_state = SVEState::Disabled;
235}
236
237RegisterInfoPOSIX_arm64 &
238NativeRegisterContextLinux_arm64::GetRegisterInfo() const {
239 return static_cast<RegisterInfoPOSIX_arm64 &>(*m_register_info_interface_up);
240}
241
242uint32_t NativeRegisterContextLinux_arm64::GetRegisterSetCount() const {
243 return GetRegisterInfo().GetRegisterSetCount();
244}
245
246const RegisterSet *
247NativeRegisterContextLinux_arm64::GetRegisterSet(uint32_t set_index) const {
248 return GetRegisterInfo().GetRegisterSet(set_index);
249}
250
251uint32_t NativeRegisterContextLinux_arm64::GetUserRegisterCount() const {
252 uint32_t count = 0;
253 for (uint32_t set_index = 0; set_index < GetRegisterSetCount(); ++set_index)
254 count += GetRegisterSet(set_index)->num_registers;
255 return count;
256}
257
258Status
259NativeRegisterContextLinux_arm64::ReadRegister(const RegisterInfo *reg_info,
260 RegisterValue &reg_value) {
261 Status error;
262
263 if (!reg_info) {
264 error = Status::FromErrorString("reg_info NULL");
265 return error;
266 }
267
268 const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];
269
270 if (reg == LLDB_INVALID_REGNUM)
271 return Status::FromErrorStringWithFormat(
272 "no lldb regnum for %s",
273 reg_info && reg_info->name ? reg_info->name : "<unknown register>");
274
275 uint8_t *src;
276 uint32_t offset = LLDB_INVALID_INDEX32;
277 uint64_t sve_vg;
278 std::vector<uint8_t> sve_reg_non_live;
279
280 if (IsGPR(reg)) {
281 error = ReadGPR();
282 if (error.Fail())
283 return error;
284
285 offset = reg_info->byte_offset;
286 assert(offset < GetGPRSize());
287 src = (uint8_t *)GetGPRBuffer() + offset;
288
289 } else if (IsFPR(reg)) {
290 if (m_sve_state == SVEState::Disabled) {
291 // SVE is disabled take legacy route for FPU register access
292 error = ReadFPR();
293 if (error.Fail())
294 return error;
295
296 offset = CalculateFprOffset(reg_info);
297 assert(offset < GetFPRSize());
298 src = (uint8_t *)GetFPRBuffer() + offset;
299 } else {
300 // SVE or SSVE enabled, we will read and cache SVE ptrace data.
301 // In SIMD or Full mode, the data comes from the SVE regset. In streaming
302 // mode it comes from the streaming SVE regset.
303 error = ReadAllSVE();
304 if (error.Fail())
305 return error;
306
307 // FPSR and FPCR will be located right after Z registers in
308 // SVEState::FPSIMD while in SVEState::Full or SVEState::Streaming they
309 // will be located at the end of register data after an alignment
310 // correction based on currently selected vector length.
311 uint32_t sve_reg_num = LLDB_INVALID_REGNUM;
312 if (reg == GetRegisterInfo().GetRegNumFPSR()) {
313 sve_reg_num = reg;
314 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
315 offset = sve::PTraceFPSROffset(sve::vq_from_vl(m_sve_header.vl));
316 else if (m_sve_state == SVEState::FPSIMD)
317 offset = sve::ptrace_fpsimd_offset + (32 * 16);
318 } else if (reg == GetRegisterInfo().GetRegNumFPCR()) {
319 sve_reg_num = reg;
320 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
321 offset = sve::PTraceFPCROffset(sve::vq_from_vl(m_sve_header.vl));
322 else if (m_sve_state == SVEState::FPSIMD)
323 offset = sve::ptrace_fpsimd_offset + (32 * 16) + 4;
324 } else {
325 // Extract SVE Z register value register number for this reg_info
326 if (reg_info->value_regs &&
327 reg_info->value_regs[0] != LLDB_INVALID_REGNUM)
328 sve_reg_num = reg_info->value_regs[0];
329 offset = CalculateSVEOffset(GetRegisterInfoAtIndex(sve_reg_num));
330 }
331
332 assert(offset < GetSVEBufferSize());
333 src = (uint8_t *)GetSVEBuffer() + offset;
334 }
335 } else if (IsTLS(reg)) {
336 error = ReadTLS();
337 if (error.Fail())
338 return error;
339
340 offset = reg_info->byte_offset - GetRegisterInfo().GetTLSOffset();
341 assert(offset < GetTLSBufferSize());
342 src = (uint8_t *)GetTLSBuffer() + offset;
343 } else if (IsSVE(reg)) {
344 if (m_sve_state == SVEState::Disabled || m_sve_state == SVEState::Unknown)
345 return Status::FromErrorString("SVE disabled or not supported");
346
347 if (GetRegisterInfo().IsSVERegVG(reg)) {
348 sve_vg = GetSVERegVG();
349 src = (uint8_t *)&sve_vg;
350 } else {
351 // SVE enabled, we will read and cache SVE ptrace data
352 error = ReadAllSVE();
353 if (error.Fail())
354 return error;
355
356 if (m_sve_state == SVEState::FPSIMD) {
357 // In FPSIMD state SVE payload mirrors legacy fpsimd struct and so
358 // just copy 16 bytes of v register to the start of z register. All
359 // other SVE register will be set to zero.
360 sve_reg_non_live.resize(reg_info->byte_size, 0);
361 src = sve_reg_non_live.data();
362
363 if (GetRegisterInfo().IsSVEZReg(reg)) {
364 offset = CalculateSVEOffset(reg_info);
365 assert(offset < GetSVEBufferSize());
366 ::memcpy(sve_reg_non_live.data(), (uint8_t *)GetSVEBuffer() + offset,
367 16);
368 }
369 } else {
370 offset = CalculateSVEOffset(reg_info);
371 assert(offset < GetSVEBufferSize());
372 src = (uint8_t *)GetSVEBuffer() + offset;
373 }
374 }
375 } else if (IsPAuth(reg)) {
376 error = ReadPAuthMask();
377 if (error.Fail())
378 return error;
379
380 offset = reg_info->byte_offset - GetRegisterInfo().GetPAuthOffset();
381 assert(offset < GetPACMaskSize());
382 src = (uint8_t *)GetPACMask() + offset;
383 } else if (IsMTE(reg)) {
384 error = ReadMTEControl();
385 if (error.Fail())
386 return error;
387
388 offset = reg_info->byte_offset - GetRegisterInfo().GetMTEOffset();
389 assert(offset < GetMTEControlSize());
390 src = (uint8_t *)GetMTEControl() + offset;
391 } else if (IsSME(reg)) {
392 if (GetRegisterInfo().IsSMERegZA(reg)) {
393 error = ReadZAHeader();
394 if (error.Fail())
395 return error;
396
397 // If there is only a header and no registers, ZA is inactive. Read as 0
398 // in this case.
399 if (m_za_header.size == sizeof(m_za_header)) {
400 // This will get reconfigured/reset later, so we are safe to use it.
401 // ZA is a square of VL * VL and the ptrace buffer also includes the
402 // header itself.
403 m_za_ptrace_payload.resize(((m_za_header.vl) * (m_za_header.vl)) +
404 GetZAHeaderSize());
405 std::fill(m_za_ptrace_payload.begin(), m_za_ptrace_payload.end(), 0);
406 } else {
407 // ZA is active, read the real register.
408 error = ReadZA();
409 if (error.Fail())
410 return error;
411 }
412
413 // ZA is part of the SME set but uses a separate member buffer for
414 // storage. Therefore its effective byte offset is always 0 even if it
415 // isn't 0 within the SME register set.
416 src = (uint8_t *)GetZABuffer() + GetZAHeaderSize();
417 } else if (GetRegisterInfo().IsSMERegZT(reg)) {
418 // Unlike ZA, the kernel will return register data for ZT0 when ZA is not
419 // enabled. This data will be all 0s so we don't have to invent anything
420 // like we did for ZA.
421 error = ReadZT();
422 if (error.Fail())
423 return error;
424
425 src = (uint8_t *)GetZTBuffer();
426 } else {
427 error = ReadSMESVG();
428 if (error.Fail())
429 return error;
430
431 // This is a psuedo so it never fails.
432 ReadSMEControl();
433
434 offset = reg_info->byte_offset - GetRegisterInfo().GetSMEOffset();
435 assert(offset < GetSMEPseudoBufferSize());
436 src = (uint8_t *)GetSMEPseudoBuffer() + offset;
437 }
438 } else if (IsFPMR(reg)) {
439 error = ReadFPMR();
440 if (error.Fail())
441 return error;
442
443 offset = reg_info->byte_offset - GetRegisterInfo().GetFPMROffset();
444 assert(offset < GetFPMRBufferSize());
445 src = (uint8_t *)GetFPMRBuffer() + offset;
446 } else if (IsGCS(reg)) {
447 error = ReadGCS();
448 if (error.Fail())
449 return error;
450
451 offset = reg_info->byte_offset - GetRegisterInfo().GetGCSOffset();
452 assert(offset < GetGCSBufferSize());
453 src = (uint8_t *)GetGCSBuffer() + offset;
454 } else
455 return Status::FromErrorString(
456 "failed - register wasn't recognized to be a GPR or an FPR, "
457 "write strategy unknown");
458
459 reg_value.SetFromMemoryData(*reg_info, src, reg_info->byte_size,
460 eByteOrderLittle, error);
461
462 return error;
463}
464
465Status NativeRegisterContextLinux_arm64::WriteRegister(
466 const RegisterInfo *reg_info, const RegisterValue &reg_value) {
467 Status error;
468
469 if (!reg_info)
470 return Status::FromErrorString("reg_info NULL");
471
472 const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];
473
474 if (reg == LLDB_INVALID_REGNUM)
475 return Status::FromErrorStringWithFormat(
476 "no lldb regnum for %s",
477 reg_info && reg_info->name ? reg_info->name : "<unknown register>");
478
479 uint8_t *dst;
480 uint32_t offset = LLDB_INVALID_INDEX32;
481 std::vector<uint8_t> sve_reg_non_live;
482
483 if (IsGPR(reg)) {
484 error = ReadGPR();
485 if (error.Fail())
486 return error;
487
488 assert(reg_info->byte_offset < GetGPRSize());
489 dst = (uint8_t *)GetGPRBuffer() + reg_info->byte_offset;
490 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
491
492 return WriteGPR();
493 } else if (IsFPR(reg)) {
494 if (m_sve_state == SVEState::Disabled) {
495 // SVE is disabled take legacy route for FPU register access
496 error = ReadFPR();
497 if (error.Fail())
498 return error;
499
500 offset = CalculateFprOffset(reg_info);
501 assert(offset < GetFPRSize());
502 dst = (uint8_t *)GetFPRBuffer() + offset;
503 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
504
505 return WriteFPR();
506 } else {
507 // SVE enabled, we will read and cache SVE ptrace data.
508 error = ReadAllSVE();
509 if (error.Fail())
510 return error;
511
512 // FPSR and FPCR will be located right after Z registers in
513 // SVEState::FPSIMD while in SVEState::Full or SVEState::Streaming they
514 // will be located at the end of register data after an alignment
515 // correction based on currently selected vector length.
516 uint32_t sve_reg_num = LLDB_INVALID_REGNUM;
517 if (reg == GetRegisterInfo().GetRegNumFPSR()) {
518 sve_reg_num = reg;
519 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
520 offset = sve::PTraceFPSROffset(sve::vq_from_vl(m_sve_header.vl));
521 else if (m_sve_state == SVEState::FPSIMD)
522 offset = sve::ptrace_fpsimd_offset + (32 * 16);
523 } else if (reg == GetRegisterInfo().GetRegNumFPCR()) {
524 sve_reg_num = reg;
525 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
526 offset = sve::PTraceFPCROffset(sve::vq_from_vl(m_sve_header.vl));
527 else if (m_sve_state == SVEState::FPSIMD)
528 offset = sve::ptrace_fpsimd_offset + (32 * 16) + 4;
529 } else {
530 // Extract SVE Z register value register number for this reg_info
531 if (reg_info->value_regs &&
532 reg_info->value_regs[0] != LLDB_INVALID_REGNUM)
533 sve_reg_num = reg_info->value_regs[0];
534 offset = CalculateSVEOffset(GetRegisterInfoAtIndex(sve_reg_num));
535 }
536
537 assert(offset < GetSVEBufferSize());
538 dst = (uint8_t *)GetSVEBuffer() + offset;
539 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
540 return WriteAllSVE();
541 }
542 } else if (IsSVE(reg)) {
543 if (m_sve_state == SVEState::Disabled || m_sve_state == SVEState::Unknown)
544 return Status::FromErrorString("SVE disabled or not supported");
545 else {
546 // Target has SVE enabled, we will read and cache SVE ptrace data
547 error = ReadAllSVE();
548 if (error.Fail())
549 return error;
550
551 if (GetRegisterInfo().IsSVERegVG(reg)) {
552 uint64_t vg_value = reg_value.GetAsUInt64();
553
554 if (sve::vl_valid(vg_value * 8)) {
555 if (m_sve_header_is_valid && vg_value == GetSVERegVG())
556 return error;
557
558 SetSVERegVG(vg_value);
559
560 error = WriteSVEHeader();
561 if (error.Success()) {
562 // Changing VG during streaming mode also changes the size of ZA.
563 if (m_sve_state == SVEState::Streaming)
564 m_za_header_is_valid = false;
565 ConfigureRegisterContext();
566 }
567
568 if (m_sve_header_is_valid && vg_value == GetSVERegVG())
569 return error;
570 }
571
572 return Status::FromErrorString("SVE vector length update failed.");
573 }
574
575 // If target supports SVE but currently in FPSIMD mode.
576 if (m_sve_state == SVEState::FPSIMD) {
577 // Here we will check if writing this SVE register enables
578 // SVEState::Full
579 bool set_sve_state_full = false;
580 const uint8_t *reg_bytes = (const uint8_t *)reg_value.GetBytes();
581 if (GetRegisterInfo().IsSVEZReg(reg)) {
582 for (uint32_t i = 16; i < reg_info->byte_size; i++) {
583 if (reg_bytes[i]) {
584 set_sve_state_full = true;
585 break;
586 }
587 }
588 } else if (GetRegisterInfo().IsSVEPReg(reg) ||
589 reg == GetRegisterInfo().GetRegNumSVEFFR()) {
590 for (uint32_t i = 0; i < reg_info->byte_size; i++) {
591 if (reg_bytes[i]) {
592 set_sve_state_full = true;
593 break;
594 }
595 }
596 }
597
598 if (!set_sve_state_full && GetRegisterInfo().IsSVEZReg(reg)) {
599 // We are writing a Z register which is zero beyond 16 bytes so copy
600 // first 16 bytes only as SVE payload mirrors legacy fpsimd structure
601 offset = CalculateSVEOffset(reg_info);
602 assert(offset < GetSVEBufferSize());
603 dst = (uint8_t *)GetSVEBuffer() + offset;
604 ::memcpy(dst, reg_value.GetBytes(), 16);
605
606 return WriteAllSVE();
607 } else
608 return Status::FromErrorString(
609 "SVE state change operation not supported");
610 } else {
611 offset = CalculateSVEOffset(reg_info);
612 assert(offset < GetSVEBufferSize());
613 dst = (uint8_t *)GetSVEBuffer() + offset;
614 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
615 return WriteAllSVE();
616 }
617 }
618 } else if (IsMTE(reg)) {
619 error = ReadMTEControl();
620 if (error.Fail())
621 return error;
622
623 offset = reg_info->byte_offset - GetRegisterInfo().GetMTEOffset();
624 assert(offset < GetMTEControlSize());
625 dst = (uint8_t *)GetMTEControl() + offset;
626 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
627
628 return WriteMTEControl();
629 } else if (IsTLS(reg)) {
630 error = ReadTLS();
631 if (error.Fail())
632 return error;
633
634 offset = reg_info->byte_offset - GetRegisterInfo().GetTLSOffset();
635 assert(offset < GetTLSBufferSize());
636 dst = (uint8_t *)GetTLSBuffer() + offset;
637 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
638
639 return WriteTLS();
640 } else if (IsSME(reg)) {
641 if (GetRegisterInfo().IsSMERegZA(reg)) {
642 error = ReadZA();
643 if (error.Fail())
644 return error;
645
646 // ZA is part of the SME set but not stored with the other SME registers.
647 // So its byte offset is effectively always 0.
648 dst = (uint8_t *)GetZABuffer() + GetZAHeaderSize();
649 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
650
651 // While this is writing a header that contains a vector length, the only
652 // way to change that is via the vg register. So here we assume the length
653 // will always be the current length and no reconfigure is needed.
654 return WriteZA();
655 } else if (GetRegisterInfo().IsSMERegZT(reg)) {
656 error = ReadZT();
657 if (error.Fail())
658 return error;
659
660 dst = (uint8_t *)GetZTBuffer();
661 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
662
663 return WriteZT();
664 } else
665 return Status::FromErrorString(
666 "Writing to SVG or SVCR is not supported.");
667 } else if (IsFPMR(reg)) {
668 error = ReadFPMR();
669 if (error.Fail())
670 return error;
671
672 offset = reg_info->byte_offset - GetRegisterInfo().GetFPMROffset();
673 assert(offset < GetFPMRBufferSize());
674 dst = (uint8_t *)GetFPMRBuffer() + offset;
675 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
676
677 return WriteFPMR();
678 } else if (IsGCS(reg)) {
679 error = ReadGCS();
680 if (error.Fail())
681 return error;
682
683 offset = reg_info->byte_offset - GetRegisterInfo().GetGCSOffset();
684 assert(offset < GetGCSBufferSize());
685 dst = (uint8_t *)GetGCSBuffer() + offset;
686 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
687
688 return WriteGCS();
689 }
690
691 return Status::FromErrorString("Failed to write register value");
692}
693
694enum RegisterSetType : uint32_t {
695 GPR,
696 SVE, // Used for SVE and SSVE.
697 FPR, // When there is no SVE, or SVE in FPSIMD mode.
698 // Pointer authentication registers are read only, so not included here.
699 MTE,
700 TLS,
701 SME, // ZA only, because SVCR and SVG are pseudo registers.
702 SME2, // ZT only.
703 FPMR,
704 GCS, // Guarded Control Stack registers.
705};
706
707static uint8_t *AddRegisterSetType(uint8_t *dst,
708 RegisterSetType register_set_type) {
709 *(reinterpret_cast<uint32_t *>(dst)) = register_set_type;
710 return dst + sizeof(uint32_t);
711}
712
713static uint8_t *AddSavedRegistersData(uint8_t *dst, void *src, size_t size) {
714 ::memcpy(dst, src, size);
715 return dst + size;
716}
717
718static uint8_t *AddSavedRegisters(uint8_t *dst,
719 enum RegisterSetType register_set_type,
720 void *src, size_t size) {
721 dst = AddRegisterSetType(dst, register_set_type);
722 return AddSavedRegistersData(dst, src, size);
723}
724
725Status
726NativeRegisterContextLinux_arm64::CacheAllRegisters(uint32_t &cached_size) {
727 Status error;
728 cached_size = sizeof(RegisterSetType) + GetGPRBufferSize();
729 error = ReadGPR();
730 if (error.Fail())
731 return error;
732
733 if (GetRegisterInfo().IsZAPresent()) {
734 error = ReadZAHeader();
735 if (error.Fail())
736 return error;
737 // Use header size here because the buffer may contain fake data when ZA is
738 // disabled. We do not want to write this fake data (all 0s) because this
739 // would tell the kernel that we want ZA to become active. Which is the
740 // opposite of what we want in the case where it is currently inactive.
741 cached_size += sizeof(RegisterSetType) + m_za_header.size;
742 // For the same reason, we need to force it to be re-read so that it will
743 // always contain the real header.
744 m_za_buffer_is_valid = false;
745 error = ReadZA();
746 if (error.Fail())
747 return error;
748
749 // We will only be restoring ZT data if ZA is active. As writing to an
750 // inactive ZT enables ZA, which may not be desireable.
751 if (
752 // If we have ZT0, or in other words, if we have SME2.
753 GetRegisterInfo().IsZTPresent() &&
754 // And ZA is active, which means that ZT0 is also active.
755 m_za_header.size > sizeof(m_za_header)) {
756 cached_size += sizeof(RegisterSetType) + GetZTBufferSize();
757 // The kernel handles an inactive ZT0 for us, and it will read as 0s if
758 // inactive (unlike ZA where we fake that behaviour).
759 error = ReadZT();
760 if (error.Fail())
761 return error;
762 }
763 }
764
765 // If SVE is enabled we need not copy FPR separately.
766 if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent()) {
767 // Store mode and register data.
768 cached_size +=
769 sizeof(RegisterSetType) + sizeof(m_sve_state) + GetSVEBufferSize();
770 error = ReadAllSVE();
771 } else {
772 cached_size += sizeof(RegisterSetType) + GetFPRSize();
773 error = ReadFPR();
774 }
775 if (error.Fail())
776 return error;
777
778 if (GetRegisterInfo().IsMTEPresent()) {
779 cached_size += sizeof(RegisterSetType) + GetMTEControlSize();
780 error = ReadMTEControl();
781 if (error.Fail())
782 return error;
783 }
784
785 if (GetRegisterInfo().IsFPMRPresent()) {
786 cached_size += sizeof(RegisterSetType) + GetFPMRBufferSize();
787 error = ReadFPMR();
788 if (error.Fail())
789 return error;
790 }
791
792 if (GetRegisterInfo().IsGCSPresent()) {
793 cached_size += sizeof(RegisterSetType) + GetGCSBufferSize();
794 error = ReadGCS();
795 if (error.Fail())
796 return error;
797 }
798
799 // tpidr is always present but tpidr2 depends on SME.
800 cached_size += sizeof(RegisterSetType) + GetTLSBufferSize();
801 error = ReadTLS();
802
803 return error;
804}
805
806Status NativeRegisterContextLinux_arm64::ReadAllRegisterValues(
807 lldb::WritableDataBufferSP &data_sp) {
808 // AArch64 register data must contain GPRs and either FPR or SVE registers.
809 // SVE registers can be non-streaming (aka SVE) or streaming (aka SSVE).
810 // Finally an optional MTE register. Pointer Authentication (PAC) registers
811 // are read-only and will be skipped.
812
813 // In order to create register data checkpoint we first read all register
814 // values if not done already and calculate total size of register set data.
815 // We store all register values in data_sp by copying full PTrace data that
816 // corresponds to register sets enabled by current register context.
817
818 uint32_t reg_data_byte_size = 0;
819 Status error = CacheAllRegisters(reg_data_byte_size);
820 if (error.Fail())
821 return error;
822
823 data_sp.reset(new DataBufferHeap(reg_data_byte_size, 0));
824 uint8_t *dst = data_sp->GetBytes();
825
826 dst = AddSavedRegisters(dst, RegisterSetType::GPR, GetGPRBuffer(),
827 GetGPRBufferSize());
828
829 // Streaming SVE and the ZA register both use the streaming vector length.
830 // When you change this, the kernel will invalidate parts of the process
831 // state. Therefore we need a specific order of restoration for each mode, if
832 // we also have ZA to restore.
833 //
834 // Streaming mode enabled, ZA enabled:
835 // * Write streaming registers. This sets SVCR.SM and clears SVCR.ZA.
836 // * Write ZA, this set SVCR.ZA. The register data we provide is written to
837 // ZA.
838 // * Result is SVCR.SM and SVCR.ZA set, with the expected data in both
839 // register sets.
840 //
841 // Streaming mode disabled, ZA enabled:
842 // * Write ZA. This sets SVCR.ZA, and the ZA content. In the majority of cases
843 // the streaming vector length is changing, so the thread is converted into
844 // an FPSIMD thread if it is not already one. This also clears SVCR.SM.
845 // * Write SVE registers, which also clears SVCR.SM but most importantly, puts
846 // us into full SVE mode instead of FPSIMD mode (where the registers are
847 // actually the 128 bit Neon registers).
848 // * Result is we have SVCR.SM = 0, SVCR.ZA = 1 and the expected register
849 // state.
850 //
851 // Restoring in different orders leads to things like the SVE registers being
852 // truncated due to the FPSIMD mode and ZA being disabled or filled with 0s
853 // (disabled and 0s looks the same from inside lldb since we fake the value
854 // when it's disabled).
855 //
856 // For more information on this, look up the uses of the relevant NT_ARM_
857 // constants and the functions vec_set_vector_length, sve_set_common and
858 // za_set in the Linux Kernel.
859
860 if ((m_sve_state != SVEState::Streaming) && GetRegisterInfo().IsZAPresent()) {
861 // Use the header size not the buffer size, as we may be using the buffer
862 // for fake data, which we do not want to write out.
863 assert(m_za_header.size <= GetZABufferSize());
864 dst = AddSavedRegisters(dst, RegisterSetType::SME, GetZABuffer(),
865 m_za_header.size);
866 }
867
868 if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent()) {
869 dst = AddRegisterSetType(dst, RegisterSetType::SVE);
870 *(reinterpret_cast<SVEState *>(dst)) = m_sve_state;
871 dst += sizeof(m_sve_state);
872 dst = AddSavedRegistersData(dst, GetSVEBuffer(), GetSVEBufferSize());
873 } else {
874 dst = AddSavedRegisters(dst, RegisterSetType::FPR, GetFPRBuffer(),
875 GetFPRSize());
876 }
877
878 if ((m_sve_state == SVEState::Streaming) && GetRegisterInfo().IsZAPresent()) {
879 assert(m_za_header.size <= GetZABufferSize());
880 dst = AddSavedRegisters(dst, RegisterSetType::SME, GetZABuffer(),
881 m_za_header.size);
882 }
883
884 // If ZT0 is present and we are going to be restoring an active ZA (which
885 // implies an active ZT0), then restore ZT0 after ZA has been set. This
886 // prevents us enabling ZA accidentally after the restore of ZA disabled it.
887 // If we leave ZA/ZT0 inactive and read ZT0, the kernel returns 0s. Therefore
888 // there's nothing for us to restore if ZA was originally inactive.
889 if (
890 // If we have SME2 and therefore ZT0.
891 GetRegisterInfo().IsZTPresent() &&
892 // And ZA is enabled.
893 m_za_header.size > sizeof(m_za_header))
894 dst = AddSavedRegisters(dst, RegisterSetType::SME2, GetZTBuffer(),
895 GetZTBufferSize());
896
897 if (GetRegisterInfo().IsMTEPresent()) {
898 dst = AddSavedRegisters(dst, RegisterSetType::MTE, GetMTEControl(),
899 GetMTEControlSize());
900 }
901
902 if (GetRegisterInfo().IsFPMRPresent()) {
903 dst = AddSavedRegisters(dst, RegisterSetType::FPMR, GetFPMRBuffer(),
904 GetFPMRBufferSize());
905 }
906
907 if (GetRegisterInfo().IsGCSPresent()) {
908 dst = AddSavedRegisters(dst, RegisterSetType::GCS, GetGCSBuffer(),
909 GetGCSBufferSize());
910 }
911
912 dst = AddSavedRegisters(dst, RegisterSetType::TLS, GetTLSBuffer(),
913 GetTLSBufferSize());
914
915 return error;
916}
917
918static Status RestoreRegisters(void *buffer, const uint8_t **src, size_t len,
919 bool &is_valid, std::function<Status()> writer) {
920 ::memcpy(buffer, *src, len);
921 is_valid = true;
922 *src += len;
923 return writer();
924}
925
926Status NativeRegisterContextLinux_arm64::WriteAllRegisterValues(
927 const lldb::DataBufferSP &data_sp) {
928 // AArch64 register data must contain GPRs, either FPR or SVE registers
929 // (which can be streaming or non-streaming) and optional MTE register.
930 // Pointer Authentication (PAC) registers are read-only and will be skipped.
931
932 // We store all register values in data_sp by copying full PTrace data that
933 // corresponds to register sets enabled by current register context. In order
934 // to restore from register data checkpoint we will first restore GPRs, based
935 // on size of remaining register data either SVE or FPRs should be restored
936 // next. SVE is not enabled if we have register data size less than or equal
937 // to size of GPR + FPR + MTE.
938
939 Status error;
940 if (!data_sp) {
941 error = Status::FromErrorStringWithFormat(
942 "NativeRegisterContextLinux_arm64::%s invalid data_sp provided",
943 __FUNCTION__);
944 return error;
945 }
946
947 const uint8_t *src = data_sp->GetBytes();
948 if (src == nullptr) {
949 error = Status::FromErrorStringWithFormat(
950 "NativeRegisterContextLinux_arm64::%s "
951 "DataBuffer::GetBytes() returned a null "
952 "pointer",
953 __FUNCTION__);
954 return error;
955 }
956
957 uint64_t reg_data_min_size =
958 GetGPRBufferSize() + GetFPRSize() + 2 * (sizeof(RegisterSetType));
959 if (data_sp->GetByteSize() < reg_data_min_size) {
960 error = Status::FromErrorStringWithFormat(
961 "NativeRegisterContextLinux_arm64::%s data_sp contained insufficient "
962 "register data bytes, expected at least %" PRIu64 ", actual %" PRIu64,
963 __FUNCTION__, reg_data_min_size, data_sp->GetByteSize());
964 return error;
965 }
966
967 const uint8_t *end = src + data_sp->GetByteSize();
968 while (src < end) {
969 const RegisterSetType kind =
970 *reinterpret_cast<const RegisterSetType *>(src);
971 src += sizeof(RegisterSetType);
972
973 switch (kind) {
974 case RegisterSetType::GPR:
975 error = RestoreRegisters(
976 GetGPRBuffer(), &src, GetGPRBufferSize(), m_gpr_is_valid,
977 std::bind(&NativeRegisterContextLinux_arm64::WriteGPR, this));
978 break;
979 case RegisterSetType::SVE:
980 // Restore to the correct mode, streaming or not.
981 m_sve_state = static_cast<SVEState>(*src);
982 src += sizeof(m_sve_state);
983
984 // First write SVE header. We do not use RestoreRegisters because we do
985 // not want src to be modified yet.
986 ::memcpy(GetSVEHeader(), src, GetSVEHeaderSize());
987 if (!sve::vl_valid(m_sve_header.vl)) {
988 m_sve_header_is_valid = false;
989 error = Status::FromErrorStringWithFormat(
990 "NativeRegisterContextLinux_arm64::%s "
991 "Invalid SVE header in data_sp",
992 __FUNCTION__);
993 return error;
994 }
995 m_sve_header_is_valid = true;
996 error = WriteSVEHeader();
997 if (error.Fail())
998 return error;
999
1000 // SVE header has been written configure SVE vector length if needed.
1001 // This could change ZA data too, but that will be restored again later
1002 // anyway.
1003 ConfigureRegisterContext();
1004
1005 // Write header and register data, incrementing src this time.
1006 error = RestoreRegisters(
1007 GetSVEBuffer(), &src, GetSVEBufferSize(), m_sve_buffer_is_valid,
1008 std::bind(&NativeRegisterContextLinux_arm64::WriteAllSVE, this));
1009 break;
1010 case RegisterSetType::FPR:
1011 error = RestoreRegisters(
1012 GetFPRBuffer(), &src, GetFPRSize(), m_fpu_is_valid,
1013 std::bind(&NativeRegisterContextLinux_arm64::WriteFPR, this));
1014 break;
1015 case RegisterSetType::MTE:
1016 error = RestoreRegisters(
1017 GetMTEControl(), &src, GetMTEControlSize(), m_mte_ctrl_is_valid,
1018 std::bind(&NativeRegisterContextLinux_arm64::WriteMTEControl, this));
1019 break;
1020 case RegisterSetType::TLS:
1021 error = RestoreRegisters(
1022 GetTLSBuffer(), &src, GetTLSBufferSize(), m_tls_is_valid,
1023 std::bind(&NativeRegisterContextLinux_arm64::WriteTLS, this));
1024 break;
1025 case RegisterSetType::SME:
1026 // To enable or disable ZA you write the regset with or without register
1027 // data. The kernel detects this by looking at the ioVec's length, not the
1028 // ZA header size you pass in. Therefore we must write header and register
1029 // data (if present) in one go every time. Read the header only first just
1030 // to get the size.
1031 ::memcpy(GetZAHeader(), src, GetZAHeaderSize());
1032 // Read the header and register data. Can't use the buffer size here, it
1033 // may be incorrect due to being filled with dummy data previously. Resize
1034 // this so WriteZA uses the correct size.
1035 m_za_ptrace_payload.resize(m_za_header.size);
1036 ::memcpy(GetZABuffer(), src, GetZABufferSize());
1037 m_za_buffer_is_valid = true;
1038
1039 error = WriteZA();
1040 if (error.Fail())
1041 return error;
1042
1043 // Update size of ZA, which resizes the ptrace payload potentially
1044 // trashing our copy of the data we just wrote.
1045 ConfigureRegisterContext();
1046
1047 // ZA buffer now has proper size, read back the data we wrote above, from
1048 // ptrace.
1049 error = ReadZA();
1050 src += GetZABufferSize();
1051 break;
1052 case RegisterSetType::SME2:
1053 // Doing this would activate an inactive ZA, however we will only get here
1054 // if the state we are restoring had an active ZA. Restoring ZT0 will
1055 // always come after restoring ZA.
1056 error = RestoreRegisters(
1057 GetZTBuffer(), &src, GetZTBufferSize(), m_zt_buffer_is_valid,
1058 std::bind(&NativeRegisterContextLinux_arm64::WriteZT, this));
1059 break;
1060 case RegisterSetType::FPMR:
1061 error = RestoreRegisters(
1062 GetFPMRBuffer(), &src, GetFPMRBufferSize(), m_fpmr_is_valid,
1063 std::bind(&NativeRegisterContextLinux_arm64::WriteFPMR, this));
1064 break;
1065 case RegisterSetType::GCS:
1066 // It is not permitted to enable GCS via ptrace. We can disable it, but
1067 // to keep things simple we will not revert any change to the
1068 // PR_SHADOW_STACK_ENABLE bit. Instead patch in the current enable bit
1069 // into the registers we are about to restore.
1070 m_gcs_is_valid = false;
1071 error = ReadGCS();
1072 if (error.Fail())
1073 return error;
1074
1075 uint64_t enable_bit = m_gcs_regs.features_enabled & 1UL;
1076 gcs_regs new_gcs_regs = *reinterpret_cast<const gcs_regs *>(src);
1077 new_gcs_regs.features_enabled =
1078 (new_gcs_regs.features_enabled & ~1UL) | enable_bit;
1079
1080 const uint8_t *new_gcs_src =
1081 reinterpret_cast<const uint8_t *>(&new_gcs_regs);
1082 error = RestoreRegisters(
1083 GetGCSBuffer(), &new_gcs_src, GetGCSBufferSize(), m_gcs_is_valid,
1084 std::bind(&NativeRegisterContextLinux_arm64::WriteGCS, this));
1085 src += GetGCSBufferSize();
1086
1087 break;
1088 }
1089
1090 if (error.Fail())
1091 return error;
1092 }
1093
1094 return error;
1095}
1096
1097bool NativeRegisterContextLinux_arm64::IsGPR(unsigned reg) const {
1098 if (GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg) ==
1099 RegisterInfoPOSIX_arm64::GPRegSet)
1100 return true;
1101 return false;
1102}
1103
1104bool NativeRegisterContextLinux_arm64::IsFPR(unsigned reg) const {
1105 if (GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg) ==
1106 RegisterInfoPOSIX_arm64::FPRegSet)
1107 return true;
1108 return false;
1109}
1110
1111bool NativeRegisterContextLinux_arm64::IsSVE(unsigned reg) const {
1112 return GetRegisterInfo().IsSVEReg(reg);
1113}
1114
1115bool NativeRegisterContextLinux_arm64::IsSME(unsigned reg) const {
1116 return GetRegisterInfo().IsSMEReg(reg);
1117}
1118
1119bool NativeRegisterContextLinux_arm64::IsPAuth(unsigned reg) const {
1120 return GetRegisterInfo().IsPAuthReg(reg);
1121}
1122
1123bool NativeRegisterContextLinux_arm64::IsMTE(unsigned reg) const {
1124 return GetRegisterInfo().IsMTEReg(reg);
1125}
1126
1127bool NativeRegisterContextLinux_arm64::IsTLS(unsigned reg) const {
1128 return GetRegisterInfo().IsTLSReg(reg);
1129}
1130
1131bool NativeRegisterContextLinux_arm64::IsFPMR(unsigned reg) const {
1132 return GetRegisterInfo().IsFPMRReg(reg);
1133}
1134
1135bool NativeRegisterContextLinux_arm64::IsGCS(unsigned reg) const {
1136 return GetRegisterInfo().IsGCSReg(reg);
1137}
1138
1139llvm::Error NativeRegisterContextLinux_arm64::ReadHardwareDebugInfo() {
1140 if (!m_refresh_hwdebug_info) {
1141 return llvm::Error::success();
1142 }
1143
1144 ::pid_t tid = m_thread.GetID();
1145
1146 int regset = NT_ARM_HW_WATCH;
1147 struct iovec ioVec;
1148 struct user_hwdebug_state dreg_state;
1149 Status error;
1150
1151 ioVec.iov_base = &dreg_state;
1152 ioVec.iov_len = sizeof(dreg_state);
1153 error = NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET, tid, &regset,
1154 &ioVec, ioVec.iov_len);
1155
1156 if (error.Fail())
1157 return error.ToError();
1158
1159 m_max_hwp_supported = dreg_state.dbg_info & 0xff;
1160
1161 regset = NT_ARM_HW_BREAK;
1162 error = NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET, tid, &regset,
1163 &ioVec, ioVec.iov_len);
1164
1165 if (error.Fail())
1166 return error.ToError();
1167
1168 m_max_hbp_supported = dreg_state.dbg_info & 0xff;
1169 m_refresh_hwdebug_info = false;
1170
1171 return llvm::Error::success();
1172}
1173
1174llvm::Error
1175NativeRegisterContextLinux_arm64::WriteHardwareDebugRegs(DREGType hwbType) {
1176 struct iovec ioVec;
1177 struct user_hwdebug_state dreg_state;
1178 int regset;
1179
1180 memset(&dreg_state, 0, sizeof(dreg_state));
1181 ioVec.iov_base = &dreg_state;
1182
1183 switch (hwbType) {
1184 case eDREGTypeWATCH:
1185 regset = NT_ARM_HW_WATCH;
1186 ioVec.iov_len = sizeof(dreg_state.dbg_info) + sizeof(dreg_state.pad) +
1187 (sizeof(dreg_state.dbg_regs[0]) * m_max_hwp_supported);
1188
1189 for (uint32_t i = 0; i < m_max_hwp_supported; i++) {
1190 dreg_state.dbg_regs[i].addr = m_hwp_regs[i].address;
1191 dreg_state.dbg_regs[i].ctrl = m_hwp_regs[i].control;
1192 }
1193 break;
1194 case eDREGTypeBREAK:
1195 regset = NT_ARM_HW_BREAK;
1196 ioVec.iov_len = sizeof(dreg_state.dbg_info) + sizeof(dreg_state.pad) +
1197 (sizeof(dreg_state.dbg_regs[0]) * m_max_hbp_supported);
1198
1199 for (uint32_t i = 0; i < m_max_hbp_supported; i++) {
1200 dreg_state.dbg_regs[i].addr = m_hbp_regs[i].address;
1201 dreg_state.dbg_regs[i].ctrl = m_hbp_regs[i].control;
1202 }
1203 break;
1204 }
1205
1206 return NativeProcessLinux::PtraceWrapper(PTRACE_SETREGSET, m_thread.GetID(),
1207 &regset, &ioVec, ioVec.iov_len)
1208 .ToError();
1209}
1210
1211Status NativeRegisterContextLinux_arm64::ReadGPR() {
1212 Status error;
1213
1214 if (m_gpr_is_valid)
1215 return error;
1216
1217 struct iovec ioVec;
1218 ioVec.iov_base = GetGPRBuffer();
1219 ioVec.iov_len = GetGPRBufferSize();
1220
1221 error = ReadRegisterSet(&ioVec, GetGPRBufferSize(), NT_PRSTATUS);
1222
1223 if (error.Success())
1224 m_gpr_is_valid = true;
1225
1226 return error;
1227}
1228
1229Status NativeRegisterContextLinux_arm64::WriteGPR() {
1230 Status error = ReadGPR();
1231 if (error.Fail())
1232 return error;
1233
1234 struct iovec ioVec;
1235 ioVec.iov_base = GetGPRBuffer();
1236 ioVec.iov_len = GetGPRBufferSize();
1237
1238 m_gpr_is_valid = false;
1239
1240 return WriteRegisterSet(&ioVec, GetGPRBufferSize(), NT_PRSTATUS);
1241}
1242
1243Status NativeRegisterContextLinux_arm64::ReadFPR() {
1244 Status error;
1245
1246 if (m_fpu_is_valid)
1247 return error;
1248
1249 struct iovec ioVec;
1250 ioVec.iov_base = GetFPRBuffer();
1251 ioVec.iov_len = GetFPRSize();
1252
1253 error = ReadRegisterSet(&ioVec, GetFPRSize(), NT_FPREGSET);
1254
1255 if (error.Success())
1256 m_fpu_is_valid = true;
1257
1258 return error;
1259}
1260
1261Status NativeRegisterContextLinux_arm64::WriteFPR() {
1262 Status error = ReadFPR();
1263 if (error.Fail())
1264 return error;
1265
1266 struct iovec ioVec;
1267 ioVec.iov_base = GetFPRBuffer();
1268 ioVec.iov_len = GetFPRSize();
1269
1270 m_fpu_is_valid = false;
1271
1272 return WriteRegisterSet(&ioVec, GetFPRSize(), NT_FPREGSET);
1273}
1274
1275void NativeRegisterContextLinux_arm64::InvalidateAllRegisters() {
1276 m_gpr_is_valid = false;
1277 m_fpu_is_valid = false;
1278 m_sve_buffer_is_valid = false;
1279 m_sve_header_is_valid = false;
1280 m_za_buffer_is_valid = false;
1281 m_za_header_is_valid = false;
1282 m_pac_mask_is_valid = false;
1283 m_mte_ctrl_is_valid = false;
1284 m_tls_is_valid = false;
1285 m_zt_buffer_is_valid = false;
1286 m_fpmr_is_valid = false;
1287 m_gcs_is_valid = false;
1288
1289 // Update SVE and ZA registers in case there is change in configuration.
1290 ConfigureRegisterContext();
1291}
1292
1293unsigned NativeRegisterContextLinux_arm64::GetSVERegSet() {
1294 return m_sve_state == SVEState::Streaming ? NT_ARM_SSVE : NT_ARM_SVE;
1295}
1296
1297Status NativeRegisterContextLinux_arm64::ReadSVEHeader() {
1298 Status error;
1299
1300 if (m_sve_header_is_valid)
1301 return error;
1302
1303 struct iovec ioVec;
1304 ioVec.iov_base = GetSVEHeader();
1305 ioVec.iov_len = GetSVEHeaderSize();
1306
1307 error = ReadRegisterSet(&ioVec, GetSVEHeaderSize(), GetSVERegSet());
1308
1309 if (error.Success())
1310 m_sve_header_is_valid = true;
1311
1312 return error;
1313}
1314
1315Status NativeRegisterContextLinux_arm64::ReadPAuthMask() {
1316 Status error;
1317
1318 if (m_pac_mask_is_valid)
1319 return error;
1320
1321 struct iovec ioVec;
1322 ioVec.iov_base = GetPACMask();
1323 ioVec.iov_len = GetPACMaskSize();
1324
1325 error = ReadRegisterSet(&ioVec, GetPACMaskSize(), NT_ARM_PAC_MASK);
1326
1327 if (error.Success())
1328 m_pac_mask_is_valid = true;
1329
1330 return error;
1331}
1332
1333Status NativeRegisterContextLinux_arm64::WriteSVEHeader() {
1334 Status error;
1335
1336 error = ReadSVEHeader();
1337 if (error.Fail())
1338 return error;
1339
1340 struct iovec ioVec;
1341 ioVec.iov_base = GetSVEHeader();
1342 ioVec.iov_len = GetSVEHeaderSize();
1343
1344 m_sve_buffer_is_valid = false;
1345 m_sve_header_is_valid = false;
1346 m_fpu_is_valid = false;
1347
1348 return WriteRegisterSet(&ioVec, GetSVEHeaderSize(), GetSVERegSet());
1349}
1350
1351Status NativeRegisterContextLinux_arm64::ReadAllSVE() {
1352 Status error;
1353 if (m_sve_buffer_is_valid)
1354 return error;
1355
1356 struct iovec ioVec;
1357 ioVec.iov_base = GetSVEBuffer();
1358 ioVec.iov_len = GetSVEBufferSize();
1359
1360 error = ReadRegisterSet(&ioVec, GetSVEBufferSize(), GetSVERegSet());
1361
1362 if (error.Success())
1363 m_sve_buffer_is_valid = true;
1364
1365 return error;
1366}
1367
1368Status NativeRegisterContextLinux_arm64::WriteAllSVE() {
1369 Status error;
1370
1371 error = ReadAllSVE();
1372 if (error.Fail())
1373 return error;
1374
1375 struct iovec ioVec;
1376
1377 ioVec.iov_base = GetSVEBuffer();
1378 ioVec.iov_len = GetSVEBufferSize();
1379
1380 m_sve_buffer_is_valid = false;
1381 m_sve_header_is_valid = false;
1382 m_fpu_is_valid = false;
1383
1384 return WriteRegisterSet(&ioVec, GetSVEBufferSize(), GetSVERegSet());
1385}
1386
1387Status NativeRegisterContextLinux_arm64::ReadSMEControl() {
1388 // The real register is SVCR and is accessible from EL0. However we don't want
1389 // to have to JIT code into the target process so we'll just recreate it using
1390 // what we know from ptrace.
1391
1392 // Bit 0 indicates whether streaming mode is active.
1393 m_sme_pseudo_regs.ctrl_reg = m_sve_state == SVEState::Streaming;
1394
1395 // Bit 1 indicates whether the array storage is active.
1396 // It is active if we can read the header and the size field tells us that
1397 // there is register data following it.
1398 Status error = ReadZAHeader();
1399 if (error.Success() && (m_za_header.size > sizeof(m_za_header)))
1400 m_sme_pseudo_regs.ctrl_reg |= 2;
1401
1402 return error;
1403}
1404
1405Status NativeRegisterContextLinux_arm64::ReadMTEControl() {
1406 Status error;
1407
1408 if (m_mte_ctrl_is_valid)
1409 return error;
1410
1411 struct iovec ioVec;
1412 ioVec.iov_base = GetMTEControl();
1413 ioVec.iov_len = GetMTEControlSize();
1414
1415 error = ReadRegisterSet(&ioVec, GetMTEControlSize(), NT_ARM_TAGGED_ADDR_CTRL);
1416
1417 if (error.Success())
1418 m_mte_ctrl_is_valid = true;
1419
1420 return error;
1421}
1422
1423Status NativeRegisterContextLinux_arm64::WriteMTEControl() {
1424 Status error;
1425
1426 error = ReadMTEControl();
1427 if (error.Fail())
1428 return error;
1429
1430 struct iovec ioVec;
1431 ioVec.iov_base = GetMTEControl();
1432 ioVec.iov_len = GetMTEControlSize();
1433
1434 m_mte_ctrl_is_valid = false;
1435
1436 return WriteRegisterSet(&ioVec, GetMTEControlSize(), NT_ARM_TAGGED_ADDR_CTRL);
1437}
1438
1439Status NativeRegisterContextLinux_arm64::ReadTLS() {
1440 Status error;
1441
1442 if (m_tls_is_valid)
1443 return error;
1444
1445 struct iovec ioVec;
1446 ioVec.iov_base = GetTLSBuffer();
1447 ioVec.iov_len = GetTLSBufferSize();
1448
1449 error = ReadRegisterSet(&ioVec, GetTLSBufferSize(), NT_ARM_TLS);
1450
1451 if (error.Success())
1452 m_tls_is_valid = true;
1453
1454 return error;
1455}
1456
1457Status NativeRegisterContextLinux_arm64::WriteTLS() {
1458 Status error;
1459
1460 error = ReadTLS();
1461 if (error.Fail())
1462 return error;
1463
1464 struct iovec ioVec;
1465 ioVec.iov_base = GetTLSBuffer();
1466 ioVec.iov_len = GetTLSBufferSize();
1467
1468 m_tls_is_valid = false;
1469
1470 return WriteRegisterSet(&ioVec, GetTLSBufferSize(), NT_ARM_TLS);
1471}
1472
1473Status NativeRegisterContextLinux_arm64::ReadGCS() {
1474 Status error;
1475
1476 if (m_gcs_is_valid)
1477 return error;
1478
1479 struct iovec ioVec;
1480 ioVec.iov_base = GetGCSBuffer();
1481 ioVec.iov_len = GetGCSBufferSize();
1482
1483 error = ReadRegisterSet(&ioVec, GetGCSBufferSize(), NT_ARM_GCS);
1484
1485 if (error.Success())
1486 m_gcs_is_valid = true;
1487
1488 return error;
1489}
1490
1491Status NativeRegisterContextLinux_arm64::WriteGCS() {
1492 Status error;
1493
1494 error = ReadGCS();
1495 if (error.Fail())
1496 return error;
1497
1498 struct iovec ioVec;
1499 ioVec.iov_base = GetGCSBuffer();
1500 ioVec.iov_len = GetGCSBufferSize();
1501
1502 m_gcs_is_valid = false;
1503
1504 return WriteRegisterSet(&ioVec, GetGCSBufferSize(), NT_ARM_GCS);
1505}
1506
1507Status NativeRegisterContextLinux_arm64::ReadZAHeader() {
1508 Status error;
1509
1510 if (m_za_header_is_valid)
1511 return error;
1512
1513 struct iovec ioVec;
1514 ioVec.iov_base = GetZAHeader();
1515 ioVec.iov_len = GetZAHeaderSize();
1516
1517 error = ReadRegisterSet(&ioVec, GetZAHeaderSize(), NT_ARM_ZA);
1518
1519 if (error.Success())
1520 m_za_header_is_valid = true;
1521
1522 return error;
1523}
1524
1525Status NativeRegisterContextLinux_arm64::ReadZA() {
1526 Status error;
1527
1528 if (m_za_buffer_is_valid)
1529 return error;
1530
1531 struct iovec ioVec;
1532 ioVec.iov_base = GetZABuffer();
1533 ioVec.iov_len = GetZABufferSize();
1534
1535 error = ReadRegisterSet(&ioVec, GetZABufferSize(), NT_ARM_ZA);
1536
1537 if (error.Success())
1538 m_za_buffer_is_valid = true;
1539
1540 return error;
1541}
1542
1543Status NativeRegisterContextLinux_arm64::WriteZA() {
1544 // Note that because the ZA ptrace payload contains the header also, this
1545 // method will write both. This is done because writing only the header
1546 // will disable ZA, even if .size in the header is correct for an enabled ZA.
1547 Status error;
1548
1549 error = ReadZA();
1550 if (error.Fail())
1551 return error;
1552
1553 struct iovec ioVec;
1554 ioVec.iov_base = GetZABuffer();
1555 ioVec.iov_len = GetZABufferSize();
1556
1557 m_za_buffer_is_valid = false;
1558 m_za_header_is_valid = false;
1559 // Writing to ZA may enable ZA, which means ZT0 may change too.
1560 m_zt_buffer_is_valid = false;
1561
1562 return WriteRegisterSet(&ioVec, GetZABufferSize(), NT_ARM_ZA);
1563}
1564
1565Status NativeRegisterContextLinux_arm64::ReadZT() {
1566 Status error;
1567
1568 if (m_zt_buffer_is_valid)
1569 return error;
1570
1571 struct iovec ioVec;
1572 ioVec.iov_base = GetZTBuffer();
1573 ioVec.iov_len = GetZTBufferSize();
1574
1575 error = ReadRegisterSet(&ioVec, GetZTBufferSize(), NT_ARM_ZT);
1576 m_zt_buffer_is_valid = error.Success();
1577
1578 return error;
1579}
1580
1581Status NativeRegisterContextLinux_arm64::WriteZT() {
1582 Status error;
1583
1584 error = ReadZT();
1585 if (error.Fail())
1586 return error;
1587
1588 struct iovec ioVec;
1589 ioVec.iov_base = GetZTBuffer();
1590 ioVec.iov_len = GetZTBufferSize();
1591
1592 m_zt_buffer_is_valid = false;
1593 // Writing to an inactive ZT0 will enable ZA as well, which invalidates our
1594 // current copy of it.
1595 m_za_buffer_is_valid = false;
1596 m_za_header_is_valid = false;
1597
1598 return WriteRegisterSet(&ioVec, GetZTBufferSize(), NT_ARM_ZT);
1599}
1600
1601Status NativeRegisterContextLinux_arm64::ReadFPMR() {
1602 Status error;
1603
1604 if (m_fpmr_is_valid)
1605 return error;
1606
1607 struct iovec ioVec;
1608 ioVec.iov_base = GetFPMRBuffer();
1609 ioVec.iov_len = GetFPMRBufferSize();
1610
1611 error = ReadRegisterSet(&ioVec, GetFPMRBufferSize(), NT_ARM_FPMR);
1612
1613 if (error.Success())
1614 m_fpmr_is_valid = true;
1615
1616 return error;
1617}
1618
1619Status NativeRegisterContextLinux_arm64::WriteFPMR() {
1620 Status error;
1621
1622 error = ReadFPMR();
1623 if (error.Fail())
1624 return error;
1625
1626 struct iovec ioVec;
1627 ioVec.iov_base = GetFPMRBuffer();
1628 ioVec.iov_len = GetFPMRBufferSize();
1629
1630 m_fpmr_is_valid = false;
1631
1632 return WriteRegisterSet(&ioVec, GetFPMRBufferSize(), NT_ARM_FPMR);
1633}
1634
1635void NativeRegisterContextLinux_arm64::ConfigureRegisterContext() {
1636 // ConfigureRegisterContext gets called from InvalidateAllRegisters
1637 // on every stop and configures SVE vector length and whether we are in
1638 // streaming SVE mode.
1639 // If m_sve_state is set to SVEState::Disabled on first stop, code below will
1640 // be deemed non operational for the lifetime of current process.
1641 if (!m_sve_header_is_valid && m_sve_state != SVEState::Disabled) {
1642 // If we have SVE we may also have the SVE streaming mode that SME added.
1643 // We can read the header of either mode, but only the active mode will
1644 // have valid register data.
1645
1646 // Check whether SME is present and the streaming SVE mode is active.
1647 m_sve_header_is_valid = false;
1648 m_sve_buffer_is_valid = false;
1649 m_sve_state = SVEState::Streaming;
1650 Status error = ReadSVEHeader();
1651
1652 // Streaming mode is active if the header has the SVE active flag set.
1653 if (!(error.Success() && ((m_sve_header.flags & sve::ptrace_regs_mask) ==
1654 sve::ptrace_regs_sve))) {
1655 // Non-streaming might be active instead.
1656 m_sve_header_is_valid = false;
1657 m_sve_buffer_is_valid = false;
1658 m_sve_state = SVEState::Full;
1659 error = ReadSVEHeader();
1660 if (error.Success()) {
1661 // If SVE is enabled thread can switch between SVEState::FPSIMD and
1662 // SVEState::Full on every stop.
1663 if ((m_sve_header.flags & sve::ptrace_regs_mask) ==
1664 sve::ptrace_regs_fpsimd)
1665 m_sve_state = SVEState::FPSIMD;
1666 // Else we are in SVEState::Full.
1667 } else {
1668 m_sve_state = SVEState::Disabled;
1669 }
1670 }
1671
1672 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::FPSIMD ||
1673 m_sve_state == SVEState::Streaming) {
1674 // On every stop we configure SVE vector length by calling
1675 // ConfigureVectorLengthSVE regardless of current SVEState of this thread.
1676 uint32_t vq = RegisterInfoPOSIX_arm64::eVectorQuadwordAArch64SVE;
1677 if (sve::vl_valid(m_sve_header.vl))
1678 vq = sve::vq_from_vl(m_sve_header.vl);
1679
1680 GetRegisterInfo().ConfigureVectorLengthSVE(vq);
1681 m_sve_ptrace_payload.resize(sve::PTraceSize(vq, sve::ptrace_regs_sve));
1682 }
1683 }
1684
1685 if (!m_za_header_is_valid) {
1686 Status error = ReadZAHeader();
1687 if (error.Success()) {
1688 uint32_t vq = RegisterInfoPOSIX_arm64::eVectorQuadwordAArch64SVE;
1689 if (sve::vl_valid(m_za_header.vl))
1690 vq = sve::vq_from_vl(m_za_header.vl);
1691
1692 GetRegisterInfo().ConfigureVectorLengthZA(vq);
1693 m_za_ptrace_payload.resize(m_za_header.size);
1694 m_za_buffer_is_valid = false;
1695 }
1696 }
1697}
1698
1699uint32_t NativeRegisterContextLinux_arm64::CalculateFprOffset(
1700 const RegisterInfo *reg_info) const {
1701 return reg_info->byte_offset - GetGPRSize();
1702}
1703
1704uint32_t NativeRegisterContextLinux_arm64::CalculateSVEOffset(
1705 const RegisterInfo *reg_info) const {
1706 // Start of Z0 data is after GPRs plus 8 bytes of vg register
1707 uint32_t sve_reg_offset = LLDB_INVALID_INDEX32;
1708 if (m_sve_state == SVEState::FPSIMD) {
1709 const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];
1710 sve_reg_offset = sve::ptrace_fpsimd_offset +
1711 (reg - GetRegisterInfo().GetRegNumSVEZ0()) * 16;
1712 // Between non-streaming and streaming mode, the layout is identical.
1713 } else if (m_sve_state == SVEState::Full ||
1714 m_sve_state == SVEState::Streaming) {
1715 uint32_t sve_z0_offset = GetGPRSize() + 16;
1716 sve_reg_offset =
1717 sve::SigRegsOffset() + reg_info->byte_offset - sve_z0_offset;
1718 }
1719 return sve_reg_offset;
1720}
1721
1722Status NativeRegisterContextLinux_arm64::ReadSMESVG() {
1723 // This register is the streaming vector length, so we will get it from
1724 // NT_ARM_ZA regardless of the current streaming mode.
1725 Status error = ReadZAHeader();
1726 if (error.Success())
1727 m_sme_pseudo_regs.svg_reg = m_za_header.vl / 8;
1728
1729 return error;
1730}
1731
1732std::vector<uint32_t> NativeRegisterContextLinux_arm64::GetExpeditedRegisters(
1733 ExpeditedRegs expType) const {
1734 std::vector<uint32_t> expedited_reg_nums =
1735 NativeRegisterContext::GetExpeditedRegisters(expType);
1736 // SVE, non-streaming vector length.
1737 if (m_sve_state == SVEState::FPSIMD || m_sve_state == SVEState::Full)
1738 expedited_reg_nums.push_back(GetRegisterInfo().GetRegNumSVEVG());
1739 // SME, streaming vector length. This is used by the ZA register which is
1740 // present even when streaming mode is not enabled.
1741 if (GetRegisterInfo().IsSSVEPresent())
1742 expedited_reg_nums.push_back(GetRegisterInfo().GetRegNumSMESVG());
1743
1744 return expedited_reg_nums;
1745}
1746
1747llvm::Expected<NativeRegisterContextLinux::MemoryTaggingDetails>
1748NativeRegisterContextLinux_arm64::GetMemoryTaggingDetails(int32_t type) {
1749 if (type == MemoryTagManagerAArch64MTE::eMTE_allocation) {
1750 return MemoryTaggingDetails{std::make_unique<MemoryTagManagerAArch64MTE>(),
1751 PTRACE_PEEKMTETAGS, PTRACE_POKEMTETAGS};
1752 }
1753
1754 return llvm::createStringError(llvm::inconvertibleErrorCode(),
1755 "Unknown AArch64 memory tag type %d", type);
1756}
1757
1758lldb::addr_t NativeRegisterContextLinux_arm64::FixWatchpointHitAddress(
1759 lldb::addr_t hit_addr) {
1760 // Linux configures user-space virtual addresses with top byte ignored.
1761 // We set default value of mask such that top byte is masked out.
1762 lldb::addr_t mask = ~((1ULL << 56) - 1);
1763
1764 // Try to read pointer authentication data_mask register and calculate a
1765 // consolidated data address mask after ignoring the top byte.
1766 if (ReadPAuthMask().Success())
1767 mask |= m_pac_mask.data_mask;
1768
1769 return hit_addr & ~mask;
1770 ;
1771}
1772
1773#endif // defined (__arm64__) || defined (__aarch64__)
1774

source code of lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp