1//===-- NativeRegisterContextLinux_arm64.cpp ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#if defined(__arm64__) || defined(__aarch64__)
10
11#include "NativeRegisterContextLinux_arm.h"
12#include "NativeRegisterContextLinux_arm64.h"
13
14#include "lldb/Host/HostInfo.h"
15#include "lldb/Host/common/NativeProcessProtocol.h"
16#include "lldb/Host/linux/Ptrace.h"
17#include "lldb/Utility/DataBufferHeap.h"
18#include "lldb/Utility/Log.h"
19#include "lldb/Utility/RegisterValue.h"
20#include "lldb/Utility/Status.h"
21
22#include "Plugins/Process/Linux/NativeProcessLinux.h"
23#include "Plugins/Process/Linux/Procfs.h"
24#include "Plugins/Process/POSIX/ProcessPOSIXLog.h"
25#include "Plugins/Process/Utility/MemoryTagManagerAArch64MTE.h"
26#include "Plugins/Process/Utility/RegisterFlagsLinux_arm64.h"
27#include "Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h"
28
29// System includes - They have to be included after framework includes because
30// they define some macros which collide with variable names in other modules
31#include <sys/uio.h>
32// NT_PRSTATUS and NT_FPREGSET definition
33#include <elf.h>
34#include <mutex>
35#include <optional>
36
37#ifndef NT_ARM_SVE
38#define NT_ARM_SVE 0x405 /* ARM Scalable Vector Extension */
39#endif
40
41#ifndef NT_ARM_SSVE
42#define NT_ARM_SSVE \
43 0x40b /* ARM Scalable Matrix Extension, Streaming SVE mode */
44#endif
45
46#ifndef NT_ARM_ZA
47#define NT_ARM_ZA 0x40c /* ARM Scalable Matrix Extension, Array Storage */
48#endif
49
50#ifndef NT_ARM_ZT
51#define NT_ARM_ZT \
52 0x40d /* ARM Scalable Matrix Extension 2, lookup table register */
53#endif
54
55#ifndef NT_ARM_PAC_MASK
56#define NT_ARM_PAC_MASK 0x406 /* Pointer authentication code masks */
57#endif
58
59#ifndef NT_ARM_TAGGED_ADDR_CTRL
60#define NT_ARM_TAGGED_ADDR_CTRL 0x409 /* Tagged address control register */
61#endif
62
63#define HWCAP_PACA (1 << 30)
64
65#define HWCAP2_MTE (1 << 18)
66
67using namespace lldb;
68using namespace lldb_private;
69using namespace lldb_private::process_linux;
70
71// A NativeRegisterContext is constructed per thread, but all threads' registers
72// will contain the same fields. Therefore this mutex prevents each instance
73// competing with the other, and subsequent instances from having to detect the
74// fields all over again.
75static std::mutex g_register_flags_mutex;
76static LinuxArm64RegisterFlags g_register_flags;
77
78std::unique_ptr<NativeRegisterContextLinux>
79NativeRegisterContextLinux::CreateHostNativeRegisterContextLinux(
80 const ArchSpec &target_arch, NativeThreadLinux &native_thread) {
81 switch (target_arch.GetMachine()) {
82 case llvm::Triple::arm:
83 return std::make_unique<NativeRegisterContextLinux_arm>(target_arch,
84 native_thread);
85 case llvm::Triple::aarch64: {
86 // Configure register sets supported by this AArch64 target.
87 // Read SVE header to check for SVE support.
88 struct sve::user_sve_header sve_header;
89 struct iovec ioVec;
90 ioVec.iov_base = &sve_header;
91 ioVec.iov_len = sizeof(sve_header);
92 unsigned int regset = NT_ARM_SVE;
93
94 Flags opt_regsets;
95 if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
96 native_thread.GetID(), &regset,
97 &ioVec, sizeof(sve_header))
98 .Success()) {
99 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskSVE);
100
101 // We may also have the Scalable Matrix Extension (SME) which adds a
102 // streaming SVE mode.
103 ioVec.iov_len = sizeof(sve_header);
104 regset = NT_ARM_SSVE;
105 if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
106 native_thread.GetID(), &regset,
107 &ioVec, sizeof(sve_header))
108 .Success())
109 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskSSVE);
110 }
111
112 sve::user_za_header za_header;
113 ioVec.iov_base = &za_header;
114 ioVec.iov_len = sizeof(za_header);
115 regset = NT_ARM_ZA;
116 if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
117 native_thread.GetID(), &regset,
118 &ioVec, sizeof(za_header))
119 .Success())
120 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskZA);
121
122 // SME's ZT0 is a 512 bit register.
123 std::array<uint8_t, 64> zt_reg;
124 ioVec.iov_base = zt_reg.data();
125 ioVec.iov_len = zt_reg.size();
126 regset = NT_ARM_ZT;
127 if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
128 native_thread.GetID(), &regset,
129 &ioVec, zt_reg.size())
130 .Success())
131 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskZT);
132
133 NativeProcessLinux &process = native_thread.GetProcess();
134
135 std::optional<uint64_t> auxv_at_hwcap =
136 process.GetAuxValue(AuxVector::AUXV_AT_HWCAP);
137 if (auxv_at_hwcap && (*auxv_at_hwcap & HWCAP_PACA))
138 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskPAuth);
139
140 std::optional<uint64_t> auxv_at_hwcap2 =
141 process.GetAuxValue(AuxVector::AUXV_AT_HWCAP2);
142 if (auxv_at_hwcap2 && (*auxv_at_hwcap2 & HWCAP2_MTE))
143 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskMTE);
144
145 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskTLS);
146
147 std::lock_guard<std::mutex> lock(g_register_flags_mutex);
148 if (!g_register_flags.HasDetected())
149 g_register_flags.DetectFields(auxv_at_hwcap.value_or(0),
150 auxv_at_hwcap2.value_or(0));
151
152 auto register_info_up =
153 std::make_unique<RegisterInfoPOSIX_arm64>(target_arch, opt_regsets);
154 return std::make_unique<NativeRegisterContextLinux_arm64>(
155 target_arch, native_thread, std::move(register_info_up));
156 }
157 default:
158 llvm_unreachable("have no register context for architecture");
159 }
160}
161
162llvm::Expected<ArchSpec>
163NativeRegisterContextLinux::DetermineArchitecture(lldb::tid_t tid) {
164 return DetermineArchitectureViaGPR(
165 tid, RegisterInfoPOSIX_arm64::GetGPRSizeStatic());
166}
167
168NativeRegisterContextLinux_arm64::NativeRegisterContextLinux_arm64(
169 const ArchSpec &target_arch, NativeThreadProtocol &native_thread,
170 std::unique_ptr<RegisterInfoPOSIX_arm64> register_info_up)
171 : NativeRegisterContextRegisterInfo(native_thread,
172 register_info_up.release()),
173 NativeRegisterContextLinux(native_thread) {
174 g_register_flags.UpdateRegisterInfo(
175 GetRegisterInfoInterface().GetRegisterInfo(),
176 GetRegisterInfoInterface().GetRegisterCount());
177
178 ::memset(&m_fpr, 0, sizeof(m_fpr));
179 ::memset(&m_gpr_arm64, 0, sizeof(m_gpr_arm64));
180 ::memset(&m_hwp_regs, 0, sizeof(m_hwp_regs));
181 ::memset(&m_hbp_regs, 0, sizeof(m_hbp_regs));
182 ::memset(&m_sve_header, 0, sizeof(m_sve_header));
183 ::memset(&m_pac_mask, 0, sizeof(m_pac_mask));
184 ::memset(&m_tls_regs, 0, sizeof(m_tls_regs));
185 ::memset(&m_sme_pseudo_regs, 0, sizeof(m_sme_pseudo_regs));
186 std::fill(m_zt_reg.begin(), m_zt_reg.end(), 0);
187
188 m_mte_ctrl_reg = 0;
189
190 // 16 is just a maximum value, query hardware for actual watchpoint count
191 m_max_hwp_supported = 16;
192 m_max_hbp_supported = 16;
193
194 m_refresh_hwdebug_info = true;
195
196 m_gpr_is_valid = false;
197 m_fpu_is_valid = false;
198 m_sve_buffer_is_valid = false;
199 m_sve_header_is_valid = false;
200 m_pac_mask_is_valid = false;
201 m_mte_ctrl_is_valid = false;
202 m_tls_is_valid = false;
203 m_zt_buffer_is_valid = false;
204
205 // SME adds the tpidr2 register
206 m_tls_size = GetRegisterInfo().IsSSVEPresent() ? sizeof(m_tls_regs)
207 : sizeof(m_tls_regs.tpidr_reg);
208
209 if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent())
210 m_sve_state = SVEState::Unknown;
211 else
212 m_sve_state = SVEState::Disabled;
213}
214
215RegisterInfoPOSIX_arm64 &
216NativeRegisterContextLinux_arm64::GetRegisterInfo() const {
217 return static_cast<RegisterInfoPOSIX_arm64 &>(*m_register_info_interface_up);
218}
219
220uint32_t NativeRegisterContextLinux_arm64::GetRegisterSetCount() const {
221 return GetRegisterInfo().GetRegisterSetCount();
222}
223
224const RegisterSet *
225NativeRegisterContextLinux_arm64::GetRegisterSet(uint32_t set_index) const {
226 return GetRegisterInfo().GetRegisterSet(set_index);
227}
228
229uint32_t NativeRegisterContextLinux_arm64::GetUserRegisterCount() const {
230 uint32_t count = 0;
231 for (uint32_t set_index = 0; set_index < GetRegisterSetCount(); ++set_index)
232 count += GetRegisterSet(set_index)->num_registers;
233 return count;
234}
235
236Status
237NativeRegisterContextLinux_arm64::ReadRegister(const RegisterInfo *reg_info,
238 RegisterValue &reg_value) {
239 Status error;
240
241 if (!reg_info) {
242 error.SetErrorString("reg_info NULL");
243 return error;
244 }
245
246 const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];
247
248 if (reg == LLDB_INVALID_REGNUM)
249 return Status("no lldb regnum for %s", reg_info && reg_info->name
250 ? reg_info->name
251 : "<unknown register>");
252
253 uint8_t *src;
254 uint32_t offset = LLDB_INVALID_INDEX32;
255 uint64_t sve_vg;
256 std::vector<uint8_t> sve_reg_non_live;
257
258 if (IsGPR(reg)) {
259 error = ReadGPR();
260 if (error.Fail())
261 return error;
262
263 offset = reg_info->byte_offset;
264 assert(offset < GetGPRSize());
265 src = (uint8_t *)GetGPRBuffer() + offset;
266
267 } else if (IsFPR(reg)) {
268 if (m_sve_state == SVEState::Disabled) {
269 // SVE is disabled take legacy route for FPU register access
270 error = ReadFPR();
271 if (error.Fail())
272 return error;
273
274 offset = CalculateFprOffset(reg_info);
275 assert(offset < GetFPRSize());
276 src = (uint8_t *)GetFPRBuffer() + offset;
277 } else {
278 // SVE or SSVE enabled, we will read and cache SVE ptrace data.
279 // In SIMD or Full mode, the data comes from the SVE regset. In streaming
280 // mode it comes from the streaming SVE regset.
281 error = ReadAllSVE();
282 if (error.Fail())
283 return error;
284
285 // FPSR and FPCR will be located right after Z registers in
286 // SVEState::FPSIMD while in SVEState::Full or SVEState::Streaming they
287 // will be located at the end of register data after an alignment
288 // correction based on currently selected vector length.
289 uint32_t sve_reg_num = LLDB_INVALID_REGNUM;
290 if (reg == GetRegisterInfo().GetRegNumFPSR()) {
291 sve_reg_num = reg;
292 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
293 offset = sve::PTraceFPSROffset(sve::vq_from_vl(m_sve_header.vl));
294 else if (m_sve_state == SVEState::FPSIMD)
295 offset = sve::ptrace_fpsimd_offset + (32 * 16);
296 } else if (reg == GetRegisterInfo().GetRegNumFPCR()) {
297 sve_reg_num = reg;
298 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
299 offset = sve::PTraceFPCROffset(sve::vq_from_vl(m_sve_header.vl));
300 else if (m_sve_state == SVEState::FPSIMD)
301 offset = sve::ptrace_fpsimd_offset + (32 * 16) + 4;
302 } else {
303 // Extract SVE Z register value register number for this reg_info
304 if (reg_info->value_regs &&
305 reg_info->value_regs[0] != LLDB_INVALID_REGNUM)
306 sve_reg_num = reg_info->value_regs[0];
307 offset = CalculateSVEOffset(GetRegisterInfoAtIndex(sve_reg_num));
308 }
309
310 assert(offset < GetSVEBufferSize());
311 src = (uint8_t *)GetSVEBuffer() + offset;
312 }
313 } else if (IsTLS(reg)) {
314 error = ReadTLS();
315 if (error.Fail())
316 return error;
317
318 offset = reg_info->byte_offset - GetRegisterInfo().GetTLSOffset();
319 assert(offset < GetTLSBufferSize());
320 src = (uint8_t *)GetTLSBuffer() + offset;
321 } else if (IsSVE(reg)) {
322 if (m_sve_state == SVEState::Disabled || m_sve_state == SVEState::Unknown)
323 return Status("SVE disabled or not supported");
324
325 if (GetRegisterInfo().IsSVERegVG(reg)) {
326 sve_vg = GetSVERegVG();
327 src = (uint8_t *)&sve_vg;
328 } else {
329 // SVE enabled, we will read and cache SVE ptrace data
330 error = ReadAllSVE();
331 if (error.Fail())
332 return error;
333
334 if (m_sve_state == SVEState::FPSIMD) {
335 // In FPSIMD state SVE payload mirrors legacy fpsimd struct and so
336 // just copy 16 bytes of v register to the start of z register. All
337 // other SVE register will be set to zero.
338 sve_reg_non_live.resize(reg_info->byte_size, 0);
339 src = sve_reg_non_live.data();
340
341 if (GetRegisterInfo().IsSVEZReg(reg)) {
342 offset = CalculateSVEOffset(reg_info);
343 assert(offset < GetSVEBufferSize());
344 ::memcpy(sve_reg_non_live.data(), (uint8_t *)GetSVEBuffer() + offset,
345 16);
346 }
347 } else {
348 offset = CalculateSVEOffset(reg_info);
349 assert(offset < GetSVEBufferSize());
350 src = (uint8_t *)GetSVEBuffer() + offset;
351 }
352 }
353 } else if (IsPAuth(reg)) {
354 error = ReadPAuthMask();
355 if (error.Fail())
356 return error;
357
358 offset = reg_info->byte_offset - GetRegisterInfo().GetPAuthOffset();
359 assert(offset < GetPACMaskSize());
360 src = (uint8_t *)GetPACMask() + offset;
361 } else if (IsMTE(reg)) {
362 error = ReadMTEControl();
363 if (error.Fail())
364 return error;
365
366 offset = reg_info->byte_offset - GetRegisterInfo().GetMTEOffset();
367 assert(offset < GetMTEControlSize());
368 src = (uint8_t *)GetMTEControl() + offset;
369 } else if (IsSME(reg)) {
370 if (GetRegisterInfo().IsSMERegZA(reg)) {
371 error = ReadZAHeader();
372 if (error.Fail())
373 return error;
374
375 // If there is only a header and no registers, ZA is inactive. Read as 0
376 // in this case.
377 if (m_za_header.size == sizeof(m_za_header)) {
378 // This will get reconfigured/reset later, so we are safe to use it.
379 // ZA is a square of VL * VL and the ptrace buffer also includes the
380 // header itself.
381 m_za_ptrace_payload.resize(((m_za_header.vl) * (m_za_header.vl)) +
382 GetZAHeaderSize());
383 std::fill(m_za_ptrace_payload.begin(), m_za_ptrace_payload.end(), 0);
384 } else {
385 // ZA is active, read the real register.
386 error = ReadZA();
387 if (error.Fail())
388 return error;
389 }
390
391 // ZA is part of the SME set but uses a seperate member buffer for
392 // storage. Therefore its effective byte offset is always 0 even if it
393 // isn't 0 within the SME register set.
394 src = (uint8_t *)GetZABuffer() + GetZAHeaderSize();
395 } else if (GetRegisterInfo().IsSMERegZT(reg)) {
396 // Unlike ZA, the kernel will return register data for ZT0 when ZA is not
397 // enabled. This data will be all 0s so we don't have to invent anything
398 // like we did for ZA.
399 error = ReadZT();
400 if (error.Fail())
401 return error;
402
403 src = (uint8_t *)GetZTBuffer();
404 } else {
405 error = ReadSMESVG();
406 if (error.Fail())
407 return error;
408
409 // This is a psuedo so it never fails.
410 ReadSMEControl();
411
412 offset = reg_info->byte_offset - GetRegisterInfo().GetSMEOffset();
413 assert(offset < GetSMEPseudoBufferSize());
414 src = (uint8_t *)GetSMEPseudoBuffer() + offset;
415 }
416 } else
417 return Status("failed - register wasn't recognized to be a GPR or an FPR, "
418 "write strategy unknown");
419
420 reg_value.SetFromMemoryData(*reg_info, src, reg_info->byte_size,
421 eByteOrderLittle, error);
422
423 return error;
424}
425
426Status NativeRegisterContextLinux_arm64::WriteRegister(
427 const RegisterInfo *reg_info, const RegisterValue &reg_value) {
428 Status error;
429
430 if (!reg_info)
431 return Status("reg_info NULL");
432
433 const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];
434
435 if (reg == LLDB_INVALID_REGNUM)
436 return Status("no lldb regnum for %s", reg_info && reg_info->name
437 ? reg_info->name
438 : "<unknown register>");
439
440 uint8_t *dst;
441 uint32_t offset = LLDB_INVALID_INDEX32;
442 std::vector<uint8_t> sve_reg_non_live;
443
444 if (IsGPR(reg)) {
445 error = ReadGPR();
446 if (error.Fail())
447 return error;
448
449 assert(reg_info->byte_offset < GetGPRSize());
450 dst = (uint8_t *)GetGPRBuffer() + reg_info->byte_offset;
451 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
452
453 return WriteGPR();
454 } else if (IsFPR(reg)) {
455 if (m_sve_state == SVEState::Disabled) {
456 // SVE is disabled take legacy route for FPU register access
457 error = ReadFPR();
458 if (error.Fail())
459 return error;
460
461 offset = CalculateFprOffset(reg_info);
462 assert(offset < GetFPRSize());
463 dst = (uint8_t *)GetFPRBuffer() + offset;
464 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
465
466 return WriteFPR();
467 } else {
468 // SVE enabled, we will read and cache SVE ptrace data.
469 error = ReadAllSVE();
470 if (error.Fail())
471 return error;
472
473 // FPSR and FPCR will be located right after Z registers in
474 // SVEState::FPSIMD while in SVEState::Full or SVEState::Streaming they
475 // will be located at the end of register data after an alignment
476 // correction based on currently selected vector length.
477 uint32_t sve_reg_num = LLDB_INVALID_REGNUM;
478 if (reg == GetRegisterInfo().GetRegNumFPSR()) {
479 sve_reg_num = reg;
480 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
481 offset = sve::PTraceFPSROffset(sve::vq_from_vl(m_sve_header.vl));
482 else if (m_sve_state == SVEState::FPSIMD)
483 offset = sve::ptrace_fpsimd_offset + (32 * 16);
484 } else if (reg == GetRegisterInfo().GetRegNumFPCR()) {
485 sve_reg_num = reg;
486 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
487 offset = sve::PTraceFPCROffset(sve::vq_from_vl(m_sve_header.vl));
488 else if (m_sve_state == SVEState::FPSIMD)
489 offset = sve::ptrace_fpsimd_offset + (32 * 16) + 4;
490 } else {
491 // Extract SVE Z register value register number for this reg_info
492 if (reg_info->value_regs &&
493 reg_info->value_regs[0] != LLDB_INVALID_REGNUM)
494 sve_reg_num = reg_info->value_regs[0];
495 offset = CalculateSVEOffset(GetRegisterInfoAtIndex(sve_reg_num));
496 }
497
498 assert(offset < GetSVEBufferSize());
499 dst = (uint8_t *)GetSVEBuffer() + offset;
500 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
501 return WriteAllSVE();
502 }
503 } else if (IsSVE(reg)) {
504 if (m_sve_state == SVEState::Disabled || m_sve_state == SVEState::Unknown)
505 return Status("SVE disabled or not supported");
506 else {
507 // Target has SVE enabled, we will read and cache SVE ptrace data
508 error = ReadAllSVE();
509 if (error.Fail())
510 return error;
511
512 if (GetRegisterInfo().IsSVERegVG(reg)) {
513 uint64_t vg_value = reg_value.GetAsUInt64();
514
515 if (sve::vl_valid(vg_value * 8)) {
516 if (m_sve_header_is_valid && vg_value == GetSVERegVG())
517 return error;
518
519 SetSVERegVG(vg_value);
520
521 error = WriteSVEHeader();
522 if (error.Success()) {
523 // Changing VG during streaming mode also changes the size of ZA.
524 if (m_sve_state == SVEState::Streaming)
525 m_za_header_is_valid = false;
526 ConfigureRegisterContext();
527 }
528
529 if (m_sve_header_is_valid && vg_value == GetSVERegVG())
530 return error;
531 }
532
533 return Status("SVE vector length update failed.");
534 }
535
536 // If target supports SVE but currently in FPSIMD mode.
537 if (m_sve_state == SVEState::FPSIMD) {
538 // Here we will check if writing this SVE register enables
539 // SVEState::Full
540 bool set_sve_state_full = false;
541 const uint8_t *reg_bytes = (const uint8_t *)reg_value.GetBytes();
542 if (GetRegisterInfo().IsSVEZReg(reg)) {
543 for (uint32_t i = 16; i < reg_info->byte_size; i++) {
544 if (reg_bytes[i]) {
545 set_sve_state_full = true;
546 break;
547 }
548 }
549 } else if (GetRegisterInfo().IsSVEPReg(reg) ||
550 reg == GetRegisterInfo().GetRegNumSVEFFR()) {
551 for (uint32_t i = 0; i < reg_info->byte_size; i++) {
552 if (reg_bytes[i]) {
553 set_sve_state_full = true;
554 break;
555 }
556 }
557 }
558
559 if (!set_sve_state_full && GetRegisterInfo().IsSVEZReg(reg)) {
560 // We are writing a Z register which is zero beyond 16 bytes so copy
561 // first 16 bytes only as SVE payload mirrors legacy fpsimd structure
562 offset = CalculateSVEOffset(reg_info);
563 assert(offset < GetSVEBufferSize());
564 dst = (uint8_t *)GetSVEBuffer() + offset;
565 ::memcpy(dst, reg_value.GetBytes(), 16);
566
567 return WriteAllSVE();
568 } else
569 return Status("SVE state change operation not supported");
570 } else {
571 offset = CalculateSVEOffset(reg_info);
572 assert(offset < GetSVEBufferSize());
573 dst = (uint8_t *)GetSVEBuffer() + offset;
574 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
575 return WriteAllSVE();
576 }
577 }
578 } else if (IsMTE(reg)) {
579 error = ReadMTEControl();
580 if (error.Fail())
581 return error;
582
583 offset = reg_info->byte_offset - GetRegisterInfo().GetMTEOffset();
584 assert(offset < GetMTEControlSize());
585 dst = (uint8_t *)GetMTEControl() + offset;
586 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
587
588 return WriteMTEControl();
589 } else if (IsTLS(reg)) {
590 error = ReadTLS();
591 if (error.Fail())
592 return error;
593
594 offset = reg_info->byte_offset - GetRegisterInfo().GetTLSOffset();
595 assert(offset < GetTLSBufferSize());
596 dst = (uint8_t *)GetTLSBuffer() + offset;
597 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
598
599 return WriteTLS();
600 } else if (IsSME(reg)) {
601 if (GetRegisterInfo().IsSMERegZA(reg)) {
602 error = ReadZA();
603 if (error.Fail())
604 return error;
605
606 // ZA is part of the SME set but not stored with the other SME registers.
607 // So its byte offset is effectively always 0.
608 dst = (uint8_t *)GetZABuffer() + GetZAHeaderSize();
609 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
610
611 // While this is writing a header that contains a vector length, the only
612 // way to change that is via the vg register. So here we assume the length
613 // will always be the current length and no reconfigure is needed.
614 return WriteZA();
615 } else if (GetRegisterInfo().IsSMERegZT(reg)) {
616 error = ReadZT();
617 if (error.Fail())
618 return error;
619
620 dst = (uint8_t *)GetZTBuffer();
621 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
622
623 return WriteZT();
624 } else
625 return Status("Writing to SVG or SVCR is not supported.");
626 }
627
628 return Status("Failed to write register value");
629}
630
631enum RegisterSetType : uint32_t {
632 GPR,
633 SVE, // Used for SVE and SSVE.
634 FPR, // When there is no SVE, or SVE in FPSIMD mode.
635 // Pointer authentication registers are read only, so not included here.
636 MTE,
637 TLS,
638 SME, // ZA only, because SVCR and SVG are pseudo registers.
639 SME2, // ZT only.
640};
641
642static uint8_t *AddRegisterSetType(uint8_t *dst,
643 RegisterSetType register_set_type) {
644 *(reinterpret_cast<uint32_t *>(dst)) = register_set_type;
645 return dst + sizeof(uint32_t);
646}
647
648static uint8_t *AddSavedRegistersData(uint8_t *dst, void *src, size_t size) {
649 ::memcpy(dst, src, size);
650 return dst + size;
651}
652
653static uint8_t *AddSavedRegisters(uint8_t *dst,
654 enum RegisterSetType register_set_type,
655 void *src, size_t size) {
656 dst = AddRegisterSetType(dst, register_set_type);
657 return AddSavedRegistersData(dst, src, size);
658}
659
660Status
661NativeRegisterContextLinux_arm64::CacheAllRegisters(uint32_t &cached_size) {
662 Status error;
663 cached_size = sizeof(RegisterSetType) + GetGPRBufferSize();
664 error = ReadGPR();
665 if (error.Fail())
666 return error;
667
668 if (GetRegisterInfo().IsZAPresent()) {
669 error = ReadZAHeader();
670 if (error.Fail())
671 return error;
672 // Use header size here because the buffer may contain fake data when ZA is
673 // disabled. We do not want to write this fake data (all 0s) because this
674 // would tell the kernel that we want ZA to become active. Which is the
675 // opposite of what we want in the case where it is currently inactive.
676 cached_size += sizeof(RegisterSetType) + m_za_header.size;
677 // For the same reason, we need to force it to be re-read so that it will
678 // always contain the real header.
679 m_za_buffer_is_valid = false;
680 error = ReadZA();
681 if (error.Fail())
682 return error;
683
684 // We will only be restoring ZT data if ZA is active. As writing to an
685 // inactive ZT enables ZA, which may not be desireable.
686 if (
687 // If we have ZT0, or in other words, if we have SME2.
688 GetRegisterInfo().IsZTPresent() &&
689 // And ZA is active, which means that ZT0 is also active.
690 m_za_header.size > sizeof(m_za_header)) {
691 cached_size += sizeof(RegisterSetType) + GetZTBufferSize();
692 // The kernel handles an inactive ZT0 for us, and it will read as 0s if
693 // inactive (unlike ZA where we fake that behaviour).
694 error = ReadZT();
695 if (error.Fail())
696 return error;
697 }
698 }
699
700 // If SVE is enabled we need not copy FPR separately.
701 if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent()) {
702 // Store mode and register data.
703 cached_size +=
704 sizeof(RegisterSetType) + sizeof(m_sve_state) + GetSVEBufferSize();
705 error = ReadAllSVE();
706 } else {
707 cached_size += sizeof(RegisterSetType) + GetFPRSize();
708 error = ReadFPR();
709 }
710 if (error.Fail())
711 return error;
712
713 if (GetRegisterInfo().IsMTEPresent()) {
714 cached_size += sizeof(RegisterSetType) + GetMTEControlSize();
715 error = ReadMTEControl();
716 if (error.Fail())
717 return error;
718 }
719
720 // tpidr is always present but tpidr2 depends on SME.
721 cached_size += sizeof(RegisterSetType) + GetTLSBufferSize();
722 error = ReadTLS();
723
724 return error;
725}
726
727Status NativeRegisterContextLinux_arm64::ReadAllRegisterValues(
728 lldb::WritableDataBufferSP &data_sp) {
729 // AArch64 register data must contain GPRs and either FPR or SVE registers.
730 // SVE registers can be non-streaming (aka SVE) or streaming (aka SSVE).
731 // Finally an optional MTE register. Pointer Authentication (PAC) registers
732 // are read-only and will be skipped.
733
734 // In order to create register data checkpoint we first read all register
735 // values if not done already and calculate total size of register set data.
736 // We store all register values in data_sp by copying full PTrace data that
737 // corresponds to register sets enabled by current register context.
738
739 uint32_t reg_data_byte_size = 0;
740 Status error = CacheAllRegisters(reg_data_byte_size);
741 if (error.Fail())
742 return error;
743
744 data_sp.reset(new DataBufferHeap(reg_data_byte_size, 0));
745 uint8_t *dst = data_sp->GetBytes();
746
747 dst = AddSavedRegisters(dst, RegisterSetType::GPR, GetGPRBuffer(),
748 GetGPRBufferSize());
749
750 // Streaming SVE and the ZA register both use the streaming vector length.
751 // When you change this, the kernel will invalidate parts of the process
752 // state. Therefore we need a specific order of restoration for each mode, if
753 // we also have ZA to restore.
754 //
755 // Streaming mode enabled, ZA enabled:
756 // * Write streaming registers. This sets SVCR.SM and clears SVCR.ZA.
757 // * Write ZA, this set SVCR.ZA. The register data we provide is written to
758 // ZA.
759 // * Result is SVCR.SM and SVCR.ZA set, with the expected data in both
760 // register sets.
761 //
762 // Streaming mode disabled, ZA enabled:
763 // * Write ZA. This sets SVCR.ZA, and the ZA content. In the majority of cases
764 // the streaming vector length is changing, so the thread is converted into
765 // an FPSIMD thread if it is not already one. This also clears SVCR.SM.
766 // * Write SVE registers, which also clears SVCR.SM but most importantly, puts
767 // us into full SVE mode instead of FPSIMD mode (where the registers are
768 // actually the 128 bit Neon registers).
769 // * Result is we have SVCR.SM = 0, SVCR.ZA = 1 and the expected register
770 // state.
771 //
772 // Restoring in different orders leads to things like the SVE registers being
773 // truncated due to the FPSIMD mode and ZA being disabled or filled with 0s
774 // (disabled and 0s looks the same from inside lldb since we fake the value
775 // when it's disabled).
776 //
777 // For more information on this, look up the uses of the relevant NT_ARM_
778 // constants and the functions vec_set_vector_length, sve_set_common and
779 // za_set in the Linux Kernel.
780
781 if ((m_sve_state != SVEState::Streaming) && GetRegisterInfo().IsZAPresent()) {
782 // Use the header size not the buffer size, as we may be using the buffer
783 // for fake data, which we do not want to write out.
784 assert(m_za_header.size <= GetZABufferSize());
785 dst = AddSavedRegisters(dst, RegisterSetType::SME, GetZABuffer(),
786 m_za_header.size);
787 }
788
789 if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent()) {
790 dst = AddRegisterSetType(dst, RegisterSetType::SVE);
791 *(reinterpret_cast<SVEState *>(dst)) = m_sve_state;
792 dst += sizeof(m_sve_state);
793 dst = AddSavedRegistersData(dst, GetSVEBuffer(), GetSVEBufferSize());
794 } else {
795 dst = AddSavedRegisters(dst, RegisterSetType::FPR, GetFPRBuffer(),
796 GetFPRSize());
797 }
798
799 if ((m_sve_state == SVEState::Streaming) && GetRegisterInfo().IsZAPresent()) {
800 assert(m_za_header.size <= GetZABufferSize());
801 dst = AddSavedRegisters(dst, RegisterSetType::SME, GetZABuffer(),
802 m_za_header.size);
803 }
804
805 // If ZT0 is present and we are going to be restoring an active ZA (which
806 // implies an active ZT0), then restore ZT0 after ZA has been set. This
807 // prevents us enabling ZA accidentally after the restore of ZA disabled it.
808 // If we leave ZA/ZT0 inactive and read ZT0, the kernel returns 0s. Therefore
809 // there's nothing for us to restore if ZA was originally inactive.
810 if (
811 // If we have SME2 and therefore ZT0.
812 GetRegisterInfo().IsZTPresent() &&
813 // And ZA is enabled.
814 m_za_header.size > sizeof(m_za_header))
815 dst = AddSavedRegisters(dst, RegisterSetType::SME2, GetZTBuffer(),
816 GetZTBufferSize());
817
818 if (GetRegisterInfo().IsMTEPresent()) {
819 dst = AddSavedRegisters(dst, RegisterSetType::MTE, GetMTEControl(),
820 GetMTEControlSize());
821 }
822
823 dst = AddSavedRegisters(dst, RegisterSetType::TLS, GetTLSBuffer(),
824 GetTLSBufferSize());
825
826 return error;
827}
828
829static Status RestoreRegisters(void *buffer, const uint8_t **src, size_t len,
830 bool &is_valid, std::function<Status()> writer) {
831 ::memcpy(buffer, *src, len);
832 is_valid = true;
833 *src += len;
834 return writer();
835}
836
837Status NativeRegisterContextLinux_arm64::WriteAllRegisterValues(
838 const lldb::DataBufferSP &data_sp) {
839 // AArch64 register data must contain GPRs, either FPR or SVE registers
840 // (which can be streaming or non-streaming) and optional MTE register.
841 // Pointer Authentication (PAC) registers are read-only and will be skipped.
842
843 // We store all register values in data_sp by copying full PTrace data that
844 // corresponds to register sets enabled by current register context. In order
845 // to restore from register data checkpoint we will first restore GPRs, based
846 // on size of remaining register data either SVE or FPRs should be restored
847 // next. SVE is not enabled if we have register data size less than or equal
848 // to size of GPR + FPR + MTE.
849
850 Status error;
851 if (!data_sp) {
852 error.SetErrorStringWithFormat(
853 "NativeRegisterContextLinux_arm64::%s invalid data_sp provided",
854 __FUNCTION__);
855 return error;
856 }
857
858 const uint8_t *src = data_sp->GetBytes();
859 if (src == nullptr) {
860 error.SetErrorStringWithFormat("NativeRegisterContextLinux_arm64::%s "
861 "DataBuffer::GetBytes() returned a null "
862 "pointer",
863 __FUNCTION__);
864 return error;
865 }
866
867 uint64_t reg_data_min_size =
868 GetGPRBufferSize() + GetFPRSize() + 2 * (sizeof(RegisterSetType));
869 if (data_sp->GetByteSize() < reg_data_min_size) {
870 error.SetErrorStringWithFormat(
871 "NativeRegisterContextLinux_arm64::%s data_sp contained insufficient "
872 "register data bytes, expected at least %" PRIu64 ", actual %" PRIu64,
873 __FUNCTION__, reg_data_min_size, data_sp->GetByteSize());
874 return error;
875 }
876
877 const uint8_t *end = src + data_sp->GetByteSize();
878 while (src < end) {
879 const RegisterSetType kind =
880 *reinterpret_cast<const RegisterSetType *>(src);
881 src += sizeof(RegisterSetType);
882
883 switch (kind) {
884 case RegisterSetType::GPR:
885 error = RestoreRegisters(
886 GetGPRBuffer(), &src, GetGPRBufferSize(), m_gpr_is_valid,
887 std::bind(&NativeRegisterContextLinux_arm64::WriteGPR, this));
888 break;
889 case RegisterSetType::SVE:
890 // Restore to the correct mode, streaming or not.
891 m_sve_state = static_cast<SVEState>(*src);
892 src += sizeof(m_sve_state);
893
894 // First write SVE header. We do not use RestoreRegisters because we do
895 // not want src to be modified yet.
896 ::memcpy(GetSVEHeader(), src, GetSVEHeaderSize());
897 if (!sve::vl_valid(m_sve_header.vl)) {
898 m_sve_header_is_valid = false;
899 error.SetErrorStringWithFormat("NativeRegisterContextLinux_arm64::%s "
900 "Invalid SVE header in data_sp",
901 __FUNCTION__);
902 return error;
903 }
904 m_sve_header_is_valid = true;
905 error = WriteSVEHeader();
906 if (error.Fail())
907 return error;
908
909 // SVE header has been written configure SVE vector length if needed.
910 // This could change ZA data too, but that will be restored again later
911 // anyway.
912 ConfigureRegisterContext();
913
914 // Write header and register data, incrementing src this time.
915 error = RestoreRegisters(
916 GetSVEBuffer(), &src, GetSVEBufferSize(), m_sve_buffer_is_valid,
917 std::bind(&NativeRegisterContextLinux_arm64::WriteAllSVE, this));
918 break;
919 case RegisterSetType::FPR:
920 error = RestoreRegisters(
921 GetFPRBuffer(), &src, GetFPRSize(), m_fpu_is_valid,
922 std::bind(&NativeRegisterContextLinux_arm64::WriteFPR, this));
923 break;
924 case RegisterSetType::MTE:
925 error = RestoreRegisters(
926 GetMTEControl(), &src, GetMTEControlSize(), m_mte_ctrl_is_valid,
927 std::bind(&NativeRegisterContextLinux_arm64::WriteMTEControl, this));
928 break;
929 case RegisterSetType::TLS:
930 error = RestoreRegisters(
931 GetTLSBuffer(), &src, GetTLSBufferSize(), m_tls_is_valid,
932 std::bind(&NativeRegisterContextLinux_arm64::WriteTLS, this));
933 break;
934 case RegisterSetType::SME:
935 // To enable or disable ZA you write the regset with or without register
936 // data. The kernel detects this by looking at the ioVec's length, not the
937 // ZA header size you pass in. Therefore we must write header and register
938 // data (if present) in one go every time. Read the header only first just
939 // to get the size.
940 ::memcpy(GetZAHeader(), src, GetZAHeaderSize());
941 // Read the header and register data. Can't use the buffer size here, it
942 // may be incorrect due to being filled with dummy data previously. Resize
943 // this so WriteZA uses the correct size.
944 m_za_ptrace_payload.resize(m_za_header.size);
945 ::memcpy(GetZABuffer(), src, GetZABufferSize());
946 m_za_buffer_is_valid = true;
947
948 error = WriteZA();
949 if (error.Fail())
950 return error;
951
952 // Update size of ZA, which resizes the ptrace payload potentially
953 // trashing our copy of the data we just wrote.
954 ConfigureRegisterContext();
955
956 // ZA buffer now has proper size, read back the data we wrote above, from
957 // ptrace.
958 error = ReadZA();
959 src += GetZABufferSize();
960 break;
961 case RegisterSetType::SME2:
962 // Doing this would activate an inactive ZA, however we will only get here
963 // if the state we are restoring had an active ZA. Restoring ZT0 will
964 // always come after restoring ZA.
965 error = RestoreRegisters(
966 GetZTBuffer(), &src, GetZTBufferSize(), m_zt_buffer_is_valid,
967 std::bind(&NativeRegisterContextLinux_arm64::WriteZT, this));
968 break;
969 }
970
971 if (error.Fail())
972 return error;
973 }
974
975 return error;
976}
977
978bool NativeRegisterContextLinux_arm64::IsGPR(unsigned reg) const {
979 if (GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg) ==
980 RegisterInfoPOSIX_arm64::GPRegSet)
981 return true;
982 return false;
983}
984
985bool NativeRegisterContextLinux_arm64::IsFPR(unsigned reg) const {
986 if (GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg) ==
987 RegisterInfoPOSIX_arm64::FPRegSet)
988 return true;
989 return false;
990}
991
992bool NativeRegisterContextLinux_arm64::IsSVE(unsigned reg) const {
993 return GetRegisterInfo().IsSVEReg(reg);
994}
995
996bool NativeRegisterContextLinux_arm64::IsSME(unsigned reg) const {
997 return GetRegisterInfo().IsSMEReg(reg);
998}
999
1000bool NativeRegisterContextLinux_arm64::IsPAuth(unsigned reg) const {
1001 return GetRegisterInfo().IsPAuthReg(reg);
1002}
1003
1004bool NativeRegisterContextLinux_arm64::IsMTE(unsigned reg) const {
1005 return GetRegisterInfo().IsMTEReg(reg);
1006}
1007
1008bool NativeRegisterContextLinux_arm64::IsTLS(unsigned reg) const {
1009 return GetRegisterInfo().IsTLSReg(reg);
1010}
1011
1012llvm::Error NativeRegisterContextLinux_arm64::ReadHardwareDebugInfo() {
1013 if (!m_refresh_hwdebug_info) {
1014 return llvm::Error::success();
1015 }
1016
1017 ::pid_t tid = m_thread.GetID();
1018
1019 int regset = NT_ARM_HW_WATCH;
1020 struct iovec ioVec;
1021 struct user_hwdebug_state dreg_state;
1022 Status error;
1023
1024 ioVec.iov_base = &dreg_state;
1025 ioVec.iov_len = sizeof(dreg_state);
1026 error = NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET, tid, &regset,
1027 &ioVec, ioVec.iov_len);
1028
1029 if (error.Fail())
1030 return error.ToError();
1031
1032 m_max_hwp_supported = dreg_state.dbg_info & 0xff;
1033
1034 regset = NT_ARM_HW_BREAK;
1035 error = NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET, tid, &regset,
1036 &ioVec, ioVec.iov_len);
1037
1038 if (error.Fail())
1039 return error.ToError();
1040
1041 m_max_hbp_supported = dreg_state.dbg_info & 0xff;
1042 m_refresh_hwdebug_info = false;
1043
1044 return llvm::Error::success();
1045}
1046
1047llvm::Error
1048NativeRegisterContextLinux_arm64::WriteHardwareDebugRegs(DREGType hwbType) {
1049 struct iovec ioVec;
1050 struct user_hwdebug_state dreg_state;
1051 int regset;
1052
1053 memset(&dreg_state, 0, sizeof(dreg_state));
1054 ioVec.iov_base = &dreg_state;
1055
1056 switch (hwbType) {
1057 case eDREGTypeWATCH:
1058 regset = NT_ARM_HW_WATCH;
1059 ioVec.iov_len = sizeof(dreg_state.dbg_info) + sizeof(dreg_state.pad) +
1060 (sizeof(dreg_state.dbg_regs[0]) * m_max_hwp_supported);
1061
1062 for (uint32_t i = 0; i < m_max_hwp_supported; i++) {
1063 dreg_state.dbg_regs[i].addr = m_hwp_regs[i].address;
1064 dreg_state.dbg_regs[i].ctrl = m_hwp_regs[i].control;
1065 }
1066 break;
1067 case eDREGTypeBREAK:
1068 regset = NT_ARM_HW_BREAK;
1069 ioVec.iov_len = sizeof(dreg_state.dbg_info) + sizeof(dreg_state.pad) +
1070 (sizeof(dreg_state.dbg_regs[0]) * m_max_hbp_supported);
1071
1072 for (uint32_t i = 0; i < m_max_hbp_supported; i++) {
1073 dreg_state.dbg_regs[i].addr = m_hbp_regs[i].address;
1074 dreg_state.dbg_regs[i].ctrl = m_hbp_regs[i].control;
1075 }
1076 break;
1077 }
1078
1079 return NativeProcessLinux::PtraceWrapper(PTRACE_SETREGSET, m_thread.GetID(),
1080 &regset, &ioVec, ioVec.iov_len)
1081 .ToError();
1082}
1083
1084Status NativeRegisterContextLinux_arm64::ReadGPR() {
1085 Status error;
1086
1087 if (m_gpr_is_valid)
1088 return error;
1089
1090 struct iovec ioVec;
1091 ioVec.iov_base = GetGPRBuffer();
1092 ioVec.iov_len = GetGPRBufferSize();
1093
1094 error = ReadRegisterSet(&ioVec, GetGPRBufferSize(), NT_PRSTATUS);
1095
1096 if (error.Success())
1097 m_gpr_is_valid = true;
1098
1099 return error;
1100}
1101
1102Status NativeRegisterContextLinux_arm64::WriteGPR() {
1103 Status error = ReadGPR();
1104 if (error.Fail())
1105 return error;
1106
1107 struct iovec ioVec;
1108 ioVec.iov_base = GetGPRBuffer();
1109 ioVec.iov_len = GetGPRBufferSize();
1110
1111 m_gpr_is_valid = false;
1112
1113 return WriteRegisterSet(&ioVec, GetGPRBufferSize(), NT_PRSTATUS);
1114}
1115
1116Status NativeRegisterContextLinux_arm64::ReadFPR() {
1117 Status error;
1118
1119 if (m_fpu_is_valid)
1120 return error;
1121
1122 struct iovec ioVec;
1123 ioVec.iov_base = GetFPRBuffer();
1124 ioVec.iov_len = GetFPRSize();
1125
1126 error = ReadRegisterSet(&ioVec, GetFPRSize(), NT_FPREGSET);
1127
1128 if (error.Success())
1129 m_fpu_is_valid = true;
1130
1131 return error;
1132}
1133
1134Status NativeRegisterContextLinux_arm64::WriteFPR() {
1135 Status error = ReadFPR();
1136 if (error.Fail())
1137 return error;
1138
1139 struct iovec ioVec;
1140 ioVec.iov_base = GetFPRBuffer();
1141 ioVec.iov_len = GetFPRSize();
1142
1143 m_fpu_is_valid = false;
1144
1145 return WriteRegisterSet(&ioVec, GetFPRSize(), NT_FPREGSET);
1146}
1147
1148void NativeRegisterContextLinux_arm64::InvalidateAllRegisters() {
1149 m_gpr_is_valid = false;
1150 m_fpu_is_valid = false;
1151 m_sve_buffer_is_valid = false;
1152 m_sve_header_is_valid = false;
1153 m_za_buffer_is_valid = false;
1154 m_za_header_is_valid = false;
1155 m_pac_mask_is_valid = false;
1156 m_mte_ctrl_is_valid = false;
1157 m_tls_is_valid = false;
1158 m_zt_buffer_is_valid = false;
1159
1160 // Update SVE and ZA registers in case there is change in configuration.
1161 ConfigureRegisterContext();
1162}
1163
1164unsigned NativeRegisterContextLinux_arm64::GetSVERegSet() {
1165 return m_sve_state == SVEState::Streaming ? NT_ARM_SSVE : NT_ARM_SVE;
1166}
1167
1168Status NativeRegisterContextLinux_arm64::ReadSVEHeader() {
1169 Status error;
1170
1171 if (m_sve_header_is_valid)
1172 return error;
1173
1174 struct iovec ioVec;
1175 ioVec.iov_base = GetSVEHeader();
1176 ioVec.iov_len = GetSVEHeaderSize();
1177
1178 error = ReadRegisterSet(&ioVec, GetSVEHeaderSize(), GetSVERegSet());
1179
1180 if (error.Success())
1181 m_sve_header_is_valid = true;
1182
1183 return error;
1184}
1185
1186Status NativeRegisterContextLinux_arm64::ReadPAuthMask() {
1187 Status error;
1188
1189 if (m_pac_mask_is_valid)
1190 return error;
1191
1192 struct iovec ioVec;
1193 ioVec.iov_base = GetPACMask();
1194 ioVec.iov_len = GetPACMaskSize();
1195
1196 error = ReadRegisterSet(&ioVec, GetPACMaskSize(), NT_ARM_PAC_MASK);
1197
1198 if (error.Success())
1199 m_pac_mask_is_valid = true;
1200
1201 return error;
1202}
1203
1204Status NativeRegisterContextLinux_arm64::WriteSVEHeader() {
1205 Status error;
1206
1207 error = ReadSVEHeader();
1208 if (error.Fail())
1209 return error;
1210
1211 struct iovec ioVec;
1212 ioVec.iov_base = GetSVEHeader();
1213 ioVec.iov_len = GetSVEHeaderSize();
1214
1215 m_sve_buffer_is_valid = false;
1216 m_sve_header_is_valid = false;
1217 m_fpu_is_valid = false;
1218
1219 return WriteRegisterSet(&ioVec, GetSVEHeaderSize(), GetSVERegSet());
1220}
1221
1222Status NativeRegisterContextLinux_arm64::ReadAllSVE() {
1223 Status error;
1224 if (m_sve_buffer_is_valid)
1225 return error;
1226
1227 struct iovec ioVec;
1228 ioVec.iov_base = GetSVEBuffer();
1229 ioVec.iov_len = GetSVEBufferSize();
1230
1231 error = ReadRegisterSet(&ioVec, GetSVEBufferSize(), GetSVERegSet());
1232
1233 if (error.Success())
1234 m_sve_buffer_is_valid = true;
1235
1236 return error;
1237}
1238
1239Status NativeRegisterContextLinux_arm64::WriteAllSVE() {
1240 Status error;
1241
1242 error = ReadAllSVE();
1243 if (error.Fail())
1244 return error;
1245
1246 struct iovec ioVec;
1247
1248 ioVec.iov_base = GetSVEBuffer();
1249 ioVec.iov_len = GetSVEBufferSize();
1250
1251 m_sve_buffer_is_valid = false;
1252 m_sve_header_is_valid = false;
1253 m_fpu_is_valid = false;
1254
1255 return WriteRegisterSet(&ioVec, GetSVEBufferSize(), GetSVERegSet());
1256}
1257
1258Status NativeRegisterContextLinux_arm64::ReadSMEControl() {
1259 // The real register is SVCR and is accessible from EL0. However we don't want
1260 // to have to JIT code into the target process so we'll just recreate it using
1261 // what we know from ptrace.
1262
1263 // Bit 0 indicates whether streaming mode is active.
1264 m_sme_pseudo_regs.ctrl_reg = m_sve_state == SVEState::Streaming;
1265
1266 // Bit 1 indicates whether the array storage is active.
1267 // It is active if we can read the header and the size field tells us that
1268 // there is register data following it.
1269 Status error = ReadZAHeader();
1270 if (error.Success() && (m_za_header.size > sizeof(m_za_header)))
1271 m_sme_pseudo_regs.ctrl_reg |= 2;
1272
1273 return error;
1274}
1275
1276Status NativeRegisterContextLinux_arm64::ReadMTEControl() {
1277 Status error;
1278
1279 if (m_mte_ctrl_is_valid)
1280 return error;
1281
1282 struct iovec ioVec;
1283 ioVec.iov_base = GetMTEControl();
1284 ioVec.iov_len = GetMTEControlSize();
1285
1286 error = ReadRegisterSet(&ioVec, GetMTEControlSize(), NT_ARM_TAGGED_ADDR_CTRL);
1287
1288 if (error.Success())
1289 m_mte_ctrl_is_valid = true;
1290
1291 return error;
1292}
1293
1294Status NativeRegisterContextLinux_arm64::WriteMTEControl() {
1295 Status error;
1296
1297 error = ReadMTEControl();
1298 if (error.Fail())
1299 return error;
1300
1301 struct iovec ioVec;
1302 ioVec.iov_base = GetMTEControl();
1303 ioVec.iov_len = GetMTEControlSize();
1304
1305 m_mte_ctrl_is_valid = false;
1306
1307 return WriteRegisterSet(&ioVec, GetMTEControlSize(), NT_ARM_TAGGED_ADDR_CTRL);
1308}
1309
1310Status NativeRegisterContextLinux_arm64::ReadTLS() {
1311 Status error;
1312
1313 if (m_tls_is_valid)
1314 return error;
1315
1316 struct iovec ioVec;
1317 ioVec.iov_base = GetTLSBuffer();
1318 ioVec.iov_len = GetTLSBufferSize();
1319
1320 error = ReadRegisterSet(&ioVec, GetTLSBufferSize(), NT_ARM_TLS);
1321
1322 if (error.Success())
1323 m_tls_is_valid = true;
1324
1325 return error;
1326}
1327
1328Status NativeRegisterContextLinux_arm64::WriteTLS() {
1329 Status error;
1330
1331 error = ReadTLS();
1332 if (error.Fail())
1333 return error;
1334
1335 struct iovec ioVec;
1336 ioVec.iov_base = GetTLSBuffer();
1337 ioVec.iov_len = GetTLSBufferSize();
1338
1339 m_tls_is_valid = false;
1340
1341 return WriteRegisterSet(&ioVec, GetTLSBufferSize(), NT_ARM_TLS);
1342}
1343
1344Status NativeRegisterContextLinux_arm64::ReadZAHeader() {
1345 Status error;
1346
1347 if (m_za_header_is_valid)
1348 return error;
1349
1350 struct iovec ioVec;
1351 ioVec.iov_base = GetZAHeader();
1352 ioVec.iov_len = GetZAHeaderSize();
1353
1354 error = ReadRegisterSet(&ioVec, GetZAHeaderSize(), NT_ARM_ZA);
1355
1356 if (error.Success())
1357 m_za_header_is_valid = true;
1358
1359 return error;
1360}
1361
1362Status NativeRegisterContextLinux_arm64::ReadZA() {
1363 Status error;
1364
1365 if (m_za_buffer_is_valid)
1366 return error;
1367
1368 struct iovec ioVec;
1369 ioVec.iov_base = GetZABuffer();
1370 ioVec.iov_len = GetZABufferSize();
1371
1372 error = ReadRegisterSet(&ioVec, GetZABufferSize(), NT_ARM_ZA);
1373
1374 if (error.Success())
1375 m_za_buffer_is_valid = true;
1376
1377 return error;
1378}
1379
1380Status NativeRegisterContextLinux_arm64::WriteZA() {
1381 // Note that because the ZA ptrace payload contains the header also, this
1382 // method will write both. This is done because writing only the header
1383 // will disable ZA, even if .size in the header is correct for an enabled ZA.
1384 Status error;
1385
1386 error = ReadZA();
1387 if (error.Fail())
1388 return error;
1389
1390 struct iovec ioVec;
1391 ioVec.iov_base = GetZABuffer();
1392 ioVec.iov_len = GetZABufferSize();
1393
1394 m_za_buffer_is_valid = false;
1395 m_za_header_is_valid = false;
1396 // Writing to ZA may enable ZA, which means ZT0 may change too.
1397 m_zt_buffer_is_valid = false;
1398
1399 return WriteRegisterSet(&ioVec, GetZABufferSize(), NT_ARM_ZA);
1400}
1401
1402Status NativeRegisterContextLinux_arm64::ReadZT() {
1403 Status error;
1404
1405 if (m_zt_buffer_is_valid)
1406 return error;
1407
1408 struct iovec ioVec;
1409 ioVec.iov_base = GetZTBuffer();
1410 ioVec.iov_len = GetZTBufferSize();
1411
1412 error = ReadRegisterSet(&ioVec, GetZTBufferSize(), NT_ARM_ZT);
1413 m_zt_buffer_is_valid = error.Success();
1414
1415 return error;
1416}
1417
1418Status NativeRegisterContextLinux_arm64::WriteZT() {
1419 Status error;
1420
1421 error = ReadZT();
1422 if (error.Fail())
1423 return error;
1424
1425 struct iovec ioVec;
1426 ioVec.iov_base = GetZTBuffer();
1427 ioVec.iov_len = GetZTBufferSize();
1428
1429 m_zt_buffer_is_valid = false;
1430 // Writing to an inactive ZT0 will enable ZA as well, which invalidates our
1431 // current copy of it.
1432 m_za_buffer_is_valid = false;
1433 m_za_header_is_valid = false;
1434
1435 return WriteRegisterSet(&ioVec, GetZTBufferSize(), NT_ARM_ZT);
1436}
1437
1438void NativeRegisterContextLinux_arm64::ConfigureRegisterContext() {
1439 // ConfigureRegisterContext gets called from InvalidateAllRegisters
1440 // on every stop and configures SVE vector length and whether we are in
1441 // streaming SVE mode.
1442 // If m_sve_state is set to SVEState::Disabled on first stop, code below will
1443 // be deemed non operational for the lifetime of current process.
1444 if (!m_sve_header_is_valid && m_sve_state != SVEState::Disabled) {
1445 // If we have SVE we may also have the SVE streaming mode that SME added.
1446 // We can read the header of either mode, but only the active mode will
1447 // have valid register data.
1448
1449 // Check whether SME is present and the streaming SVE mode is active.
1450 m_sve_header_is_valid = false;
1451 m_sve_buffer_is_valid = false;
1452 m_sve_state = SVEState::Streaming;
1453 Status error = ReadSVEHeader();
1454
1455 // Streaming mode is active if the header has the SVE active flag set.
1456 if (!(error.Success() && ((m_sve_header.flags & sve::ptrace_regs_mask) ==
1457 sve::ptrace_regs_sve))) {
1458 // Non-streaming might be active instead.
1459 m_sve_header_is_valid = false;
1460 m_sve_buffer_is_valid = false;
1461 m_sve_state = SVEState::Full;
1462 error = ReadSVEHeader();
1463 if (error.Success()) {
1464 // If SVE is enabled thread can switch between SVEState::FPSIMD and
1465 // SVEState::Full on every stop.
1466 if ((m_sve_header.flags & sve::ptrace_regs_mask) ==
1467 sve::ptrace_regs_fpsimd)
1468 m_sve_state = SVEState::FPSIMD;
1469 // Else we are in SVEState::Full.
1470 } else {
1471 m_sve_state = SVEState::Disabled;
1472 }
1473 }
1474
1475 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::FPSIMD ||
1476 m_sve_state == SVEState::Streaming) {
1477 // On every stop we configure SVE vector length by calling
1478 // ConfigureVectorLengthSVE regardless of current SVEState of this thread.
1479 uint32_t vq = RegisterInfoPOSIX_arm64::eVectorQuadwordAArch64SVE;
1480 if (sve::vl_valid(m_sve_header.vl))
1481 vq = sve::vq_from_vl(m_sve_header.vl);
1482
1483 GetRegisterInfo().ConfigureVectorLengthSVE(vq);
1484 m_sve_ptrace_payload.resize(sve::PTraceSize(vq, sve::ptrace_regs_sve));
1485 }
1486 }
1487
1488 if (!m_za_header_is_valid) {
1489 Status error = ReadZAHeader();
1490 if (error.Success()) {
1491 uint32_t vq = RegisterInfoPOSIX_arm64::eVectorQuadwordAArch64SVE;
1492 if (sve::vl_valid(m_za_header.vl))
1493 vq = sve::vq_from_vl(m_za_header.vl);
1494
1495 GetRegisterInfo().ConfigureVectorLengthZA(vq);
1496 m_za_ptrace_payload.resize(m_za_header.size);
1497 m_za_buffer_is_valid = false;
1498 }
1499 }
1500}
1501
1502uint32_t NativeRegisterContextLinux_arm64::CalculateFprOffset(
1503 const RegisterInfo *reg_info) const {
1504 return reg_info->byte_offset - GetGPRSize();
1505}
1506
1507uint32_t NativeRegisterContextLinux_arm64::CalculateSVEOffset(
1508 const RegisterInfo *reg_info) const {
1509 // Start of Z0 data is after GPRs plus 8 bytes of vg register
1510 uint32_t sve_reg_offset = LLDB_INVALID_INDEX32;
1511 if (m_sve_state == SVEState::FPSIMD) {
1512 const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];
1513 sve_reg_offset = sve::ptrace_fpsimd_offset +
1514 (reg - GetRegisterInfo().GetRegNumSVEZ0()) * 16;
1515 // Between non-streaming and streaming mode, the layout is identical.
1516 } else if (m_sve_state == SVEState::Full ||
1517 m_sve_state == SVEState::Streaming) {
1518 uint32_t sve_z0_offset = GetGPRSize() + 16;
1519 sve_reg_offset =
1520 sve::SigRegsOffset() + reg_info->byte_offset - sve_z0_offset;
1521 }
1522 return sve_reg_offset;
1523}
1524
1525Status NativeRegisterContextLinux_arm64::ReadSMESVG() {
1526 // This register is the streaming vector length, so we will get it from
1527 // NT_ARM_ZA regardless of the current streaming mode.
1528 Status error = ReadZAHeader();
1529 if (error.Success())
1530 m_sme_pseudo_regs.svg_reg = m_za_header.vl / 8;
1531
1532 return error;
1533}
1534
1535std::vector<uint32_t> NativeRegisterContextLinux_arm64::GetExpeditedRegisters(
1536 ExpeditedRegs expType) const {
1537 std::vector<uint32_t> expedited_reg_nums =
1538 NativeRegisterContext::GetExpeditedRegisters(expType);
1539 // SVE, non-streaming vector length.
1540 if (m_sve_state == SVEState::FPSIMD || m_sve_state == SVEState::Full)
1541 expedited_reg_nums.push_back(GetRegisterInfo().GetRegNumSVEVG());
1542 // SME, streaming vector length. This is used by the ZA register which is
1543 // present even when streaming mode is not enabled.
1544 if (GetRegisterInfo().IsSSVEPresent())
1545 expedited_reg_nums.push_back(GetRegisterInfo().GetRegNumSMESVG());
1546
1547 return expedited_reg_nums;
1548}
1549
1550llvm::Expected<NativeRegisterContextLinux::MemoryTaggingDetails>
1551NativeRegisterContextLinux_arm64::GetMemoryTaggingDetails(int32_t type) {
1552 if (type == MemoryTagManagerAArch64MTE::eMTE_allocation) {
1553 return MemoryTaggingDetails{std::make_unique<MemoryTagManagerAArch64MTE>(),
1554 PTRACE_PEEKMTETAGS, PTRACE_POKEMTETAGS};
1555 }
1556
1557 return llvm::createStringError(llvm::inconvertibleErrorCode(),
1558 "Unknown AArch64 memory tag type %d", type);
1559}
1560
1561lldb::addr_t NativeRegisterContextLinux_arm64::FixWatchpointHitAddress(
1562 lldb::addr_t hit_addr) {
1563 // Linux configures user-space virtual addresses with top byte ignored.
1564 // We set default value of mask such that top byte is masked out.
1565 lldb::addr_t mask = ~((1ULL << 56) - 1);
1566
1567 // Try to read pointer authentication data_mask register and calculate a
1568 // consolidated data address mask after ignoring the top byte.
1569 if (ReadPAuthMask().Success())
1570 mask |= m_pac_mask.data_mask;
1571
1572 return hit_addr & ~mask;
1573 ;
1574}
1575
1576#endif // defined (__arm64__) || defined (__aarch64__)
1577

source code of lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp