1	//===-- SveEmitter.cpp - Generate arm_sve.h for use with clang ------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This tablegen backend is responsible for emitting arm_sve.h, which includes
10	// a declaration and definition of each function specified by the ARM C/C++
11	// Language Extensions (ACLE).
12	//
13	// For details, visit:
14	// https://developer.arm.com/architectures/system-architectures/software-standards/acle
15	//
16	// Each SVE instruction is implemented in terms of 1 or more functions which
17	// are suffixed with the element type of the input vectors. Functions may be
18	// implemented in terms of generic vector operations such as +, *, -, etc. or
19	// by calling a __builtin_-prefixed function which will be handled by clang's
20	// CodeGen library.
21	//
22	// See also the documentation in include/clang/Basic/arm_sve.td.
23	//
24	//===----------------------------------------------------------------------===//
25
26	#include "llvm/ADT/ArrayRef.h"
27	#include "llvm/ADT/STLExtras.h"
28	#include "llvm/ADT/StringExtras.h"
29	#include "llvm/ADT/StringMap.h"
30	#include "llvm/Support/raw_ostream.h"
31	#include "llvm/TableGen/AArch64ImmCheck.h"
32	#include "llvm/TableGen/Error.h"
33	#include "llvm/TableGen/Record.h"
34	#include "llvm/TableGen/StringToOffsetTable.h"
35	#include <array>
36	#include <cctype>
37	#include <set>
38	#include <string>
39	#include <tuple>
40
41	using namespace llvm;
42
43	enum ClassKind {
44	ClassNone,
45	ClassS, // signed/unsigned, e.g., "_s8", "_u8" suffix
46	ClassG, // Overloaded name without type suffix
47	};
48
49	enum class ACLEKind { SVE, SME };
50
51	using TypeSpec = std::string;
52
53	namespace {
54	class SVEType {
55
56	enum TypeKind {
57	Invalid,
58	Void,
59	Float,
60	SInt,
61	UInt,
62	BFloat16,
63	MFloat8,
64	Svcount,
65	PrefetchOp,
66	PredicatePattern,
67	Predicate,
68	Fpm
69	};
70	TypeKind Kind;
71	bool Immediate, Constant, Pointer, DefaultType, IsScalable;
72	unsigned Bitwidth, ElementBitwidth, NumVectors;
73
74	public:
75	SVEType() : SVEType("", 'v') {}
76
77	SVEType(StringRef TS, char CharMod, unsigned NumVectors = 1)
78	: Kind(Invalid), Immediate(false), Constant(false), Pointer(false),
79	DefaultType(false), IsScalable(true), Bitwidth(128),
80	ElementBitwidth(~0U), NumVectors(NumVectors) {
81	if (!TS.empty())
82	applyTypespec(TS);
83	applyModifier(Mod: CharMod);
84	}
85
86	SVEType(const SVEType &Base, unsigned NumV) : SVEType(Base) {
87	NumVectors = NumV;
88	}
89
90	bool isPointer() const { return Pointer; }
91	bool isConstant() const { return Constant; }
92	bool isImmediate() const { return Immediate; }
93	bool isScalar() const { return NumVectors == 0; }
94	bool isVector() const { return NumVectors > 0; }
95	bool isScalableVector() const { return isVector() && IsScalable; }
96	bool isFixedLengthVector() const { return isVector() && !IsScalable; }
97	bool isChar() const { return ElementBitwidth == 8 && isInteger(); }
98	bool isVoid() const { return Kind == Void; }
99	bool isDefault() const { return DefaultType; }
100	bool isFloat() const { return Kind == Float; }
101	bool isBFloat() const { return Kind == BFloat16; }
102	bool isMFloat() const { return Kind == MFloat8; }
103	bool isFloatingPoint() const {
104	return Kind == Float || Kind == BFloat16 || Kind == MFloat8;
105	}
106	bool isInteger() const { return Kind == SInt || Kind == UInt; }
107	bool isSignedInteger() const { return Kind == SInt; }
108	bool isUnsignedInteger() const { return Kind == UInt; }
109	bool isScalarPredicate() const {
110	return Kind == Predicate && NumVectors == 0;
111	}
112	bool isPredicate() const { return Kind == Predicate; }
113	bool isPredicatePattern() const { return Kind == PredicatePattern; }
114	bool isPrefetchOp() const { return Kind == PrefetchOp; }
115	bool isSvcount() const { return Kind == Svcount; }
116	bool isFpm() const { return Kind == Fpm; }
117	bool isInvalid() const { return Kind == Invalid; }
118	unsigned getElementSizeInBits() const { return ElementBitwidth; }
119	unsigned getNumVectors() const { return NumVectors; }
120
121	unsigned getNumElements() const {
122	assert(ElementBitwidth != ~0U);
123	return isPredicate() ? 16 : (Bitwidth / ElementBitwidth);
124	}
125	unsigned getSizeInBits() const {
126	return Bitwidth;
127	}
128
129	/// Return the string representation of a type, which is an encoded
130	/// string for passing to the BUILTIN() macro in Builtins.def.
131	std::string builtin_str() const;
132
133	/// Return the C/C++ string representation of a type for use in the
134	/// arm_sve.h header file.
135	std::string str() const;
136
137	private:
138	/// Creates the type based on the typespec string in TS.
139	void applyTypespec(StringRef TS);
140
141	/// Applies a prototype modifier to the type.
142	void applyModifier(char Mod);
143
144	/// Get the builtin base for this SVEType, e.g. 'Wi' for svint64_t.
145	std::string builtinBaseType() const;
146	};
147
148	class SVEEmitter;
149
150	/// The main grunt class. This represents an instantiation of an intrinsic with
151	/// a particular typespec and prototype.
152	class Intrinsic {
153	/// The unmangled name.
154	std::string Name;
155
156	/// The name of the corresponding LLVM IR intrinsic.
157	std::string LLVMName;
158
159	/// Intrinsic prototype.
160	std::string Proto;
161
162	/// The base type spec for this intrinsic.
163	TypeSpec BaseTypeSpec;
164
165	/// The base class kind. Most intrinsics use ClassS, which has full type
166	/// info for integers (_s32/_u32), or ClassG which is used for overloaded
167	/// intrinsics.
168	ClassKind Class;
169
170	/// The architectural #ifdef guard.
171	std::string SVEGuard, SMEGuard;
172
173	// The merge suffix such as _m, _x or _z.
174	std::string MergeSuffix;
175
176	/// The types of return value [0] and parameters [1..].
177	std::vector<SVEType> Types;
178
179	/// The "base type", which is VarType('d', BaseTypeSpec).
180	SVEType BaseType;
181
182	uint64_t Flags;
183
184	SmallVector<ImmCheck, 2> ImmChecks;
185
186	bool SetsFPMR;
187
188	public:
189	Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
190	StringRef MergeSuffix, uint64_t MemoryElementTy, StringRef LLVMName,
191	uint64_t Flags, ArrayRef<ImmCheck> ImmChecks, TypeSpec BT,
192	ClassKind Class, SVEEmitter &Emitter, StringRef SVEGuard,
193	StringRef SMEGuard);
194
195	~Intrinsic()=default;
196
197	std::string getName() const { return Name; }
198	std::string getLLVMName() const { return LLVMName; }
199	std::string getProto() const { return Proto; }
200	TypeSpec getBaseTypeSpec() const { return BaseTypeSpec; }
201	SVEType getBaseType() const { return BaseType; }
202
203	StringRef getSVEGuard() const { return SVEGuard; }
204	StringRef getSMEGuard() const { return SMEGuard; }
205	std::string getGuard() const {
206	std::string Guard;
207	llvm::raw_string_ostream OS(Guard);
208	if (!SVEGuard.empty() && SMEGuard.empty())
209	OS << SVEGuard;
210	else if (SVEGuard.empty() && !SMEGuard.empty())
211	OS << SMEGuard;
212	else {
213	if (SVEGuard.find(s: ",") != std::string::npos ||
214	SVEGuard.find(s: "|") != std::string::npos)
215	OS << "(" << SVEGuard << ")";
216	else
217	OS << SVEGuard;
218	OS << "|";
219	if (SMEGuard.find(s: ",") != std::string::npos ||
220	SMEGuard.find(s: "|") != std::string::npos)
221	OS << "(" << SMEGuard << ")";
222	else
223	OS << SMEGuard;
224	}
225	return Guard;
226	}
227	ClassKind getClassKind() const { return Class; }
228
229	SVEType getReturnType() const { return Types[0]; }
230	ArrayRef<SVEType> getTypes() const { return Types; }
231	SVEType getParamType(unsigned I) const { return Types[I + 1]; }
232	unsigned getNumParams() const {
233	return Proto.size() - (2 * count(Range: Proto, Element: '.')) - 1;
234	}
235
236	uint64_t getFlags() const { return Flags; }
237	bool isFlagSet(uint64_t Flag) const { return Flags & Flag;}
238
239	ArrayRef<ImmCheck> getImmChecks() const { return ImmChecks; }
240
241	/// Return the type string for a BUILTIN() macro in Builtins.def.
242	std::string getBuiltinTypeStr();
243
244	/// Return the name, mangled with type information. The name is mangled for
245	/// ClassS, so will add type suffixes such as _u32/_s32.
246	std::string getMangledName() const { return mangleName(LocalCK: ClassS); }
247
248	/// As above, but mangles the LLVM name instead.
249	std::string getMangledLLVMName() const { return mangleLLVMName(); }
250
251	/// Returns true if the intrinsic is overloaded, in that it should also generate
252	/// a short form without the type-specifiers, e.g. 'svld1(..)' instead of
253	/// 'svld1_u32(..)'.
254	static bool isOverloadedIntrinsic(StringRef Name) {
255	return Name.contains(C: '[') && Name.contains(C: ']');
256	}
257
258	/// Return true if the intrinsic takes a splat operand.
259	bool hasSplat() const {
260	// These prototype modifiers are described in arm_sve.td.
261	return Proto.find_first_of(s: "ajfrKLR@!") != std::string::npos;
262	}
263
264	/// Return the parameter index of the splat operand.
265	unsigned getSplatIdx() const {
266	unsigned I = 1, Param = 0;
267	for (; I < Proto.size(); ++I, ++Param) {
268	if (Proto[I] == 'a' || Proto[I] == 'j' || Proto[I] == 'f' ||
269	Proto[I] == 'r' || Proto[I] == 'K' || Proto[I] == 'L' ||
270	Proto[I] == 'R' || Proto[I] == '@' || Proto[I] == '!')
271	break;
272
273	// Multivector modifier can be skipped
274	if (Proto[I] == '.')
275	I += 2;
276	}
277	assert(I != Proto.size() && "Prototype has no splat operand");
278	return Param;
279	}
280
281	/// Emits the intrinsic declaration to the ostream.
282	void emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter, ACLEKind Kind) const;
283
284	private:
285	std::string getMergeSuffix() const { return MergeSuffix; }
286	StringRef getFPMSuffix() const { return SetsFPMR ? "_fpm" : ""; }
287	std::string mangleName(ClassKind LocalCK) const;
288	std::string mangleLLVMName() const;
289	std::string replaceTemplatedArgs(std::string Name, TypeSpec TS,
290	std::string Proto) const;
291	};
292
293	class SVEEmitter {
294	private:
295	// The reinterpret builtins are generated separately because they
296	// need the cross product of all types (121 functions in total),
297	// which is inconvenient to specify in the arm_sve.td file or
298	// generate in CGBuiltin.cpp.
299	struct ReinterpretTypeInfo {
300	SVEType BaseType;
301	const char *Suffix;
302	};
303
304	static const std::array<ReinterpretTypeInfo, 13> Reinterprets;
305
306	const RecordKeeper &Records;
307	StringMap<uint64_t> EltTypes;
308	StringMap<uint64_t> MemEltTypes;
309	StringMap<uint64_t> FlagTypes;
310	StringMap<uint64_t> MergeTypes;
311	StringMap<uint64_t> ImmCheckTypes;
312
313	public:
314	SVEEmitter(const RecordKeeper &R) : Records(R) {
315	for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "EltType"))
316	EltTypes[RV->getNameInitAsString()] = RV->getValueAsInt(FieldName: "Value");
317	for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "MemEltType"))
318	MemEltTypes[RV->getNameInitAsString()] = RV->getValueAsInt(FieldName: "Value");
319	for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "FlagType"))
320	FlagTypes[RV->getNameInitAsString()] = RV->getValueAsInt(FieldName: "Value");
321	for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "MergeType"))
322	MergeTypes[RV->getNameInitAsString()] = RV->getValueAsInt(FieldName: "Value");
323	for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "ImmCheckType"))
324	ImmCheckTypes[RV->getNameInitAsString()] = RV->getValueAsInt(FieldName: "Value");
325	}
326
327	/// Returns the enum value for the immcheck type
328	unsigned getEnumValueForImmCheck(StringRef C) const {
329	auto It = ImmCheckTypes.find(Key: C);
330	if (It != ImmCheckTypes.end())
331	return It->getValue();
332	llvm_unreachable("Unsupported imm check");
333	}
334
335	/// Returns the enum value for the flag type
336	uint64_t getEnumValueForFlag(StringRef C) const {
337	auto Res = FlagTypes.find(Key: C);
338	if (Res != FlagTypes.end())
339	return Res->getValue();
340	llvm_unreachable("Unsupported flag");
341	}
342
343	// Returns the SVETypeFlags for a given value and mask.
344	uint64_t encodeFlag(uint64_t V, StringRef MaskName) const {
345	auto It = FlagTypes.find(Key: MaskName);
346	if (It != FlagTypes.end()) {
347	uint64_t Mask = It->getValue();
348	unsigned Shift = countr_zero(Val: Mask);
349	assert(Shift < 64 && "Mask value produced an invalid shift value");
350	return (V << Shift) & Mask;
351	}
352	llvm_unreachable("Unsupported flag");
353	}
354
355	// Returns the SVETypeFlags for the given element type.
356	uint64_t encodeEltType(StringRef EltName) {
357	auto It = EltTypes.find(Key: EltName);
358	if (It != EltTypes.end())
359	return encodeFlag(V: It->getValue(), MaskName: "EltTypeMask");
360	llvm_unreachable("Unsupported EltType");
361	}
362
363	// Returns the SVETypeFlags for the given memory element type.
364	uint64_t encodeMemoryElementType(uint64_t MT) {
365	return encodeFlag(V: MT, MaskName: "MemEltTypeMask");
366	}
367
368	// Returns the SVETypeFlags for the given merge type.
369	uint64_t encodeMergeType(uint64_t MT) {
370	return encodeFlag(V: MT, MaskName: "MergeTypeMask");
371	}
372
373	// Returns the SVETypeFlags for the given splat operand.
374	unsigned encodeSplatOperand(unsigned SplatIdx) {
375	assert(SplatIdx < 7 && "SplatIdx out of encodable range");
376	return encodeFlag(V: SplatIdx + 1, MaskName: "SplatOperandMask");
377	}
378
379	// Returns the SVETypeFlags value for the given SVEType.
380	uint64_t encodeTypeFlags(const SVEType &T);
381
382	/// Emit arm_sve.h.
383	void createHeader(raw_ostream &o);
384
385	// Emits core intrinsics in both arm_sme.h and arm_sve.h
386	void createCoreHeaderIntrinsics(raw_ostream &o, SVEEmitter &Emitter,
387	ACLEKind Kind);
388
389	/// Emit all the __builtin prototypes and code needed by Sema.
390	void createBuiltins(raw_ostream &o);
391
392	/// Emit all the information needed to map builtin -> LLVM IR intrinsic.
393	void createCodeGenMap(raw_ostream &o);
394
395	/// Emit all the range checks for the immediates.
396	void createRangeChecks(raw_ostream &o);
397
398	// Emit all the ImmCheckTypes to arm_immcheck_types.inc
399	void createImmCheckTypes(raw_ostream &OS);
400
401	/// Create the SVETypeFlags used in CGBuiltins
402	void createTypeFlags(raw_ostream &o);
403
404	/// Emit arm_sme.h.
405	void createSMEHeader(raw_ostream &o);
406
407	/// Emit all the SME __builtin prototypes and code needed by Sema.
408	void createSMEBuiltins(raw_ostream &o);
409
410	/// Emit all the information needed to map builtin -> LLVM IR intrinsic.
411	void createSMECodeGenMap(raw_ostream &o);
412
413	/// Create a table for a builtin's requirement for PSTATE.SM.
414	void createStreamingAttrs(raw_ostream &o, ACLEKind Kind);
415
416	/// Emit all the range checks for the immediates.
417	void createSMERangeChecks(raw_ostream &o);
418
419	/// Create a table for a builtin's requirement for PSTATE.ZA.
420	void createBuiltinZAState(raw_ostream &OS);
421
422	/// Create intrinsic and add it to \p Out
423	void createIntrinsic(const Record *R,
424	SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out);
425	};
426
427	const std::array<SVEEmitter::ReinterpretTypeInfo, 13> SVEEmitter::Reinterprets =
428	{._M_elems: {{.BaseType: SVEType("c", 'd'), .Suffix: "s8"},
429	{.BaseType: SVEType("Uc", 'd'), .Suffix: "u8"},
430	{.BaseType: SVEType("m", 'd'), .Suffix: "mf8"},
431	{.BaseType: SVEType("s", 'd'), .Suffix: "s16"},
432	{.BaseType: SVEType("Us", 'd'), .Suffix: "u16"},
433	{.BaseType: SVEType("i", 'd'), .Suffix: "s32"},
434	{.BaseType: SVEType("Ui", 'd'), .Suffix: "u32"},
435	{.BaseType: SVEType("l", 'd'), .Suffix: "s64"},
436	{.BaseType: SVEType("Ul", 'd'), .Suffix: "u64"},
437	{.BaseType: SVEType("h", 'd'), .Suffix: "f16"},
438	{.BaseType: SVEType("b", 'd'), .Suffix: "bf16"},
439	{.BaseType: SVEType("f", 'd'), .Suffix: "f32"},
440	{.BaseType: SVEType("d", 'd'), .Suffix: "f64"}}};
441
442	} // end anonymous namespace
443
444	//===----------------------------------------------------------------------===//
445	// Type implementation
446	//===----------------------------------------------------------------------===//
447
448	std::string SVEType::builtinBaseType() const {
449	switch (Kind) {
450	case TypeKind::Void:
451	return "v";
452	case TypeKind::Svcount:
453	return "Qa";
454	case TypeKind::PrefetchOp:
455	case TypeKind::PredicatePattern:
456	return "i";
457	case TypeKind::Fpm:
458	return "UWi";
459	case TypeKind::Predicate:
460	return "b";
461	case TypeKind::BFloat16:
462	assert(ElementBitwidth == 16 && "Invalid BFloat16!");
463	return "y";
464	case TypeKind::MFloat8:
465	assert(ElementBitwidth == 8 && "Invalid MFloat8!");
466	return "m";
467	case TypeKind::Float:
468	switch (ElementBitwidth) {
469	case 16:
470	return "h";
471	case 32:
472	return "f";
473	case 64:
474	return "d";
475	default:
476	llvm_unreachable("Unhandled float width!");
477	}
478	case TypeKind::SInt:
479	case TypeKind::UInt:
480	switch (ElementBitwidth) {
481	case 1:
482	return "b";
483	case 8:
484	return "c";
485	case 16:
486	return "s";
487	case 32:
488	return "i";
489	case 64:
490	return "Wi";
491	case 128:
492	return "LLLi";
493	default:
494	llvm_unreachable("Unhandled bitwidth!");
495	}
496	case TypeKind::Invalid:
497	llvm_unreachable("Attempting to resolve builtin string from Invalid type!");
498	}
499	llvm_unreachable("Unhandled TypeKind!");
500	}
501
502	std::string SVEType::builtin_str() const {
503	std::string Prefix;
504
505	if (isScalableVector())
506	Prefix = "q" + llvm::utostr(X: getNumElements() * NumVectors);
507	else if (isFixedLengthVector())
508	Prefix = "V" + llvm::utostr(X: getNumElements() * NumVectors);
509	else if (isImmediate()) {
510	assert(!isFloatingPoint() && "fp immediates are not supported");
511	Prefix = "I";
512	}
513
514	// Make chars and integer pointers explicitly signed.
515	if ((ElementBitwidth == 8 || isPointer()) && isSignedInteger())
516	Prefix += "S";
517	else if (isUnsignedInteger())
518	Prefix += "U";
519
520	std::string BuiltinStr = Prefix + builtinBaseType();
521	if (isConstant())
522	BuiltinStr += "C";
523	if (isPointer())
524	BuiltinStr += "*";
525
526	return BuiltinStr;
527	}
528
529	std::string SVEType::str() const {
530	std::string TypeStr;
531
532	switch (Kind) {
533	case TypeKind::PrefetchOp:
534	return "enum svprfop";
535	case TypeKind::PredicatePattern:
536	return "enum svpattern";
537	case TypeKind::Fpm:
538	TypeStr += "fpm";
539	break;
540	case TypeKind::Void:
541	TypeStr += "void";
542	break;
543	case TypeKind::Float:
544	TypeStr += "float" + llvm::utostr(X: ElementBitwidth);
545	break;
546	case TypeKind::Svcount:
547	TypeStr += "svcount";
548	break;
549	case TypeKind::Predicate:
550	TypeStr += "bool";
551	break;
552	case TypeKind::BFloat16:
553	TypeStr += "bfloat16";
554	break;
555	case TypeKind::MFloat8:
556	TypeStr += "mfloat8";
557	break;
558	case TypeKind::SInt:
559	TypeStr += "int" + llvm::utostr(X: ElementBitwidth);
560	break;
561	case TypeKind::UInt:
562	TypeStr += "uint" + llvm::utostr(X: ElementBitwidth);
563	break;
564	case TypeKind::Invalid:
565	llvm_unreachable("Attempting to resolve type name from Invalid type!");
566	}
567
568	if (isFixedLengthVector())
569	TypeStr += "x" + llvm::utostr(X: getNumElements());
570	else if (isScalableVector())
571	TypeStr = "sv" + TypeStr;
572
573	if (NumVectors > 1)
574	TypeStr += "x" + llvm::utostr(X: NumVectors);
575	if (!isScalarPredicate() && !isVoid())
576	TypeStr += "_t";
577	if (isConstant())
578	TypeStr += " const";
579	if (isPointer())
580	TypeStr += " *";
581
582	return TypeStr;
583	}
584
585	void SVEType::applyTypespec(StringRef TS) {
586	for (char I : TS) {
587	switch (I) {
588	case 'Q':
589	assert(isInvalid() && "Unexpected use of typespec modifier");
590	Kind = Svcount;
591	break;
592	case 'P':
593	assert(isInvalid() && "Unexpected use of typespec modifier");
594	Kind = Predicate;
595	break;
596	case 'U':
597	assert(isInvalid() && "Unexpected use of typespec modifier");
598	Kind = UInt;
599	break;
600	case 'c':
601	Kind = isInvalid() ? SInt : Kind;
602	ElementBitwidth = 8;
603	break;
604	case 's':
605	Kind = isInvalid() ? SInt : Kind;
606	ElementBitwidth = 16;
607	break;
608	case 'i':
609	Kind = isInvalid() ? SInt : Kind;
610	ElementBitwidth = 32;
611	break;
612	case 'l':
613	Kind = isInvalid() ? SInt : Kind;
614	ElementBitwidth = 64;
615	break;
616	case 'q':
617	Kind = isInvalid() ? SInt : Kind;
618	ElementBitwidth = 128;
619	break;
620	case 'h':
621	assert(isInvalid() && "Unexpected use of typespec modifier");
622	Kind = Float;
623	ElementBitwidth = 16;
624	break;
625	case 'f':
626	assert(isInvalid() && "Unexpected use of typespec modifier");
627	Kind = Float;
628	ElementBitwidth = 32;
629	break;
630	case 'd':
631	assert(isInvalid() && "Unexpected use of typespec modifier");
632	Kind = Float;
633	ElementBitwidth = 64;
634	break;
635	case 'b':
636	assert(isInvalid() && "Unexpected use of typespec modifier");
637	Kind = BFloat16;
638	ElementBitwidth = 16;
639	break;
640	case 'm':
641	assert(isInvalid() && "Unexpected use of typespec modifier");
642	Kind = MFloat8;
643	ElementBitwidth = 8;
644	break;
645	default:
646	llvm_unreachable("Unhandled type code!");
647	}
648	}
649	assert(ElementBitwidth != ~0U && "Bad element bitwidth!");
650	}
651
652	void SVEType::applyModifier(char Mod) {
653	switch (Mod) {
654	case 'v':
655	Kind = Void;
656	NumVectors = 0;
657	break;
658	case 'd':
659	DefaultType = true;
660	break;
661	case 'c':
662	Constant = true;
663	[[fallthrough]];
664	case 'p':
665	Pointer = true;
666	Bitwidth = ElementBitwidth;
667	NumVectors = 0;
668	break;
669	case 'e':
670	Kind = UInt;
671	ElementBitwidth /= 2;
672	break;
673	case 'h':
674	ElementBitwidth /= 2;
675	break;
676	case 'q':
677	ElementBitwidth /= 4;
678	break;
679	case 'b':
680	Kind = UInt;
681	ElementBitwidth /= 4;
682	break;
683	case 'o':
684	ElementBitwidth *= 4;
685	break;
686	case 'P':
687	Kind = Predicate;
688	Bitwidth = 16;
689	ElementBitwidth = 1;
690	break;
691	case '{':
692	IsScalable = false;
693	Bitwidth = 128;
694	NumVectors = 1;
695	break;
696	case 's':
697	case 'a':
698	Bitwidth = ElementBitwidth;
699	NumVectors = 0;
700	break;
701	case 'R':
702	ElementBitwidth /= 2;
703	NumVectors = 0;
704	break;
705	case 'r':
706	ElementBitwidth /= 4;
707	NumVectors = 0;
708	break;
709	case '@':
710	Kind = UInt;
711	ElementBitwidth /= 4;
712	NumVectors = 0;
713	break;
714	case 'K':
715	Kind = SInt;
716	Bitwidth = ElementBitwidth;
717	NumVectors = 0;
718	break;
719	case 'L':
720	Kind = UInt;
721	Bitwidth = ElementBitwidth;
722	NumVectors = 0;
723	break;
724	case 'u':
725	Kind = UInt;
726	break;
727	case 'x':
728	Kind = SInt;
729	break;
730	case 'i':
731	Kind = UInt;
732	ElementBitwidth = Bitwidth = 64;
733	NumVectors = 0;
734	Immediate = true;
735	break;
736	case 'I':
737	Kind = PredicatePattern;
738	ElementBitwidth = Bitwidth = 32;
739	NumVectors = 0;
740	Immediate = true;
741	break;
742	case 'J':
743	Kind = PrefetchOp;
744	ElementBitwidth = Bitwidth = 32;
745	NumVectors = 0;
746	Immediate = true;
747	break;
748	case 'k':
749	Kind = SInt;
750	ElementBitwidth = Bitwidth = 32;
751	NumVectors = 0;
752	break;
753	case 'l':
754	Kind = SInt;
755	ElementBitwidth = Bitwidth = 64;
756	NumVectors = 0;
757	break;
758	case 'm':
759	Kind = UInt;
760	ElementBitwidth = Bitwidth = 32;
761	NumVectors = 0;
762	break;
763	case '>':
764	Kind = Fpm;
765	ElementBitwidth = Bitwidth = 64;
766	NumVectors = 0;
767	break;
768	case 'n':
769	Kind = UInt;
770	ElementBitwidth = Bitwidth = 64;
771	NumVectors = 0;
772	break;
773	case 'w':
774	ElementBitwidth = 64;
775	break;
776	case 'j':
777	ElementBitwidth = Bitwidth = 64;
778	NumVectors = 0;
779	break;
780	case 'f':
781	Kind = UInt;
782	ElementBitwidth = Bitwidth = 64;
783	NumVectors = 0;
784	break;
785	case 'g':
786	Kind = UInt;
787	ElementBitwidth = 64;
788	break;
789	case '#':
790	Kind = SInt;
791	ElementBitwidth = 64;
792	break;
793	case '[':
794	Kind = UInt;
795	ElementBitwidth = 8;
796	break;
797	case 't':
798	Kind = SInt;
799	ElementBitwidth = 32;
800	break;
801	case 'z':
802	Kind = UInt;
803	ElementBitwidth = 32;
804	break;
805	case 'O':
806	Kind = Float;
807	ElementBitwidth = 16;
808	break;
809	case 'M':
810	Kind = Float;
811	ElementBitwidth = 32;
812	break;
813	case 'N':
814	Kind = Float;
815	ElementBitwidth = 64;
816	break;
817	case 'Q':
818	Kind = Void;
819	Constant = true;
820	Pointer = true;
821	NumVectors = 0;
822	break;
823	case 'S':
824	Kind = SInt;
825	Constant = true;
826	Pointer = true;
827	ElementBitwidth = Bitwidth = 8;
828	NumVectors = 0;
829	break;
830	case 'W':
831	Kind = UInt;
832	Constant = true;
833	Pointer = true;
834	ElementBitwidth = Bitwidth = 8;
835	NumVectors = 0;
836	break;
837	case 'T':
838	Kind = SInt;
839	Constant = true;
840	Pointer = true;
841	ElementBitwidth = Bitwidth = 16;
842	NumVectors = 0;
843	break;
844	case 'X':
845	Kind = UInt;
846	Constant = true;
847	Pointer = true;
848	ElementBitwidth = Bitwidth = 16;
849	NumVectors = 0;
850	break;
851	case 'Y':
852	Kind = UInt;
853	Constant = true;
854	Pointer = true;
855	ElementBitwidth = Bitwidth = 32;
856	NumVectors = 0;
857	break;
858	case 'U':
859	Kind = SInt;
860	Constant = true;
861	Pointer = true;
862	ElementBitwidth = Bitwidth = 32;
863	NumVectors = 0;
864	break;
865	case '%':
866	Kind = Void;
867	Pointer = true;
868	NumVectors = 0;
869	break;
870	case 'A':
871	Kind = SInt;
872	Pointer = true;
873	ElementBitwidth = Bitwidth = 8;
874	NumVectors = 0;
875	break;
876	case 'B':
877	Kind = SInt;
878	Pointer = true;
879	ElementBitwidth = Bitwidth = 16;
880	NumVectors = 0;
881	break;
882	case 'C':
883	Kind = SInt;
884	Pointer = true;
885	ElementBitwidth = Bitwidth = 32;
886	NumVectors = 0;
887	break;
888	case 'D':
889	Kind = SInt;
890	Pointer = true;
891	ElementBitwidth = Bitwidth = 64;
892	NumVectors = 0;
893	break;
894	case 'E':
895	Kind = UInt;
896	Pointer = true;
897	ElementBitwidth = Bitwidth = 8;
898	NumVectors = 0;
899	break;
900	case 'F':
901	Kind = UInt;
902	Pointer = true;
903	ElementBitwidth = Bitwidth = 16;
904	NumVectors = 0;
905	break;
906	case 'G':
907	Kind = UInt;
908	Pointer = true;
909	ElementBitwidth = Bitwidth = 32;
910	NumVectors = 0;
911	break;
912	case '$':
913	Kind = BFloat16;
914	ElementBitwidth = 16;
915	break;
916	case '}':
917	Kind = Svcount;
918	NumVectors = 0;
919	break;
920	case '~':
921	Kind = MFloat8;
922	ElementBitwidth = 8;
923	break;
924	case '!':
925	Kind = MFloat8;
926	Bitwidth = ElementBitwidth = 8;
927	NumVectors = 0;
928	break;
929	case '.':
930	llvm_unreachable(". is never a type in itself");
931	break;
932	default:
933	llvm_unreachable("Unhandled character!");
934	}
935	}
936
937	/// Returns the modifier and number of vectors for the given operand \p Op.
938	std::pair<char, unsigned> getProtoModifier(StringRef Proto, unsigned Op) {
939	for (unsigned P = 0; !Proto.empty(); ++P) {
940	unsigned NumVectors = 1;
941	unsigned CharsToSkip = 1;
942	char Mod = Proto[0];
943	if (Mod == '2' || Mod == '3' || Mod == '4') {
944	NumVectors = Mod - '0';
945	Mod = 'd';
946	if (Proto.size() > 1 && Proto[1] == '.') {
947	Mod = Proto[2];
948	CharsToSkip = 3;
949	}
950	}
951
952	if (P == Op)
953	return {Mod, NumVectors};
954
955	Proto = Proto.drop_front(N: CharsToSkip);
956	}
957	llvm_unreachable("Unexpected Op");
958	}
959
960	//===----------------------------------------------------------------------===//
961	// Intrinsic implementation
962	//===----------------------------------------------------------------------===//
963
964	Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
965	StringRef MergeSuffix, uint64_t MemoryElementTy,
966	StringRef LLVMName, uint64_t Flags,
967	ArrayRef<ImmCheck> Checks, TypeSpec BT, ClassKind Class,
968	SVEEmitter &Emitter, StringRef SVEGuard,
969	StringRef SMEGuard)
970	: Name(Name.str()), LLVMName(LLVMName), Proto(Proto.str()),
971	BaseTypeSpec(BT), Class(Class), MergeSuffix(MergeSuffix.str()),
972	BaseType(BT, 'd'), Flags(Flags), ImmChecks(Checks) {
973
974	auto FormatGuard = [](StringRef Guard, StringRef Base) -> std::string {
975	if (Guard.contains(C: '|'))
976	return Base.str() + ",(" + Guard.str() + ")";
977	if (Guard.empty() || Guard == Base || Guard.starts_with(Prefix: Base.str() + ","))
978	return Guard.str();
979	return Base.str() + "," + Guard.str();
980	};
981
982	this->SVEGuard = FormatGuard(SVEGuard, "sve");
983	this->SMEGuard = FormatGuard(SMEGuard, "sme");
984
985	// Types[0] is the return value.
986	for (unsigned I = 0; I < (getNumParams() + 1); ++I) {
987	char Mod;
988	unsigned NumVectors;
989	std::tie(args&: Mod, args&: NumVectors) = getProtoModifier(Proto, Op: I);
990	SVEType T(BaseTypeSpec, Mod, NumVectors);
991	Types.push_back(x: T);
992	SetsFPMR = T.isFpm();
993
994	// Add range checks for immediates
995	if (I > 0) {
996	if (T.isPredicatePattern())
997	ImmChecks.emplace_back(
998	Args: I - 1, Args: Emitter.getEnumValueForImmCheck(C: "ImmCheck0_31"));
999	else if (T.isPrefetchOp())
1000	ImmChecks.emplace_back(
1001	Args: I - 1, Args: Emitter.getEnumValueForImmCheck(C: "ImmCheck0_13"));
1002	}
1003	}
1004
1005	// Set flags based on properties
1006	this->Flags |= Emitter.encodeTypeFlags(T: BaseType);
1007	this->Flags |= Emitter.encodeMemoryElementType(MT: MemoryElementTy);
1008	this->Flags |= Emitter.encodeMergeType(MT: MergeTy);
1009	if (hasSplat())
1010	this->Flags |= Emitter.encodeSplatOperand(SplatIdx: getSplatIdx());
1011	if (SetsFPMR)
1012	this->Flags |= Emitter.getEnumValueForFlag(C: "SetsFPMR");
1013	}
1014
1015	std::string Intrinsic::getBuiltinTypeStr() {
1016	std::string S = getReturnType().builtin_str();
1017	for (unsigned I = 0; I < getNumParams(); ++I)
1018	S += getParamType(I).builtin_str();
1019
1020	return S;
1021	}
1022
1023	std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS,
1024	std::string Proto) const {
1025	std::string Ret = Name;
1026	while (Ret.find(c: '{') != std::string::npos) {
1027	size_t Pos = Ret.find(c: '{');
1028	size_t End = Ret.find(c: '}');
1029	unsigned NumChars = End - Pos + 1;
1030	assert(NumChars == 3 && "Unexpected template argument");
1031
1032	SVEType T;
1033	char C = Ret[Pos+1];
1034	switch(C) {
1035	default:
1036	llvm_unreachable("Unknown predication specifier");
1037	case 'd':
1038	T = SVEType(TS, 'd');
1039	break;
1040	case '0':
1041	case '1':
1042	case '2':
1043	case '3':
1044	// Extract the modifier before passing to SVEType to handle numeric
1045	// modifiers
1046	auto [Mod, NumVectors] = getProtoModifier(Proto, Op: (C - '0'));
1047	T = SVEType(TS, Mod);
1048	break;
1049	}
1050
1051	// Replace templated arg with the right suffix (e.g. u32)
1052	std::string TypeCode;
1053
1054	if (T.isSignedInteger())
1055	TypeCode = 's';
1056	else if (T.isUnsignedInteger())
1057	TypeCode = 'u';
1058	else if (T.isSvcount())
1059	TypeCode = 'c';
1060	else if (T.isPredicate())
1061	TypeCode = 'b';
1062	else if (T.isBFloat())
1063	TypeCode = "bf";
1064	else if (T.isMFloat())
1065	TypeCode = "mf";
1066	else
1067	TypeCode = 'f';
1068	Ret.replace(pos: Pos, n: NumChars, str: TypeCode + utostr(X: T.getElementSizeInBits()));
1069	}
1070
1071	return Ret;
1072	}
1073
1074	std::string Intrinsic::mangleLLVMName() const {
1075	std::string S = getLLVMName();
1076
1077	// Replace all {d} like expressions with e.g. 'u32'
1078	return replaceTemplatedArgs(Name: S, TS: getBaseTypeSpec(), Proto: getProto());
1079	}
1080
1081	std::string Intrinsic::mangleName(ClassKind LocalCK) const {
1082	std::string S = getName();
1083
1084	if (LocalCK == ClassG) {
1085	// Remove the square brackets and everything in between.
1086	while (S.find(c: '[') != std::string::npos) {
1087	auto Start = S.find(c: '[');
1088	auto End = S.find(c: ']');
1089	S.erase(pos: Start, n: (End-Start)+1);
1090	}
1091	} else {
1092	// Remove the square brackets.
1093	while (S.find(c: '[') != std::string::npos) {
1094	auto BrPos = S.find(c: '[');
1095	if (BrPos != std::string::npos)
1096	S.erase(pos: BrPos, n: 1);
1097	BrPos = S.find(c: ']');
1098	if (BrPos != std::string::npos)
1099	S.erase(pos: BrPos, n: 1);
1100	}
1101	}
1102
1103	// Replace all {d} like expressions with e.g. 'u32'
1104	return replaceTemplatedArgs(Name: S, TS: getBaseTypeSpec(), Proto: getProto())
1105	.append(str: getMergeSuffix())
1106	.append(svt: getFPMSuffix());
1107	}
1108
1109	void Intrinsic::emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter,
1110	ACLEKind Kind) const {
1111	bool IsOverloaded = getClassKind() == ClassG && getProto().size() > 1;
1112
1113	std::string FullName = mangleName(LocalCK: ClassS);
1114	std::string ProtoName = mangleName(LocalCK: getClassKind());
1115	OS << (IsOverloaded ? "__aio " : "__ai ")
1116	<< "__attribute__((__clang_arm_builtin_alias(";
1117
1118	switch (Kind) {
1119	case ACLEKind::SME:
1120	OS << "__builtin_sme_" << FullName << ")";
1121	break;
1122	case ACLEKind::SVE:
1123	OS << "__builtin_sve_" << FullName << ")";
1124	break;
1125	}
1126
1127	OS << "))\n";
1128
1129	OS << getTypes()[0].str() << " " << ProtoName << "(";
1130	for (unsigned I = 0; I < getTypes().size() - 1; ++I) {
1131	if (I != 0)
1132	OS << ", ";
1133	OS << getTypes()[I + 1].str();
1134	}
1135	OS << ");\n";
1136	}
1137
1138	//===----------------------------------------------------------------------===//
1139	// SVEEmitter implementation
1140	//===----------------------------------------------------------------------===//
1141	uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) {
1142	if (T.isFloat()) {
1143	switch (T.getElementSizeInBits()) {
1144	case 16:
1145	return encodeEltType(EltName: "EltTyFloat16");
1146	case 32:
1147	return encodeEltType(EltName: "EltTyFloat32");
1148	case 64:
1149	return encodeEltType(EltName: "EltTyFloat64");
1150	default:
1151	llvm_unreachable("Unhandled float element bitwidth!");
1152	}
1153	}
1154
1155	if (T.isBFloat()) {
1156	assert(T.getElementSizeInBits() == 16 && "Not a valid BFloat.");
1157	return encodeEltType(EltName: "EltTyBFloat16");
1158	}
1159
1160	if (T.isMFloat()) {
1161	assert(T.getElementSizeInBits() == 8 && "Not a valid MFloat.");
1162	return encodeEltType(EltName: "EltTyMFloat8");
1163	}
1164
1165	if (T.isPredicate() || T.isSvcount()) {
1166	switch (T.getElementSizeInBits()) {
1167	case 8:
1168	return encodeEltType(EltName: "EltTyBool8");
1169	case 16:
1170	return encodeEltType(EltName: "EltTyBool16");
1171	case 32:
1172	return encodeEltType(EltName: "EltTyBool32");
1173	case 64:
1174	return encodeEltType(EltName: "EltTyBool64");
1175	default:
1176	llvm_unreachable("Unhandled predicate element bitwidth!");
1177	}
1178	}
1179
1180	switch (T.getElementSizeInBits()) {
1181	case 8:
1182	return encodeEltType(EltName: "EltTyInt8");
1183	case 16:
1184	return encodeEltType(EltName: "EltTyInt16");
1185	case 32:
1186	return encodeEltType(EltName: "EltTyInt32");
1187	case 64:
1188	return encodeEltType(EltName: "EltTyInt64");
1189	case 128:
1190	return encodeEltType(EltName: "EltTyInt128");
1191	default:
1192	llvm_unreachable("Unhandled integer element bitwidth!");
1193	}
1194	}
1195
1196	void SVEEmitter::createIntrinsic(
1197	const Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out) {
1198	StringRef Name = R->getValueAsString(FieldName: "Name");
1199	StringRef Proto = R->getValueAsString(FieldName: "Prototype");
1200	StringRef Types = R->getValueAsString(FieldName: "Types");
1201	StringRef SVEGuard = R->getValueAsString(FieldName: "SVETargetGuard");
1202	StringRef SMEGuard = R->getValueAsString(FieldName: "SMETargetGuard");
1203	StringRef LLVMName = R->getValueAsString(FieldName: "LLVMIntrinsic");
1204	uint64_t Merge = R->getValueAsInt(FieldName: "Merge");
1205	StringRef MergeSuffix = R->getValueAsString(FieldName: "MergeSuffix");
1206	uint64_t MemEltType = R->getValueAsInt(FieldName: "MemEltType");
1207
1208	int64_t Flags = 0;
1209	for (const Record *FlagRec : R->getValueAsListOfDefs(FieldName: "Flags"))
1210	Flags |= FlagRec->getValueAsInt(FieldName: "Value");
1211
1212	// Create a dummy TypeSpec for non-overloaded builtins.
1213	if (Types.empty()) {
1214	assert((Flags & getEnumValueForFlag("IsOverloadNone")) &&
1215	"Expect TypeSpec for overloaded builtin!");
1216	Types = "i";
1217	}
1218
1219	// Extract type specs from string
1220	SmallVector<TypeSpec, 8> TypeSpecs;
1221	TypeSpec Acc;
1222	for (char I : Types) {
1223	Acc.push_back(c: I);
1224	if (islower(I)) {
1225	TypeSpecs.push_back(Elt: TypeSpec(Acc));
1226	Acc.clear();
1227	}
1228	}
1229
1230	// Remove duplicate type specs.
1231	sort(C&: TypeSpecs);
1232	TypeSpecs.erase(CS: llvm::unique(R&: TypeSpecs), CE: TypeSpecs.end());
1233
1234	// Create an Intrinsic for each type spec.
1235	for (auto TS : TypeSpecs) {
1236	// Collate a list of range/option checks for the immediates.
1237	SmallVector<ImmCheck, 2> ImmChecks;
1238	for (const Record *ImmR : R->getValueAsListOfDefs(FieldName: "ImmChecks")) {
1239	int64_t ArgIdx = ImmR->getValueAsInt(FieldName: "ImmArgIdx");
1240	int64_t EltSizeArgIdx = ImmR->getValueAsInt(FieldName: "TypeContextArgIdx");
1241	int64_t Kind = ImmR->getValueAsDef(FieldName: "Kind")->getValueAsInt(FieldName: "Value");
1242	assert(ArgIdx >= 0 && Kind >= 0 &&
1243	"ImmArgIdx and Kind must be nonnegative");
1244
1245	unsigned ElementSizeInBits = 0;
1246	auto [Mod, NumVectors] = getProtoModifier(Proto, Op: EltSizeArgIdx + 1);
1247	if (EltSizeArgIdx >= 0)
1248	ElementSizeInBits = SVEType(TS, Mod, NumVectors).getElementSizeInBits();
1249	ImmChecks.push_back(Elt: ImmCheck(ArgIdx, Kind, ElementSizeInBits));
1250	}
1251
1252	Out.push_back(Elt: std::make_unique<Intrinsic>(
1253	args&: Name, args&: Proto, args&: Merge, args&: MergeSuffix, args&: MemEltType, args&: LLVMName, args&: Flags, args&: ImmChecks,
1254	args&: TS, args: ClassS, args&: *this, args&: SVEGuard, args&: SMEGuard));
1255
1256	// Also generate the short-form (e.g. svadd_m) for the given type-spec.
1257	if (Intrinsic::isOverloadedIntrinsic(Name))
1258	Out.push_back(Elt: std::make_unique<Intrinsic>(
1259	args&: Name, args&: Proto, args&: Merge, args&: MergeSuffix, args&: MemEltType, args&: LLVMName, args&: Flags,
1260	args&: ImmChecks, args&: TS, args: ClassG, args&: *this, args&: SVEGuard, args&: SMEGuard));
1261	}
1262	}
1263
1264	void SVEEmitter::createCoreHeaderIntrinsics(raw_ostream &OS,
1265	SVEEmitter &Emitter,
1266	ACLEKind Kind) {
1267	SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1268	std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1269	for (auto *R : RV)
1270	createIntrinsic(R, Out&: Defs);
1271
1272	// Sort intrinsics in header file by following order/priority:
1273	// - Architectural guard (i.e. does it require SVE2 or SVE2_AES)
1274	// - Class (is intrinsic overloaded or not)
1275	// - Intrinsic name
1276	llvm::stable_sort(Range&: Defs, C: [](const std::unique_ptr<Intrinsic> &A,
1277	const std::unique_ptr<Intrinsic> &B) {
1278	auto ToTuple = [](const std::unique_ptr<Intrinsic> &I) {
1279	return std::make_tuple(args: I->getSVEGuard().str() + I->getSMEGuard().str(),
1280	args: (unsigned)I->getClassKind(), args: I->getName());
1281	};
1282	return ToTuple(A) < ToTuple(B);
1283	});
1284
1285	// Actually emit the intrinsic declarations.
1286	for (auto &I : Defs)
1287	I->emitIntrinsic(OS, Emitter, Kind);
1288	}
1289
1290	void SVEEmitter::createHeader(raw_ostream &OS) {
1291	OS << "/*===---- arm_sve.h - ARM SVE intrinsics "
1292	"-----------------------------------===\n"
1293	" *\n"
1294	" *\n"
1295	" * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1296	"Exceptions.\n"
1297	" * See https://llvm.org/LICENSE.txt for license information.\n"
1298	" * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1299	" *\n"
1300	" *===-----------------------------------------------------------------"
1301	"------===\n"
1302	" */\n\n";
1303
1304	OS << "#ifndef __ARM_SVE_H\n";
1305	OS << "#define __ARM_SVE_H\n\n";
1306
1307	OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1308	OS << "#error \"Big endian is currently not supported for arm_sve.h\"\n";
1309	OS << "#endif\n";
1310
1311	OS << "#include <stdint.h>\n\n";
1312	OS << "#ifdef __cplusplus\n";
1313	OS << "extern \"C\" {\n";
1314	OS << "#else\n";
1315	OS << "#include <stdbool.h>\n";
1316	OS << "#endif\n\n";
1317
1318	OS << "typedef __fp16 float16_t;\n";
1319	OS << "typedef float float32_t;\n";
1320	OS << "typedef double float64_t;\n";
1321
1322	OS << "typedef __SVInt8_t svint8_t;\n";
1323	OS << "typedef __SVInt16_t svint16_t;\n";
1324	OS << "typedef __SVInt32_t svint32_t;\n";
1325	OS << "typedef __SVInt64_t svint64_t;\n";
1326	OS << "typedef __SVUint8_t svuint8_t;\n";
1327	OS << "typedef __SVUint16_t svuint16_t;\n";
1328	OS << "typedef __SVUint32_t svuint32_t;\n";
1329	OS << "typedef __SVUint64_t svuint64_t;\n";
1330	OS << "typedef __SVFloat16_t svfloat16_t;\n\n";
1331
1332	OS << "typedef __SVBfloat16_t svbfloat16_t;\n";
1333
1334	OS << "#include <arm_bf16.h>\n";
1335	OS << "#include <arm_vector_types.h>\n";
1336
1337	OS << "typedef __SVMfloat8_t svmfloat8_t;\n\n";
1338
1339	OS << "typedef __SVFloat32_t svfloat32_t;\n";
1340	OS << "typedef __SVFloat64_t svfloat64_t;\n";
1341	OS << "typedef __clang_svint8x2_t svint8x2_t;\n";
1342	OS << "typedef __clang_svint16x2_t svint16x2_t;\n";
1343	OS << "typedef __clang_svint32x2_t svint32x2_t;\n";
1344	OS << "typedef __clang_svint64x2_t svint64x2_t;\n";
1345	OS << "typedef __clang_svuint8x2_t svuint8x2_t;\n";
1346	OS << "typedef __clang_svuint16x2_t svuint16x2_t;\n";
1347	OS << "typedef __clang_svuint32x2_t svuint32x2_t;\n";
1348	OS << "typedef __clang_svuint64x2_t svuint64x2_t;\n";
1349	OS << "typedef __clang_svfloat16x2_t svfloat16x2_t;\n";
1350	OS << "typedef __clang_svfloat32x2_t svfloat32x2_t;\n";
1351	OS << "typedef __clang_svfloat64x2_t svfloat64x2_t;\n";
1352	OS << "typedef __clang_svint8x3_t svint8x3_t;\n";
1353	OS << "typedef __clang_svint16x3_t svint16x3_t;\n";
1354	OS << "typedef __clang_svint32x3_t svint32x3_t;\n";
1355	OS << "typedef __clang_svint64x3_t svint64x3_t;\n";
1356	OS << "typedef __clang_svuint8x3_t svuint8x3_t;\n";
1357	OS << "typedef __clang_svuint16x3_t svuint16x3_t;\n";
1358	OS << "typedef __clang_svuint32x3_t svuint32x3_t;\n";
1359	OS << "typedef __clang_svuint64x3_t svuint64x3_t;\n";
1360	OS << "typedef __clang_svfloat16x3_t svfloat16x3_t;\n";
1361	OS << "typedef __clang_svfloat32x3_t svfloat32x3_t;\n";
1362	OS << "typedef __clang_svfloat64x3_t svfloat64x3_t;\n";
1363	OS << "typedef __clang_svint8x4_t svint8x4_t;\n";
1364	OS << "typedef __clang_svint16x4_t svint16x4_t;\n";
1365	OS << "typedef __clang_svint32x4_t svint32x4_t;\n";
1366	OS << "typedef __clang_svint64x4_t svint64x4_t;\n";
1367	OS << "typedef __clang_svuint8x4_t svuint8x4_t;\n";
1368	OS << "typedef __clang_svuint16x4_t svuint16x4_t;\n";
1369	OS << "typedef __clang_svuint32x4_t svuint32x4_t;\n";
1370	OS << "typedef __clang_svuint64x4_t svuint64x4_t;\n";
1371	OS << "typedef __clang_svfloat16x4_t svfloat16x4_t;\n";
1372	OS << "typedef __clang_svfloat32x4_t svfloat32x4_t;\n";
1373	OS << "typedef __clang_svfloat64x4_t svfloat64x4_t;\n";
1374	OS << "typedef __SVBool_t svbool_t;\n";
1375	OS << "typedef __clang_svboolx2_t svboolx2_t;\n";
1376	OS << "typedef __clang_svboolx4_t svboolx4_t;\n\n";
1377
1378	OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n";
1379	OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n";
1380	OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n";
1381
1382	OS << "typedef __clang_svmfloat8x2_t svmfloat8x2_t;\n";
1383	OS << "typedef __clang_svmfloat8x3_t svmfloat8x3_t;\n";
1384	OS << "typedef __clang_svmfloat8x4_t svmfloat8x4_t;\n";
1385
1386	OS << "typedef __SVCount_t svcount_t;\n\n";
1387
1388	OS << "enum svpattern\n";
1389	OS << "{\n";
1390	OS << " SV_POW2 = 0,\n";
1391	OS << " SV_VL1 = 1,\n";
1392	OS << " SV_VL2 = 2,\n";
1393	OS << " SV_VL3 = 3,\n";
1394	OS << " SV_VL4 = 4,\n";
1395	OS << " SV_VL5 = 5,\n";
1396	OS << " SV_VL6 = 6,\n";
1397	OS << " SV_VL7 = 7,\n";
1398	OS << " SV_VL8 = 8,\n";
1399	OS << " SV_VL16 = 9,\n";
1400	OS << " SV_VL32 = 10,\n";
1401	OS << " SV_VL64 = 11,\n";
1402	OS << " SV_VL128 = 12,\n";
1403	OS << " SV_VL256 = 13,\n";
1404	OS << " SV_MUL4 = 29,\n";
1405	OS << " SV_MUL3 = 30,\n";
1406	OS << " SV_ALL = 31\n";
1407	OS << "};\n\n";
1408
1409	OS << "enum svprfop\n";
1410	OS << "{\n";
1411	OS << " SV_PLDL1KEEP = 0,\n";
1412	OS << " SV_PLDL1STRM = 1,\n";
1413	OS << " SV_PLDL2KEEP = 2,\n";
1414	OS << " SV_PLDL2STRM = 3,\n";
1415	OS << " SV_PLDL3KEEP = 4,\n";
1416	OS << " SV_PLDL3STRM = 5,\n";
1417	OS << " SV_PSTL1KEEP = 8,\n";
1418	OS << " SV_PSTL1STRM = 9,\n";
1419	OS << " SV_PSTL2KEEP = 10,\n";
1420	OS << " SV_PSTL2STRM = 11,\n";
1421	OS << " SV_PSTL3KEEP = 12,\n";
1422	OS << " SV_PSTL3STRM = 13\n";
1423	OS << "};\n\n";
1424
1425	OS << "/* Function attributes */\n";
1426	OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1427	"__nodebug__))\n\n";
1428	OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1429	"__nodebug__, __overloadable__))\n\n";
1430
1431	// Add reinterpret functions.
1432	for (auto [N, Suffix] :
1433	std::initializer_list<std::pair<unsigned, const char *>>{
1434	{1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
1435	for (auto ShortForm : {false, true})
1436	for (const ReinterpretTypeInfo &To : Reinterprets) {
1437	SVEType ToV(To.BaseType, N);
1438	for (const ReinterpretTypeInfo &From : Reinterprets) {
1439	SVEType FromV(From.BaseType, N);
1440	OS << "__aio "
1441	"__attribute__((__clang_arm_builtin_alias(__builtin_sve_"
1442	"reinterpret_"
1443	<< To.Suffix << "_" << From.Suffix << Suffix << ")))\n"
1444	<< ToV.str() << " svreinterpret_" << To.Suffix;
1445	if (!ShortForm)
1446	OS << "_" << From.Suffix << Suffix;
1447	OS << "(" << FromV.str() << " op);\n";
1448	}
1449	}
1450	}
1451
1452	createCoreHeaderIntrinsics(OS, Emitter&: *this, Kind: ACLEKind::SVE);
1453
1454	OS << "#define svcvtnt_bf16_x svcvtnt_bf16_m\n";
1455	OS << "#define svcvtnt_bf16_f32_x svcvtnt_bf16_f32_m\n";
1456
1457	OS << "#define svcvtnt_f16_x svcvtnt_f16_m\n";
1458	OS << "#define svcvtnt_f16_f32_x svcvtnt_f16_f32_m\n";
1459	OS << "#define svcvtnt_f32_x svcvtnt_f32_m\n";
1460	OS << "#define svcvtnt_f32_f64_x svcvtnt_f32_f64_m\n\n";
1461
1462	OS << "#define svcvtxnt_f32_x svcvtxnt_f32_m\n";
1463	OS << "#define svcvtxnt_f32_f64_x svcvtxnt_f32_f64_m\n\n";
1464
1465	OS << "#ifdef __cplusplus\n";
1466	OS << "} // extern \"C\"\n";
1467	OS << "#endif\n\n";
1468	OS << "#undef __ai\n\n";
1469	OS << "#undef __aio\n\n";
1470	OS << "#endif /* __ARM_SVE_H */\n";
1471	}
1472
1473	void SVEEmitter::createBuiltins(raw_ostream &OS) {
1474	std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1475	SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1476	for (auto *R : RV)
1477	createIntrinsic(R, Out&: Defs);
1478
1479	// The mappings must be sorted based on BuiltinID.
1480	sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1481	const std::unique_ptr<Intrinsic> &B) {
1482	return A->getMangledName() < B->getMangledName();
1483	});
1484
1485	llvm::StringToOffsetTable Table;
1486	Table.GetOrAddStringOffset(Str: "");
1487	Table.GetOrAddStringOffset(Str: "n");
1488
1489	for (const auto &Def : Defs)
1490	if (Def->getClassKind() != ClassG) {
1491	Table.GetOrAddStringOffset(Str: Def->getMangledName());
1492	Table.GetOrAddStringOffset(Str: Def->getBuiltinTypeStr());
1493	Table.GetOrAddStringOffset(Str: Def->getGuard());
1494	}
1495
1496	Table.GetOrAddStringOffset(Str: "sme|sve");
1497	SmallVector<std::pair<std::string, std::string>> ReinterpretBuiltins;
1498	for (auto [N, Suffix] :
1499	std::initializer_list<std::pair<unsigned, const char *>>{
1500	{1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
1501	for (const ReinterpretTypeInfo &To : Reinterprets) {
1502	SVEType ToV(To.BaseType, N);
1503	for (const ReinterpretTypeInfo &From : Reinterprets) {
1504	SVEType FromV(From.BaseType, N);
1505	std::string Name =
1506	(Twine("reinterpret_") + To.Suffix + "_" + From.Suffix + Suffix)
1507	.str();
1508	std::string Type = ToV.builtin_str() + FromV.builtin_str();
1509	Table.GetOrAddStringOffset(Str: Name);
1510	Table.GetOrAddStringOffset(Str: Type);
1511	ReinterpretBuiltins.push_back(Elt: {Name, Type});
1512	}
1513	}
1514	}
1515
1516	OS << "#ifdef GET_SVE_BUILTIN_ENUMERATORS\n";
1517	for (const auto &Def : Defs)
1518	if (Def->getClassKind() != ClassG)
1519	OS << " BI__builtin_sve_" << Def->getMangledName() << ",\n";
1520	for (const auto &[Name, _] : ReinterpretBuiltins)
1521	OS << " BI__builtin_sve_" << Name << ",\n";
1522	OS << "#endif // GET_SVE_BUILTIN_ENUMERATORS\n\n";
1523
1524	OS << "#ifdef GET_SVE_BUILTIN_STR_TABLE\n";
1525	Table.EmitStringTableDef(OS, Name: "BuiltinStrings");
1526	OS << "#endif // GET_SVE_BUILTIN_STR_TABLE\n\n";
1527
1528	OS << "#ifdef GET_SVE_BUILTIN_INFOS\n";
1529	for (const auto &Def : Defs) {
1530	// Only create BUILTINs for non-overloaded intrinsics, as overloaded
1531	// declarations only live in the header file.
1532	if (Def->getClassKind() != ClassG) {
1533	OS << " Builtin::Info{Builtin::Info::StrOffsets{"
1534	<< Table.GetStringOffset(Str: Def->getMangledName()) << " /* "
1535	<< Def->getMangledName() << " */, ";
1536	OS << Table.GetStringOffset(Str: Def->getBuiltinTypeStr()) << " /* "
1537	<< Def->getBuiltinTypeStr() << " */, ";
1538	OS << Table.GetStringOffset(Str: "n") << " /* n */, ";
1539	OS << Table.GetStringOffset(Str: Def->getGuard()) << " /* " << Def->getGuard()
1540	<< " */}, ";
1541	OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
1542	}
1543	}
1544	for (const auto &[Name, Type] : ReinterpretBuiltins) {
1545	OS << " Builtin::Info{Builtin::Info::StrOffsets{"
1546	<< Table.GetStringOffset(Str: Name) << " /* " << Name << " */, ";
1547	OS << Table.GetStringOffset(Str: Type) << " /* " << Type << " */, ";
1548	OS << Table.GetStringOffset(Str: "n") << " /* n */, ";
1549	OS << Table.GetStringOffset(Str: "sme|sve") << " /* sme|sve */}, ";
1550	OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
1551	}
1552	OS << "#endif // GET_SVE_BUILTIN_INFOS\n\n";
1553	}
1554
1555	void SVEEmitter::createCodeGenMap(raw_ostream &OS) {
1556	std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1557	SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1558	for (auto *R : RV)
1559	createIntrinsic(R, Out&: Defs);
1560
1561	// The mappings must be sorted based on BuiltinID.
1562	sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1563	const std::unique_ptr<Intrinsic> &B) {
1564	return A->getMangledName() < B->getMangledName();
1565	});
1566
1567	OS << "#ifdef GET_SVE_LLVM_INTRINSIC_MAP\n";
1568	for (auto &Def : Defs) {
1569	// Builtins only exist for non-overloaded intrinsics, overloaded
1570	// declarations only live in the header file.
1571	if (Def->getClassKind() == ClassG)
1572	continue;
1573
1574	uint64_t Flags = Def->getFlags();
1575	auto FlagString = std::to_string(val: Flags);
1576
1577	std::string LLVMName = Def->getMangledLLVMName();
1578	std::string Builtin = Def->getMangledName();
1579	if (!LLVMName.empty())
1580	OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1581	<< "),\n";
1582	else
1583	OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n";
1584	}
1585	OS << "#endif\n\n";
1586	}
1587
1588	void SVEEmitter::createRangeChecks(raw_ostream &OS) {
1589	std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1590	SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1591	for (auto *R : RV)
1592	createIntrinsic(R, Out&: Defs);
1593
1594	// The mappings must be sorted based on BuiltinID.
1595	sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1596	const std::unique_ptr<Intrinsic> &B) {
1597	return A->getMangledName() < B->getMangledName();
1598	});
1599
1600	OS << "#ifdef GET_SVE_IMMEDIATE_CHECK\n";
1601
1602	// Ensure these are only emitted once.
1603	std::set<std::string> Emitted;
1604
1605	for (auto &Def : Defs) {
1606	if (Emitted.find(x: Def->getMangledName()) != Emitted.end() ||
1607	Def->getImmChecks().empty())
1608	continue;
1609
1610	OS << "case SVE::BI__builtin_sve_" << Def->getMangledName() << ":\n";
1611	for (auto &Check : Def->getImmChecks())
1612	OS << "ImmChecks.emplace_back(" << Check.getImmArgIdx() << ", "
1613	<< Check.getKind() << ", " << Check.getElementSizeInBits() << ");\n";
1614	OS << " break;\n";
1615
1616	Emitted.insert(x: Def->getMangledName());
1617	}
1618
1619	OS << "#endif\n\n";
1620	}
1621
1622	/// Create the SVETypeFlags used in CGBuiltins
1623	void SVEEmitter::createTypeFlags(raw_ostream &OS) {
1624	OS << "#ifdef LLVM_GET_SVE_TYPEFLAGS\n";
1625	for (auto &KV : FlagTypes)
1626	OS << "const uint64_t " << KV.getKey() << " = " << KV.getValue() << ";\n";
1627	OS << "#endif\n\n";
1628
1629	OS << "#ifdef LLVM_GET_SVE_ELTTYPES\n";
1630	for (auto &KV : EltTypes)
1631	OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1632	OS << "#endif\n\n";
1633
1634	OS << "#ifdef LLVM_GET_SVE_MEMELTTYPES\n";
1635	for (auto &KV : MemEltTypes)
1636	OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1637	OS << "#endif\n\n";
1638
1639	OS << "#ifdef LLVM_GET_SVE_MERGETYPES\n";
1640	for (auto &KV : MergeTypes)
1641	OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1642	OS << "#endif\n\n";
1643	}
1644
1645	void SVEEmitter::createImmCheckTypes(raw_ostream &OS) {
1646	OS << "#ifdef LLVM_GET_ARM_INTRIN_IMMCHECKTYPES\n";
1647	for (auto &KV : ImmCheckTypes)
1648	OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1649	OS << "#endif\n\n";
1650	}
1651
1652	void SVEEmitter::createSMEHeader(raw_ostream &OS) {
1653	OS << "/*===---- arm_sme.h - ARM SME intrinsics "
1654	"------===\n"
1655	" *\n"
1656	" *\n"
1657	" * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1658	"Exceptions.\n"
1659	" * See https://llvm.org/LICENSE.txt for license information.\n"
1660	" * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1661	" *\n"
1662	" *===-----------------------------------------------------------------"
1663	"------===\n"
1664	" */\n\n";
1665
1666	OS << "#ifndef __ARM_SME_H\n";
1667	OS << "#define __ARM_SME_H\n\n";
1668
1669	OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1670	OS << "#error \"Big endian is currently not supported for arm_sme.h\"\n";
1671	OS << "#endif\n";
1672
1673	OS << "#include <arm_sve.h>\n\n";
1674	OS << "#include <stddef.h>\n\n";
1675
1676	OS << "/* Function attributes */\n";
1677	OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1678	"__nodebug__))\n\n";
1679	OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1680	"__nodebug__, __overloadable__))\n\n";
1681
1682	OS << "#ifdef __cplusplus\n";
1683	OS << "extern \"C\" {\n";
1684	OS << "#endif\n\n";
1685
1686	OS << "void __arm_za_disable(void) __arm_streaming_compatible;\n\n";
1687
1688	OS << "__ai bool __arm_has_sme(void) __arm_streaming_compatible {\n";
1689	OS << " uint64_t x0, x1;\n";
1690	OS << " __builtin_arm_get_sme_state(&x0, &x1);\n";
1691	OS << " return x0 & (1ULL << 63);\n";
1692	OS << "}\n\n";
1693
1694	OS << "void *__arm_sc_memcpy(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";
1695	OS << "void *__arm_sc_memmove(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";
1696	OS << "void *__arm_sc_memset(void *s, int c, size_t n) __arm_streaming_compatible;\n";
1697	OS << "void *__arm_sc_memchr(void *s, int c, size_t n) __arm_streaming_compatible;\n\n";
1698
1699	OS << "__ai __attribute__((target(\"sme\"))) void svundef_za(void) "
1700	"__arm_streaming_compatible __arm_out(\"za\") "
1701	"{ }\n\n";
1702
1703	createCoreHeaderIntrinsics(OS, Emitter&: *this, Kind: ACLEKind::SME);
1704
1705	OS << "#ifdef __cplusplus\n";
1706	OS << "} // extern \"C\"\n";
1707	OS << "#endif\n\n";
1708	OS << "#undef __ai\n\n";
1709	OS << "#endif /* __ARM_SME_H */\n";
1710	}
1711
1712	void SVEEmitter::createSMEBuiltins(raw_ostream &OS) {
1713	std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1714	SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1715	for (auto *R : RV) {
1716	createIntrinsic(R, Out&: Defs);
1717	}
1718
1719	// The mappings must be sorted based on BuiltinID.
1720	sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1721	const std::unique_ptr<Intrinsic> &B) {
1722	return A->getMangledName() < B->getMangledName();
1723	});
1724
1725	llvm::StringToOffsetTable Table;
1726	Table.GetOrAddStringOffset(Str: "");
1727	Table.GetOrAddStringOffset(Str: "n");
1728
1729	for (const auto &Def : Defs)
1730	if (Def->getClassKind() != ClassG) {
1731	Table.GetOrAddStringOffset(Str: Def->getMangledName());
1732	Table.GetOrAddStringOffset(Str: Def->getBuiltinTypeStr());
1733	Table.GetOrAddStringOffset(Str: Def->getGuard());
1734	}
1735
1736	OS << "#ifdef GET_SME_BUILTIN_ENUMERATORS\n";
1737	for (const auto &Def : Defs)
1738	if (Def->getClassKind() != ClassG)
1739	OS << " BI__builtin_sme_" << Def->getMangledName() << ",\n";
1740	OS << "#endif // GET_SME_BUILTIN_ENUMERATORS\n\n";
1741
1742	OS << "#ifdef GET_SME_BUILTIN_STR_TABLE\n";
1743	Table.EmitStringTableDef(OS, Name: "BuiltinStrings");
1744	OS << "#endif // GET_SME_BUILTIN_STR_TABLE\n\n";
1745
1746	OS << "#ifdef GET_SME_BUILTIN_INFOS\n";
1747	for (const auto &Def : Defs) {
1748	// Only create BUILTINs for non-overloaded intrinsics, as overloaded
1749	// declarations only live in the header file.
1750	if (Def->getClassKind() != ClassG) {
1751	OS << " Builtin::Info{Builtin::Info::StrOffsets{"
1752	<< Table.GetStringOffset(Str: Def->getMangledName()) << " /* "
1753	<< Def->getMangledName() << " */, ";
1754	OS << Table.GetStringOffset(Str: Def->getBuiltinTypeStr()) << " /* "
1755	<< Def->getBuiltinTypeStr() << " */, ";
1756	OS << Table.GetStringOffset(Str: "n") << " /* n */, ";
1757	OS << Table.GetStringOffset(Str: Def->getGuard()) << " /* " << Def->getGuard()
1758	<< " */}, ";
1759	OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
1760	}
1761	}
1762	OS << "#endif // GET_SME_BUILTIN_INFOS\n\n";
1763	}
1764
1765	void SVEEmitter::createSMECodeGenMap(raw_ostream &OS) {
1766	std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1767	SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1768	for (auto *R : RV) {
1769	createIntrinsic(R, Out&: Defs);
1770	}
1771
1772	// The mappings must be sorted based on BuiltinID.
1773	sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1774	const std::unique_ptr<Intrinsic> &B) {
1775	return A->getMangledName() < B->getMangledName();
1776	});
1777
1778	OS << "#ifdef GET_SME_LLVM_INTRINSIC_MAP\n";
1779	for (auto &Def : Defs) {
1780	// Builtins only exist for non-overloaded intrinsics, overloaded
1781	// declarations only live in the header file.
1782	if (Def->getClassKind() == ClassG)
1783	continue;
1784
1785	uint64_t Flags = Def->getFlags();
1786	auto FlagString = std::to_string(val: Flags);
1787
1788	std::string LLVMName = Def->getLLVMName();
1789	std::string Builtin = Def->getMangledName();
1790	if (!LLVMName.empty())
1791	OS << "SMEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1792	<< "),\n";
1793	else
1794	OS << "SMEMAP2(" << Builtin << ", " << FlagString << "),\n";
1795	}
1796	OS << "#endif\n\n";
1797	}
1798
1799	void SVEEmitter::createSMERangeChecks(raw_ostream &OS) {
1800	std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1801	SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1802	for (auto *R : RV) {
1803	createIntrinsic(R, Out&: Defs);
1804	}
1805
1806	// The mappings must be sorted based on BuiltinID.
1807	sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1808	const std::unique_ptr<Intrinsic> &B) {
1809	return A->getMangledName() < B->getMangledName();
1810	});
1811
1812	OS << "#ifdef GET_SME_IMMEDIATE_CHECK\n";
1813
1814	// Ensure these are only emitted once.
1815	std::set<std::string> Emitted;
1816
1817	for (auto &Def : Defs) {
1818	if (Emitted.find(x: Def->getMangledName()) != Emitted.end() ||
1819	Def->getImmChecks().empty())
1820	continue;
1821
1822	OS << "case SME::BI__builtin_sme_" << Def->getMangledName() << ":\n";
1823	for (auto &Check : Def->getImmChecks())
1824	OS << "ImmChecks.push_back(std::make_tuple(" << Check.getImmArgIdx()
1825	<< ", " << Check.getKind() << ", " << Check.getElementSizeInBits()
1826	<< "));\n";
1827	OS << " break;\n";
1828
1829	Emitted.insert(x: Def->getMangledName());
1830	}
1831
1832	OS << "#endif\n\n";
1833	}
1834
1835	void SVEEmitter::createBuiltinZAState(raw_ostream &OS) {
1836	std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1837	SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1838	for (auto *R : RV)
1839	createIntrinsic(R, Out&: Defs);
1840
1841	std::map<std::string, std::set<std::string>> IntrinsicsPerState;
1842	for (auto &Def : Defs) {
1843	std::string Key;
1844	auto AddToKey = [&Key](const std::string &S) -> void {
1845	Key = Key.empty() ? S : (Key + " | " + S);
1846	};
1847
1848	if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInZA")))
1849	AddToKey("ArmInZA");
1850	else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsOutZA")))
1851	AddToKey("ArmOutZA");
1852	else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInOutZA")))
1853	AddToKey("ArmInOutZA");
1854
1855	if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInZT0")))
1856	AddToKey("ArmInZT0");
1857	else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsOutZT0")))
1858	AddToKey("ArmOutZT0");
1859	else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInOutZT0")))
1860	AddToKey("ArmInOutZT0");
1861
1862	if (!Key.empty())
1863	IntrinsicsPerState[Key].insert(x: Def->getMangledName());
1864	}
1865
1866	OS << "#ifdef GET_SME_BUILTIN_GET_STATE\n";
1867	for (auto &KV : IntrinsicsPerState) {
1868	for (StringRef Name : KV.second)
1869	OS << "case SME::BI__builtin_sme_" << Name << ":\n";
1870	OS << " return " << KV.first << ";\n";
1871	}
1872	OS << "#endif\n\n";
1873	}
1874
1875	void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) {
1876	std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1877	SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1878	for (auto *R : RV)
1879	createIntrinsic(R, Out&: Defs);
1880
1881	StringRef ExtensionKind;
1882	switch (Kind) {
1883	case ACLEKind::SME:
1884	ExtensionKind = "SME";
1885	break;
1886	case ACLEKind::SVE:
1887	ExtensionKind = "SVE";
1888	break;
1889	}
1890
1891	OS << "#ifdef GET_" << ExtensionKind << "_STREAMING_ATTRS\n";
1892
1893	StringMap<std::set<std::string>> StreamingMap;
1894
1895	uint64_t IsStreamingFlag = getEnumValueForFlag(C: "IsStreaming");
1896	uint64_t VerifyRuntimeMode = getEnumValueForFlag(C: "VerifyRuntimeMode");
1897	uint64_t IsStreamingCompatibleFlag =
1898	getEnumValueForFlag(C: "IsStreamingCompatible");
1899
1900	for (auto &Def : Defs) {
1901	if (!Def->isFlagSet(Flag: VerifyRuntimeMode) && !Def->getSVEGuard().empty() &&
1902	!Def->getSMEGuard().empty())
1903	report_fatal_error(reason: "Missing VerifyRuntimeMode flag");
1904
1905	if (Def->isFlagSet(Flag: IsStreamingFlag))
1906	StreamingMap["ArmStreaming"].insert(x: Def->getMangledName());
1907	else if (Def->isFlagSet(Flag: VerifyRuntimeMode))
1908	StreamingMap["VerifyRuntimeMode"].insert(x: Def->getMangledName());
1909	else if (Def->isFlagSet(Flag: IsStreamingCompatibleFlag))
1910	StreamingMap["ArmStreamingCompatible"].insert(x: Def->getMangledName());
1911	else
1912	StreamingMap["ArmNonStreaming"].insert(x: Def->getMangledName());
1913	}
1914
1915	for (auto BuiltinType : StreamingMap.keys()) {
1916	for (auto Name : StreamingMap[BuiltinType]) {
1917	OS << "case " << ExtensionKind << "::BI__builtin_"
1918	<< ExtensionKind.lower() << "_";
1919	OS << Name << ":\n";
1920	}
1921	OS << " BuiltinType = " << BuiltinType << ";\n";
1922	OS << " break;\n";
1923	}
1924
1925	OS << "#endif\n\n";
1926	}
1927
1928	namespace clang {
1929	void EmitSveHeader(const RecordKeeper &Records, raw_ostream &OS) {
1930	SVEEmitter(Records).createHeader(OS);
1931	}
1932
1933	void EmitSveBuiltins(const RecordKeeper &Records, raw_ostream &OS) {
1934	SVEEmitter(Records).createBuiltins(OS);
1935	}
1936
1937	void EmitSveBuiltinCG(const RecordKeeper &Records, raw_ostream &OS) {
1938	SVEEmitter(Records).createCodeGenMap(OS);
1939	}
1940
1941	void EmitSveRangeChecks(const RecordKeeper &Records, raw_ostream &OS) {
1942	SVEEmitter(Records).createRangeChecks(OS);
1943	}
1944
1945	void EmitSveTypeFlags(const RecordKeeper &Records, raw_ostream &OS) {
1946	SVEEmitter(Records).createTypeFlags(OS);
1947	}
1948
1949	void EmitImmCheckTypes(const RecordKeeper &Records, raw_ostream &OS) {
1950	SVEEmitter(Records).createImmCheckTypes(OS);
1951	}
1952
1953	void EmitSveStreamingAttrs(const RecordKeeper &Records, raw_ostream &OS) {
1954	SVEEmitter(Records).createStreamingAttrs(OS, Kind: ACLEKind::SVE);
1955	}
1956
1957	void EmitSmeHeader(const RecordKeeper &Records, raw_ostream &OS) {
1958	SVEEmitter(Records).createSMEHeader(OS);
1959	}
1960
1961	void EmitSmeBuiltins(const RecordKeeper &Records, raw_ostream &OS) {
1962	SVEEmitter(Records).createSMEBuiltins(OS);
1963	}
1964
1965	void EmitSmeBuiltinCG(const RecordKeeper &Records, raw_ostream &OS) {
1966	SVEEmitter(Records).createSMECodeGenMap(OS);
1967	}
1968
1969	void EmitSmeRangeChecks(const RecordKeeper &Records, raw_ostream &OS) {
1970	SVEEmitter(Records).createSMERangeChecks(OS);
1971	}
1972
1973	void EmitSmeStreamingAttrs(const RecordKeeper &Records, raw_ostream &OS) {
1974	SVEEmitter(Records).createStreamingAttrs(OS, Kind: ACLEKind::SME);
1975	}
1976
1977	void EmitSmeBuiltinZAState(const RecordKeeper &Records, raw_ostream &OS) {
1978	SVEEmitter(Records).createBuiltinZAState(OS);
1979	}
1980	} // End namespace clang
1981