1	//===- NeonEmitter.cpp - Generate arm_neon.h for use with clang -*- C++ -*-===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This tablegen backend is responsible for emitting arm_neon.h, which includes
10	// a declaration and definition of each function specified by the ARM NEON
11	// compiler interface. See ARM document DUI0348B.
12	//
13	// Each NEON instruction is implemented in terms of 1 or more functions which
14	// are suffixed with the element type of the input vectors. Functions may be
15	// implemented in terms of generic vector operations such as +, *, -, etc. or
16	// by calling a __builtin_-prefixed function which will be handled by clang's
17	// CodeGen library.
18	//
19	// Additional validation code can be generated by this file when runHeader() is
20	// called, rather than the normal run() entry point.
21	//
22	// See also the documentation in include/clang/Basic/arm_neon.td.
23	//
24	//===----------------------------------------------------------------------===//
25
26	#include "TableGenBackends.h"
27	#include "llvm/ADT/ArrayRef.h"
28	#include "llvm/ADT/DenseMap.h"
29	#include "llvm/ADT/STLExtras.h"
30	#include "llvm/ADT/SmallVector.h"
31	#include "llvm/ADT/StringExtras.h"
32	#include "llvm/ADT/StringRef.h"
33	#include "llvm/Support/Casting.h"
34	#include "llvm/Support/ErrorHandling.h"
35	#include "llvm/Support/raw_ostream.h"
36	#include "llvm/TableGen/Error.h"
37	#include "llvm/TableGen/Record.h"
38	#include "llvm/TableGen/SetTheory.h"
39	#include <algorithm>
40	#include <cassert>
41	#include <cctype>
42	#include <cstddef>
43	#include <cstdint>
44	#include <deque>
45	#include <map>
46	#include <optional>
47	#include <set>
48	#include <sstream>
49	#include <string>
50	#include <utility>
51	#include <vector>
52
53	using namespace llvm;
54
55	namespace {
56
57	// While globals are generally bad, this one allows us to perform assertions
58	// liberally and somehow still trace them back to the def they indirectly
59	// came from.
60	static Record *CurrentRecord = nullptr;
61	static void assert_with_loc(bool Assertion, const std::string &Str) {
62	if (!Assertion) {
63	if (CurrentRecord)
64	PrintFatalError(ErrorLoc: CurrentRecord->getLoc(), Msg: Str);
65	else
66	PrintFatalError(Msg: Str);
67	}
68	}
69
70	enum ClassKind {
71	ClassNone,
72	ClassI, // generic integer instruction, e.g., "i8" suffix
73	ClassS, // signed/unsigned/poly, e.g., "s8", "u8" or "p8" suffix
74	ClassW, // width-specific instruction, e.g., "8" suffix
75	ClassB, // bitcast arguments with enum argument to specify type
76	ClassL, // Logical instructions which are op instructions
77	// but we need to not emit any suffix for in our
78	// tests.
79	ClassNoTest // Instructions which we do not test since they are
80	// not TRUE instructions.
81	};
82
83	/// NeonTypeFlags - Flags to identify the types for overloaded Neon
84	/// builtins. These must be kept in sync with the flags in
85	/// include/clang/Basic/TargetBuiltins.h.
86	namespace NeonTypeFlags {
87
88	enum { EltTypeMask = 0xf, UnsignedFlag = 0x10, QuadFlag = 0x20 };
89
90	enum EltType {
91	Int8,
92	Int16,
93	Int32,
94	Int64,
95	Poly8,
96	Poly16,
97	Poly64,
98	Poly128,
99	Float16,
100	Float32,
101	Float64,
102	BFloat16
103	};
104
105	} // end namespace NeonTypeFlags
106
107	class NeonEmitter;
108
109	//===----------------------------------------------------------------------===//
110	// TypeSpec
111	//===----------------------------------------------------------------------===//
112
113	/// A TypeSpec is just a simple wrapper around a string, but gets its own type
114	/// for strong typing purposes.
115	///
116	/// A TypeSpec can be used to create a type.
117	class TypeSpec : public std::string {
118	public:
119	static std::vector<TypeSpec> fromTypeSpecs(StringRef Str) {
120	std::vector<TypeSpec> Ret;
121	TypeSpec Acc;
122	for (char I : Str.str()) {
123	if (islower(I)) {
124	Acc.push_back(c: I);
125	Ret.push_back(x: TypeSpec(Acc));
126	Acc.clear();
127	} else {
128	Acc.push_back(c: I);
129	}
130	}
131	return Ret;
132	}
133	};
134
135	//===----------------------------------------------------------------------===//
136	// Type
137	//===----------------------------------------------------------------------===//
138
139	/// A Type. Not much more to say here.
140	class Type {
141	private:
142	TypeSpec TS;
143
144	enum TypeKind {
145	Void,
146	Float,
147	SInt,
148	UInt,
149	Poly,
150	BFloat16,
151	};
152	TypeKind Kind;
153	bool Immediate, Constant, Pointer;
154	// ScalarForMangling and NoManglingQ are really not suited to live here as
155	// they are not related to the type. But they live in the TypeSpec (not the
156	// prototype), so this is really the only place to store them.
157	bool ScalarForMangling, NoManglingQ;
158	unsigned Bitwidth, ElementBitwidth, NumVectors;
159
160	public:
161	Type()
162	: Kind(Void), Immediate(false), Constant(false),
163	Pointer(false), ScalarForMangling(false), NoManglingQ(false),
164	Bitwidth(0), ElementBitwidth(0), NumVectors(0) {}
165
166	Type(TypeSpec TS, StringRef CharMods)
167	: TS(std::move(TS)), Kind(Void), Immediate(false),
168	Constant(false), Pointer(false), ScalarForMangling(false),
169	NoManglingQ(false), Bitwidth(0), ElementBitwidth(0), NumVectors(0) {
170	applyModifiers(Mods: CharMods);
171	}
172
173	/// Returns a type representing "void".
174	static Type getVoid() { return Type(); }
175
176	bool operator==(const Type &Other) const { return str() == Other.str(); }
177	bool operator!=(const Type &Other) const { return !operator==(Other); }
178
179	//
180	// Query functions
181	//
182	bool isScalarForMangling() const { return ScalarForMangling; }
183	bool noManglingQ() const { return NoManglingQ; }
184
185	bool isPointer() const { return Pointer; }
186	bool isValue() const { return !isVoid() && !isPointer(); }
187	bool isScalar() const { return isValue() && NumVectors == 0; }
188	bool isVector() const { return isValue() && NumVectors > 0; }
189	bool isConstPointer() const { return Constant; }
190	bool isFloating() const { return Kind == Float; }
191	bool isInteger() const { return Kind == SInt || Kind == UInt; }
192	bool isPoly() const { return Kind == Poly; }
193	bool isSigned() const { return Kind == SInt; }
194	bool isImmediate() const { return Immediate; }
195	bool isFloat() const { return isFloating() && ElementBitwidth == 32; }
196	bool isDouble() const { return isFloating() && ElementBitwidth == 64; }
197	bool isHalf() const { return isFloating() && ElementBitwidth == 16; }
198	bool isChar() const { return ElementBitwidth == 8; }
199	bool isShort() const { return isInteger() && ElementBitwidth == 16; }
200	bool isInt() const { return isInteger() && ElementBitwidth == 32; }
201	bool isLong() const { return isInteger() && ElementBitwidth == 64; }
202	bool isVoid() const { return Kind == Void; }
203	bool isBFloat16() const { return Kind == BFloat16; }
204	unsigned getNumElements() const { return Bitwidth / ElementBitwidth; }
205	unsigned getSizeInBits() const { return Bitwidth; }
206	unsigned getElementSizeInBits() const { return ElementBitwidth; }
207	unsigned getNumVectors() const { return NumVectors; }
208
209	//
210	// Mutator functions
211	//
212	void makeUnsigned() {
213	assert(!isVoid() && "not a potentially signed type");
214	Kind = UInt;
215	}
216	void makeSigned() {
217	assert(!isVoid() && "not a potentially signed type");
218	Kind = SInt;
219	}
220
221	void makeInteger(unsigned ElemWidth, bool Sign) {
222	assert(!isVoid() && "converting void to int probably not useful");
223	Kind = Sign ? SInt : UInt;
224	Immediate = false;
225	ElementBitwidth = ElemWidth;
226	}
227
228	void makeImmediate(unsigned ElemWidth) {
229	Kind = SInt;
230	Immediate = true;
231	ElementBitwidth = ElemWidth;
232	}
233
234	void makeScalar() {
235	Bitwidth = ElementBitwidth;
236	NumVectors = 0;
237	}
238
239	void makeOneVector() {
240	assert(isVector());
241	NumVectors = 1;
242	}
243
244	void make32BitElement() {
245	assert_with_loc(Assertion: Bitwidth > 32, Str: "Not enough bits to make it 32!");
246	ElementBitwidth = 32;
247	}
248
249	void doubleLanes() {
250	assert_with_loc(Assertion: Bitwidth != 128, Str: "Can't get bigger than 128!");
251	Bitwidth = 128;
252	}
253
254	void halveLanes() {
255	assert_with_loc(Assertion: Bitwidth != 64, Str: "Can't get smaller than 64!");
256	Bitwidth = 64;
257	}
258
259	/// Return the C string representation of a type, which is the typename
260	/// defined in stdint.h or arm_neon.h.
261	std::string str() const;
262
263	/// Return the string representation of a type, which is an encoded
264	/// string for passing to the BUILTIN() macro in Builtins.def.
265	std::string builtin_str() const;
266
267	/// Return the value in NeonTypeFlags for this type.
268	unsigned getNeonEnum() const;
269
270	/// Parse a type from a stdint.h or arm_neon.h typedef name,
271	/// for example uint32x2_t or int64_t.
272	static Type fromTypedefName(StringRef Name);
273
274	private:
275	/// Creates the type based on the typespec string in TS.
276	/// Sets "Quad" to true if the "Q" or "H" modifiers were
277	/// seen. This is needed by applyModifier as some modifiers
278	/// only take effect if the type size was changed by "Q" or "H".
279	void applyTypespec(bool &Quad);
280	/// Applies prototype modifiers to the type.
281	void applyModifiers(StringRef Mods);
282	};
283
284	//===----------------------------------------------------------------------===//
285	// Variable
286	//===----------------------------------------------------------------------===//
287
288	/// A variable is a simple class that just has a type and a name.
289	class Variable {
290	Type T;
291	std::string N;
292
293	public:
294	Variable() : T(Type::getVoid()) {}
295	Variable(Type T, std::string N) : T(std::move(T)), N(std::move(N)) {}
296
297	Type getType() const { return T; }
298	std::string getName() const { return "__" + N; }
299	};
300
301	//===----------------------------------------------------------------------===//
302	// Intrinsic
303	//===----------------------------------------------------------------------===//
304
305	/// The main grunt class. This represents an instantiation of an intrinsic with
306	/// a particular typespec and prototype.
307	class Intrinsic {
308	/// The Record this intrinsic was created from.
309	Record *R;
310	/// The unmangled name.
311	std::string Name;
312	/// The input and output typespecs. InTS == OutTS except when
313	/// CartesianProductWith is non-empty - this is the case for vreinterpret.
314	TypeSpec OutTS, InTS;
315	/// The base class kind. Most intrinsics use ClassS, which has full type
316	/// info for integers (s32/u32). Some use ClassI, which doesn't care about
317	/// signedness (i32), while some (ClassB) have no type at all, only a width
318	/// (32).
319	ClassKind CK;
320	/// The list of DAGs for the body. May be empty, in which case we should
321	/// emit a builtin call.
322	ListInit *Body;
323	/// The architectural ifdef guard.
324	std::string ArchGuard;
325	/// The architectural target() guard.
326	std::string TargetGuard;
327	/// Set if the Unavailable bit is 1. This means we don't generate a body,
328	/// just an "unavailable" attribute on a declaration.
329	bool IsUnavailable;
330	/// Is this intrinsic safe for big-endian? or does it need its arguments
331	/// reversing?
332	bool BigEndianSafe;
333
334	/// The types of return value [0] and parameters [1..].
335	std::vector<Type> Types;
336	/// The index of the key type passed to CGBuiltin.cpp for polymorphic calls.
337	int PolymorphicKeyType;
338	/// The local variables defined.
339	std::map<std::string, Variable> Variables;
340	/// NeededEarly - set if any other intrinsic depends on this intrinsic.
341	bool NeededEarly;
342	/// UseMacro - set if we should implement using a macro or unset for a
343	/// function.
344	bool UseMacro;
345	/// The set of intrinsics that this intrinsic uses/requires.
346	std::set<Intrinsic *> Dependencies;
347	/// The "base type", which is Type('d', OutTS). InBaseType is only
348	/// different if CartesianProductWith is non-empty (for vreinterpret).
349	Type BaseType, InBaseType;
350	/// The return variable.
351	Variable RetVar;
352	/// A postfix to apply to every variable. Defaults to "".
353	std::string VariablePostfix;
354
355	NeonEmitter &Emitter;
356	std::stringstream OS;
357
358	bool isBigEndianSafe() const {
359	if (BigEndianSafe)
360	return true;
361
362	for (const auto &T : Types){
363	if (T.isVector() && T.getNumElements() > 1)
364	return false;
365	}
366	return true;
367	}
368
369	public:
370	Intrinsic(Record *R, StringRef Name, StringRef Proto, TypeSpec OutTS,
371	TypeSpec InTS, ClassKind CK, ListInit *Body, NeonEmitter &Emitter,
372	StringRef ArchGuard, StringRef TargetGuard, bool IsUnavailable, bool BigEndianSafe)
373	: R(R), Name(Name.str()), OutTS(OutTS), InTS(InTS), CK(CK), Body(Body),
374	ArchGuard(ArchGuard.str()), TargetGuard(TargetGuard.str()), IsUnavailable(IsUnavailable),
375	BigEndianSafe(BigEndianSafe), PolymorphicKeyType(0), NeededEarly(false),
376	UseMacro(false), BaseType(OutTS, "."), InBaseType(InTS, "."),
377	Emitter(Emitter) {
378	// Modify the TypeSpec per-argument to get a concrete Type, and create
379	// known variables for each.
380	// Types[0] is the return value.
381	unsigned Pos = 0;
382	Types.emplace_back(args&: OutTS, args: getNextModifiers(Proto, Pos));
383	StringRef Mods = getNextModifiers(Proto, Pos);
384	while (!Mods.empty()) {
385	Types.emplace_back(args&: InTS, args&: Mods);
386	if (Mods.contains(C: '!'))
387	PolymorphicKeyType = Types.size() - 1;
388
389	Mods = getNextModifiers(Proto, Pos);
390	}
391
392	for (const auto &Type : Types) {
393	// If this builtin takes an immediate argument, we need to #define it rather
394	// than use a standard declaration, so that SemaChecking can range check
395	// the immediate passed by the user.
396
397	// Pointer arguments need to use macros to avoid hiding aligned attributes
398	// from the pointer type.
399
400	// It is not permitted to pass or return an __fp16 by value, so intrinsics
401	// taking a scalar float16_t must be implemented as macros.
402	if (Type.isImmediate() || Type.isPointer() ||
403	(Type.isScalar() && Type.isHalf()))
404	UseMacro = true;
405	}
406	}
407
408	/// Get the Record that this intrinsic is based off.
409	Record *getRecord() const { return R; }
410	/// Get the set of Intrinsics that this intrinsic calls.
411	/// this is the set of immediate dependencies, NOT the
412	/// transitive closure.
413	const std::set<Intrinsic *> &getDependencies() const { return Dependencies; }
414	/// Get the architectural guard string (#ifdef).
415	std::string getArchGuard() const { return ArchGuard; }
416	std::string getTargetGuard() const { return TargetGuard; }
417	/// Get the non-mangled name.
418	std::string getName() const { return Name; }
419
420	/// Return true if the intrinsic takes an immediate operand.
421	bool hasImmediate() const {
422	return llvm::any_of(Range: Types, P: [](const Type &T) { return T.isImmediate(); });
423	}
424
425	/// Return the parameter index of the immediate operand.
426	unsigned getImmediateIdx() const {
427	for (unsigned Idx = 0; Idx < Types.size(); ++Idx)
428	if (Types[Idx].isImmediate())
429	return Idx - 1;
430	llvm_unreachable("Intrinsic has no immediate");
431	}
432
433
434	unsigned getNumParams() const { return Types.size() - 1; }
435	Type getReturnType() const { return Types[0]; }
436	Type getParamType(unsigned I) const { return Types[I + 1]; }
437	Type getBaseType() const { return BaseType; }
438	Type getPolymorphicKeyType() const { return Types[PolymorphicKeyType]; }
439
440	/// Return true if the prototype has a scalar argument.
441	bool protoHasScalar() const;
442
443	/// Return the index that parameter PIndex will sit at
444	/// in a generated function call. This is often just PIndex,
445	/// but may not be as things such as multiple-vector operands
446	/// and sret parameters need to be taken into account.
447	unsigned getGeneratedParamIdx(unsigned PIndex) {
448	unsigned Idx = 0;
449	if (getReturnType().getNumVectors() > 1)
450	// Multiple vectors are passed as sret.
451	++Idx;
452
453	for (unsigned I = 0; I < PIndex; ++I)
454	Idx += std::max(a: 1U, b: getParamType(I).getNumVectors());
455
456	return Idx;
457	}
458
459	bool hasBody() const { return Body && !Body->getValues().empty(); }
460
461	void setNeededEarly() { NeededEarly = true; }
462
463	bool operator<(const Intrinsic &Other) const {
464	// Sort lexicographically on a three-tuple (ArchGuard, TargetGuard, Name)
465	if (ArchGuard != Other.ArchGuard)
466	return ArchGuard < Other.ArchGuard;
467	if (TargetGuard != Other.TargetGuard)
468	return TargetGuard < Other.TargetGuard;
469	return Name < Other.Name;
470	}
471
472	ClassKind getClassKind(bool UseClassBIfScalar = false) {
473	if (UseClassBIfScalar && !protoHasScalar())
474	return ClassB;
475	return CK;
476	}
477
478	/// Return the name, mangled with type information.
479	/// If ForceClassS is true, use ClassS (u32/s32) instead
480	/// of the intrinsic's own type class.
481	std::string getMangledName(bool ForceClassS = false) const;
482	/// Return the type code for a builtin function call.
483	std::string getInstTypeCode(Type T, ClassKind CK) const;
484	/// Return the type string for a BUILTIN() macro in Builtins.def.
485	std::string getBuiltinTypeStr();
486
487	/// Generate the intrinsic, returning code.
488	std::string generate();
489	/// Perform type checking and populate the dependency graph, but
490	/// don't generate code yet.
491	void indexBody();
492
493	private:
494	StringRef getNextModifiers(StringRef Proto, unsigned &Pos) const;
495
496	std::string mangleName(std::string Name, ClassKind CK) const;
497
498	void initVariables();
499	std::string replaceParamsIn(std::string S);
500
501	void emitBodyAsBuiltinCall();
502
503	void generateImpl(bool ReverseArguments,
504	StringRef NamePrefix, StringRef CallPrefix);
505	void emitReturn();
506	void emitBody(StringRef CallPrefix);
507	void emitShadowedArgs();
508	void emitArgumentReversal();
509	void emitReturnVarDecl();
510	void emitReturnReversal();
511	void emitReverseVariable(Variable &Dest, Variable &Src);
512	void emitNewLine();
513	void emitClosingBrace();
514	void emitOpeningBrace();
515	void emitPrototype(StringRef NamePrefix);
516
517	class DagEmitter {
518	Intrinsic &Intr;
519	StringRef CallPrefix;
520
521	public:
522	DagEmitter(Intrinsic &Intr, StringRef CallPrefix) :
523	Intr(Intr), CallPrefix(CallPrefix) {
524	}
525	std::pair<Type, std::string> emitDagArg(Init *Arg, std::string ArgName);
526	std::pair<Type, std::string> emitDagSaveTemp(DagInit *DI);
527	std::pair<Type, std::string> emitDagSplat(DagInit *DI);
528	std::pair<Type, std::string> emitDagDup(DagInit *DI);
529	std::pair<Type, std::string> emitDagDupTyped(DagInit *DI);
530	std::pair<Type, std::string> emitDagShuffle(DagInit *DI);
531	std::pair<Type, std::string> emitDagCast(DagInit *DI, bool IsBitCast);
532	std::pair<Type, std::string> emitDagCall(DagInit *DI,
533	bool MatchMangledName);
534	std::pair<Type, std::string> emitDagNameReplace(DagInit *DI);
535	std::pair<Type, std::string> emitDagLiteral(DagInit *DI);
536	std::pair<Type, std::string> emitDagOp(DagInit *DI);
537	std::pair<Type, std::string> emitDag(DagInit *DI);
538	};
539	};
540
541	//===----------------------------------------------------------------------===//
542	// NeonEmitter
543	//===----------------------------------------------------------------------===//
544
545	class NeonEmitter {
546	RecordKeeper &Records;
547	DenseMap<Record *, ClassKind> ClassMap;
548	std::map<std::string, std::deque<Intrinsic>> IntrinsicMap;
549	unsigned UniqueNumber;
550
551	void createIntrinsic(Record *R, SmallVectorImpl<Intrinsic *> &Out);
552	void genBuiltinsDef(raw_ostream &OS, SmallVectorImpl<Intrinsic *> &Defs);
553	void genStreamingSVECompatibleList(raw_ostream &OS,
554	SmallVectorImpl<Intrinsic *> &Defs);
555	void genOverloadTypeCheckCode(raw_ostream &OS,
556	SmallVectorImpl<Intrinsic *> &Defs);
557	void genIntrinsicRangeCheckCode(raw_ostream &OS,
558	SmallVectorImpl<Intrinsic *> &Defs);
559
560	public:
561	/// Called by Intrinsic - this attempts to get an intrinsic that takes
562	/// the given types as arguments.
563	Intrinsic &getIntrinsic(StringRef Name, ArrayRef<Type> Types,
564	std::optional<std::string> MangledName);
565
566	/// Called by Intrinsic - returns a globally-unique number.
567	unsigned getUniqueNumber() { return UniqueNumber++; }
568
569	NeonEmitter(RecordKeeper &R) : Records(R), UniqueNumber(0) {
570	Record *SI = R.getClass(Name: "SInst");
571	Record *II = R.getClass(Name: "IInst");
572	Record *WI = R.getClass(Name: "WInst");
573	Record *SOpI = R.getClass(Name: "SOpInst");
574	Record *IOpI = R.getClass(Name: "IOpInst");
575	Record *WOpI = R.getClass(Name: "WOpInst");
576	Record *LOpI = R.getClass(Name: "LOpInst");
577	Record *NoTestOpI = R.getClass(Name: "NoTestOpInst");
578
579	ClassMap[SI] = ClassS;
580	ClassMap[II] = ClassI;
581	ClassMap[WI] = ClassW;
582	ClassMap[SOpI] = ClassS;
583	ClassMap[IOpI] = ClassI;
584	ClassMap[WOpI] = ClassW;
585	ClassMap[LOpI] = ClassL;
586	ClassMap[NoTestOpI] = ClassNoTest;
587	}
588
589	// Emit arm_neon.h.inc
590	void run(raw_ostream &o);
591
592	// Emit arm_fp16.h.inc
593	void runFP16(raw_ostream &o);
594
595	// Emit arm_bf16.h.inc
596	void runBF16(raw_ostream &o);
597
598	void runVectorTypes(raw_ostream &o);
599
600	// Emit all the __builtin prototypes used in arm_neon.h, arm_fp16.h and
601	// arm_bf16.h
602	void runHeader(raw_ostream &o);
603	};
604
605	} // end anonymous namespace
606
607	//===----------------------------------------------------------------------===//
608	// Type implementation
609	//===----------------------------------------------------------------------===//
610
611	std::string Type::str() const {
612	if (isVoid())
613	return "void";
614	std::string S;
615
616	if (isInteger() && !isSigned())
617	S += "u";
618
619	if (isPoly())
620	S += "poly";
621	else if (isFloating())
622	S += "float";
623	else if (isBFloat16())
624	S += "bfloat";
625	else
626	S += "int";
627
628	S += utostr(X: ElementBitwidth);
629	if (isVector())
630	S += "x" + utostr(X: getNumElements());
631	if (NumVectors > 1)
632	S += "x" + utostr(X: NumVectors);
633	S += "_t";
634
635	if (Constant)
636	S += " const";
637	if (Pointer)
638	S += " *";
639
640	return S;
641	}
642
643	std::string Type::builtin_str() const {
644	std::string S;
645	if (isVoid())
646	return "v";
647
648	if (isPointer()) {
649	// All pointers are void pointers.
650	S = "v";
651	if (isConstPointer())
652	S += "C";
653	S += "*";
654	return S;
655	} else if (isInteger())
656	switch (ElementBitwidth) {
657	case 8: S += "c"; break;
658	case 16: S += "s"; break;
659	case 32: S += "i"; break;
660	case 64: S += "Wi"; break;
661	case 128: S += "LLLi"; break;
662	default: llvm_unreachable("Unhandled case!");
663	}
664	else if (isBFloat16()) {
665	assert(ElementBitwidth == 16 && "BFloat16 can only be 16 bits");
666	S += "y";
667	} else
668	switch (ElementBitwidth) {
669	case 16: S += "h"; break;
670	case 32: S += "f"; break;
671	case 64: S += "d"; break;
672	default: llvm_unreachable("Unhandled case!");
673	}
674
675	// FIXME: NECESSARY???????????????????????????????????????????????????????????????????????
676	if (isChar() && !isPointer() && isSigned())
677	// Make chars explicitly signed.
678	S = "S" + S;
679	else if (isInteger() && !isSigned())
680	S = "U" + S;
681
682	// Constant indices are "int", but have the "constant expression" modifier.
683	if (isImmediate()) {
684	assert(isInteger() && isSigned());
685	S = "I" + S;
686	}
687
688	if (isScalar())
689	return S;
690
691	std::string Ret;
692	for (unsigned I = 0; I < NumVectors; ++I)
693	Ret += "V" + utostr(X: getNumElements()) + S;
694
695	return Ret;
696	}
697
698	unsigned Type::getNeonEnum() const {
699	unsigned Addend;
700	switch (ElementBitwidth) {
701	case 8: Addend = 0; break;
702	case 16: Addend = 1; break;
703	case 32: Addend = 2; break;
704	case 64: Addend = 3; break;
705	case 128: Addend = 4; break;
706	default: llvm_unreachable("Unhandled element bitwidth!");
707	}
708
709	unsigned Base = (unsigned)NeonTypeFlags::Int8 + Addend;
710	if (isPoly()) {
711	// Adjustment needed because Poly32 doesn't exist.
712	if (Addend >= 2)
713	--Addend;
714	Base = (unsigned)NeonTypeFlags::Poly8 + Addend;
715	}
716	if (isFloating()) {
717	assert(Addend != 0 && "Float8 doesn't exist!");
718	Base = (unsigned)NeonTypeFlags::Float16 + (Addend - 1);
719	}
720
721	if (isBFloat16()) {
722	assert(Addend == 1 && "BFloat16 is only 16 bit");
723	Base = (unsigned)NeonTypeFlags::BFloat16;
724	}
725
726	if (Bitwidth == 128)
727	Base |= (unsigned)NeonTypeFlags::QuadFlag;
728	if (isInteger() && !isSigned())
729	Base |= (unsigned)NeonTypeFlags::UnsignedFlag;
730
731	return Base;
732	}
733
734	Type Type::fromTypedefName(StringRef Name) {
735	Type T;
736	T.Kind = SInt;
737
738	if (Name.consume_front(Prefix: "u"))
739	T.Kind = UInt;
740
741	if (Name.consume_front(Prefix: "float")) {
742	T.Kind = Float;
743	} else if (Name.consume_front(Prefix: "poly")) {
744	T.Kind = Poly;
745	} else if (Name.consume_front(Prefix: "bfloat")) {
746	T.Kind = BFloat16;
747	} else {
748	assert(Name.starts_with("int"));
749	Name = Name.drop_front(N: 3);
750	}
751
752	unsigned I = 0;
753	for (I = 0; I < Name.size(); ++I) {
754	if (!isdigit(Name[I]))
755	break;
756	}
757	Name.substr(Start: 0, N: I).getAsInteger(Radix: 10, Result&: T.ElementBitwidth);
758	Name = Name.drop_front(N: I);
759
760	T.Bitwidth = T.ElementBitwidth;
761	T.NumVectors = 1;
762
763	if (Name.consume_front(Prefix: "x")) {
764	unsigned I = 0;
765	for (I = 0; I < Name.size(); ++I) {
766	if (!isdigit(Name[I]))
767	break;
768	}
769	unsigned NumLanes;
770	Name.substr(Start: 0, N: I).getAsInteger(Radix: 10, Result&: NumLanes);
771	Name = Name.drop_front(N: I);
772	T.Bitwidth = T.ElementBitwidth * NumLanes;
773	} else {
774	// Was scalar.
775	T.NumVectors = 0;
776	}
777	if (Name.consume_front(Prefix: "x")) {
778	unsigned I = 0;
779	for (I = 0; I < Name.size(); ++I) {
780	if (!isdigit(Name[I]))
781	break;
782	}
783	Name.substr(Start: 0, N: I).getAsInteger(Radix: 10, Result&: T.NumVectors);
784	Name = Name.drop_front(N: I);
785	}
786
787	assert(Name.starts_with("_t") && "Malformed typedef!");
788	return T;
789	}
790
791	void Type::applyTypespec(bool &Quad) {
792	std::string S = TS;
793	ScalarForMangling = false;
794	Kind = SInt;
795	ElementBitwidth = ~0U;
796	NumVectors = 1;
797
798	for (char I : S) {
799	switch (I) {
800	case 'S':
801	ScalarForMangling = true;
802	break;
803	case 'H':
804	NoManglingQ = true;
805	Quad = true;
806	break;
807	case 'Q':
808	Quad = true;
809	break;
810	case 'P':
811	Kind = Poly;
812	break;
813	case 'U':
814	Kind = UInt;
815	break;
816	case 'c':
817	ElementBitwidth = 8;
818	break;
819	case 'h':
820	Kind = Float;
821	[[fallthrough]];
822	case 's':
823	ElementBitwidth = 16;
824	break;
825	case 'f':
826	Kind = Float;
827	[[fallthrough]];
828	case 'i':
829	ElementBitwidth = 32;
830	break;
831	case 'd':
832	Kind = Float;
833	[[fallthrough]];
834	case 'l':
835	ElementBitwidth = 64;
836	break;
837	case 'k':
838	ElementBitwidth = 128;
839	// Poly doesn't have a 128x1 type.
840	if (isPoly())
841	NumVectors = 0;
842	break;
843	case 'b':
844	Kind = BFloat16;
845	ElementBitwidth = 16;
846	break;
847	default:
848	llvm_unreachable("Unhandled type code!");
849	}
850	}
851	assert(ElementBitwidth != ~0U && "Bad element bitwidth!");
852
853	Bitwidth = Quad ? 128 : 64;
854	}
855
856	void Type::applyModifiers(StringRef Mods) {
857	bool AppliedQuad = false;
858	applyTypespec(Quad&: AppliedQuad);
859
860	for (char Mod : Mods) {
861	switch (Mod) {
862	case '.':
863	break;
864	case 'v':
865	Kind = Void;
866	break;
867	case 'S':
868	Kind = SInt;
869	break;
870	case 'U':
871	Kind = UInt;
872	break;
873	case 'B':
874	Kind = BFloat16;
875	ElementBitwidth = 16;
876	break;
877	case 'F':
878	Kind = Float;
879	break;
880	case 'P':
881	Kind = Poly;
882	break;
883	case '>':
884	assert(ElementBitwidth < 128);
885	ElementBitwidth *= 2;
886	break;
887	case '<':
888	assert(ElementBitwidth > 8);
889	ElementBitwidth /= 2;
890	break;
891	case '1':
892	NumVectors = 0;
893	break;
894	case '2':
895	NumVectors = 2;
896	break;
897	case '3':
898	NumVectors = 3;
899	break;
900	case '4':
901	NumVectors = 4;
902	break;
903	case '*':
904	Pointer = true;
905	break;
906	case 'c':
907	Constant = true;
908	break;
909	case 'Q':
910	Bitwidth = 128;
911	break;
912	case 'q':
913	Bitwidth = 64;
914	break;
915	case 'I':
916	Kind = SInt;
917	ElementBitwidth = Bitwidth = 32;
918	NumVectors = 0;
919	Immediate = true;
920	break;
921	case 'p':
922	if (isPoly())
923	Kind = UInt;
924	break;
925	case '!':
926	// Key type, handled elsewhere.
927	break;
928	default:
929	llvm_unreachable("Unhandled character!");
930	}
931	}
932	}
933
934	//===----------------------------------------------------------------------===//
935	// Intrinsic implementation
936	//===----------------------------------------------------------------------===//
937
938	StringRef Intrinsic::getNextModifiers(StringRef Proto, unsigned &Pos) const {
939	if (Proto.size() == Pos)
940	return StringRef();
941	else if (Proto[Pos] != '(')
942	return Proto.substr(Start: Pos++, N: 1);
943
944	size_t Start = Pos + 1;
945	size_t End = Proto.find(C: ')', From: Start);
946	assert_with_loc(Assertion: End != StringRef::npos, Str: "unmatched modifier group paren");
947	Pos = End + 1;
948	return Proto.slice(Start, End);
949	}
950
951	std::string Intrinsic::getInstTypeCode(Type T, ClassKind CK) const {
952	char typeCode = '\0';
953	bool printNumber = true;
954
955	if (CK == ClassB && TargetGuard == "")
956	return "";
957
958	if (T.isBFloat16())
959	return "bf16";
960
961	if (T.isPoly())
962	typeCode = 'p';
963	else if (T.isInteger())
964	typeCode = T.isSigned() ? 's' : 'u';
965	else
966	typeCode = 'f';
967
968	if (CK == ClassI) {
969	switch (typeCode) {
970	default:
971	break;
972	case 's':
973	case 'u':
974	case 'p':
975	typeCode = 'i';
976	break;
977	}
978	}
979	if (CK == ClassB && TargetGuard == "") {
980	typeCode = '\0';
981	}
982
983	std::string S;
984	if (typeCode != '\0')
985	S.push_back(c: typeCode);
986	if (printNumber)
987	S += utostr(X: T.getElementSizeInBits());
988
989	return S;
990	}
991
992	std::string Intrinsic::getBuiltinTypeStr() {
993	ClassKind LocalCK = getClassKind(UseClassBIfScalar: true);
994	std::string S;
995
996	Type RetT = getReturnType();
997	if ((LocalCK == ClassI || LocalCK == ClassW) && RetT.isScalar() &&
998	!RetT.isFloating() && !RetT.isBFloat16())
999	RetT.makeInteger(ElemWidth: RetT.getElementSizeInBits(), Sign: false);
1000
1001	// Since the return value must be one type, return a vector type of the
1002	// appropriate width which we will bitcast. An exception is made for
1003	// returning structs of 2, 3, or 4 vectors which are returned in a sret-like
1004	// fashion, storing them to a pointer arg.
1005	if (RetT.getNumVectors() > 1) {
1006	S += "vv*"; // void result with void* first argument
1007	} else {
1008	if (RetT.isPoly())
1009	RetT.makeInteger(ElemWidth: RetT.getElementSizeInBits(), Sign: false);
1010	if (!RetT.isScalar() && RetT.isInteger() && !RetT.isSigned())
1011	RetT.makeSigned();
1012
1013	if (LocalCK == ClassB && RetT.isValue() && !RetT.isScalar())
1014	// Cast to vector of 8-bit elements.
1015	RetT.makeInteger(ElemWidth: 8, Sign: true);
1016
1017	S += RetT.builtin_str();
1018	}
1019
1020	for (unsigned I = 0; I < getNumParams(); ++I) {
1021	Type T = getParamType(I);
1022	if (T.isPoly())
1023	T.makeInteger(ElemWidth: T.getElementSizeInBits(), Sign: false);
1024
1025	if (LocalCK == ClassB && !T.isScalar())
1026	T.makeInteger(ElemWidth: 8, Sign: true);
1027	// Halves always get converted to 8-bit elements.
1028	if (T.isHalf() && T.isVector() && !T.isScalarForMangling())
1029	T.makeInteger(ElemWidth: 8, Sign: true);
1030
1031	if (LocalCK == ClassI && T.isInteger())
1032	T.makeSigned();
1033
1034	if (hasImmediate() && getImmediateIdx() == I)
1035	T.makeImmediate(ElemWidth: 32);
1036
1037	S += T.builtin_str();
1038	}
1039
1040	// Extra constant integer to hold type class enum for this function, e.g. s8
1041	if (LocalCK == ClassB)
1042	S += "i";
1043
1044	return S;
1045	}
1046
1047	std::string Intrinsic::getMangledName(bool ForceClassS) const {
1048	// Check if the prototype has a scalar operand with the type of the vector
1049	// elements. If not, bitcasting the args will take care of arg checking.
1050	// The actual signedness etc. will be taken care of with special enums.
1051	ClassKind LocalCK = CK;
1052	if (!protoHasScalar())
1053	LocalCK = ClassB;
1054
1055	return mangleName(Name, CK: ForceClassS ? ClassS : LocalCK);
1056	}
1057
1058	std::string Intrinsic::mangleName(std::string Name, ClassKind LocalCK) const {
1059	std::string typeCode = getInstTypeCode(T: BaseType, CK: LocalCK);
1060	std::string S = Name;
1061
1062	if (Name == "vcvt_f16_f32" || Name == "vcvt_f32_f16" ||
1063	Name == "vcvt_f32_f64" || Name == "vcvt_f64_f32" ||
1064	Name == "vcvt_f32_bf16")
1065	return Name;
1066
1067	if (!typeCode.empty()) {
1068	// If the name ends with _xN (N = 2,3,4), insert the typeCode before _xN.
1069	if (Name.size() >= 3 && isdigit(Name.back()) &&
1070	Name[Name.length() - 2] == 'x' && Name[Name.length() - 3] == '_')
1071	S.insert(pos1: S.length() - 3, str: "_" + typeCode);
1072	else
1073	S += "_" + typeCode;
1074	}
1075
1076	if (BaseType != InBaseType) {
1077	// A reinterpret - out the input base type at the end.
1078	S += "_" + getInstTypeCode(T: InBaseType, CK: LocalCK);
1079	}
1080
1081	if (LocalCK == ClassB && TargetGuard == "")
1082	S += "_v";
1083
1084	// Insert a 'q' before the first '_' character so that it ends up before
1085	// _lane or _n on vector-scalar operations.
1086	if (BaseType.getSizeInBits() == 128 && !BaseType.noManglingQ()) {
1087	size_t Pos = S.find(c: '_');
1088	S.insert(pos: Pos, s: "q");
1089	}
1090
1091	char Suffix = '\0';
1092	if (BaseType.isScalarForMangling()) {
1093	switch (BaseType.getElementSizeInBits()) {
1094	case 8: Suffix = 'b'; break;
1095	case 16: Suffix = 'h'; break;
1096	case 32: Suffix = 's'; break;
1097	case 64: Suffix = 'd'; break;
1098	default: llvm_unreachable("Bad suffix!");
1099	}
1100	}
1101	if (Suffix != '\0') {
1102	size_t Pos = S.find(c: '_');
1103	S.insert(pos: Pos, s: &Suffix, n: 1);
1104	}
1105
1106	return S;
1107	}
1108
1109	std::string Intrinsic::replaceParamsIn(std::string S) {
1110	while (S.find(c: '$') != std::string::npos) {
1111	size_t Pos = S.find(c: '$');
1112	size_t End = Pos + 1;
1113	while (isalpha(S[End]))
1114	++End;
1115
1116	std::string VarName = S.substr(pos: Pos + 1, n: End - Pos - 1);
1117	assert_with_loc(Assertion: Variables.find(x: VarName) != Variables.end(),
1118	Str: "Variable not defined!");
1119	S.replace(pos: Pos, n: End - Pos, str: Variables.find(x: VarName)->second.getName());
1120	}
1121
1122	return S;
1123	}
1124
1125	void Intrinsic::initVariables() {
1126	Variables.clear();
1127
1128	// Modify the TypeSpec per-argument to get a concrete Type, and create
1129	// known variables for each.
1130	for (unsigned I = 1; I < Types.size(); ++I) {
1131	char NameC = '0' + (I - 1);
1132	std::string Name = "p";
1133	Name.push_back(c: NameC);
1134
1135	Variables[Name] = Variable(Types[I], Name + VariablePostfix);
1136	}
1137	RetVar = Variable(Types[0], "ret" + VariablePostfix);
1138	}
1139
1140	void Intrinsic::emitPrototype(StringRef NamePrefix) {
1141	if (UseMacro) {
1142	OS << "#define ";
1143	} else {
1144	OS << "__ai ";
1145	if (TargetGuard != "")
1146	OS << "__attribute__((target(\"" << TargetGuard << "\"))) ";
1147	OS << Types[0].str() << " ";
1148	}
1149
1150	OS << NamePrefix.str() << mangleName(Name, LocalCK: ClassS) << "(";
1151
1152	for (unsigned I = 0; I < getNumParams(); ++I) {
1153	if (I != 0)
1154	OS << ", ";
1155
1156	char NameC = '0' + I;
1157	std::string Name = "p";
1158	Name.push_back(c: NameC);
1159	assert(Variables.find(Name) != Variables.end());
1160	Variable &V = Variables[Name];
1161
1162	if (!UseMacro)
1163	OS << V.getType().str() << " ";
1164	OS << V.getName();
1165	}
1166
1167	OS << ")";
1168	}
1169
1170	void Intrinsic::emitOpeningBrace() {
1171	if (UseMacro)
1172	OS << " __extension__ ({";
1173	else
1174	OS << " {";
1175	emitNewLine();
1176	}
1177
1178	void Intrinsic::emitClosingBrace() {
1179	if (UseMacro)
1180	OS << "})";
1181	else
1182	OS << "}";
1183	}
1184
1185	void Intrinsic::emitNewLine() {
1186	if (UseMacro)
1187	OS << " \\\n";
1188	else
1189	OS << "\n";
1190	}
1191
1192	void Intrinsic::emitReverseVariable(Variable &Dest, Variable &Src) {
1193	if (Dest.getType().getNumVectors() > 1) {
1194	emitNewLine();
1195
1196	for (unsigned K = 0; K < Dest.getType().getNumVectors(); ++K) {
1197	OS << " " << Dest.getName() << ".val[" << K << "] = "
1198	<< "__builtin_shufflevector("
1199	<< Src.getName() << ".val[" << K << "], "
1200	<< Src.getName() << ".val[" << K << "]";
1201	for (int J = Dest.getType().getNumElements() - 1; J >= 0; --J)
1202	OS << ", " << J;
1203	OS << ");";
1204	emitNewLine();
1205	}
1206	} else {
1207	OS << " " << Dest.getName()
1208	<< " = __builtin_shufflevector(" << Src.getName() << ", " << Src.getName();
1209	for (int J = Dest.getType().getNumElements() - 1; J >= 0; --J)
1210	OS << ", " << J;
1211	OS << ");";
1212	emitNewLine();
1213	}
1214	}
1215
1216	void Intrinsic::emitArgumentReversal() {
1217	if (isBigEndianSafe())
1218	return;
1219
1220	// Reverse all vector arguments.
1221	for (unsigned I = 0; I < getNumParams(); ++I) {
1222	std::string Name = "p" + utostr(X: I);
1223	std::string NewName = "rev" + utostr(X: I);
1224
1225	Variable &V = Variables[Name];
1226	Variable NewV(V.getType(), NewName + VariablePostfix);
1227
1228	if (!NewV.getType().isVector() || NewV.getType().getNumElements() == 1)
1229	continue;
1230
1231	OS << " " << NewV.getType().str() << " " << NewV.getName() << ";";
1232	emitReverseVariable(Dest&: NewV, Src&: V);
1233	V = NewV;
1234	}
1235	}
1236
1237	void Intrinsic::emitReturnVarDecl() {
1238	assert(RetVar.getType() == Types[0]);
1239	// Create a return variable, if we're not void.
1240	if (!RetVar.getType().isVoid()) {
1241	OS << " " << RetVar.getType().str() << " " << RetVar.getName() << ";";
1242	emitNewLine();
1243	}
1244	}
1245
1246	void Intrinsic::emitReturnReversal() {
1247	if (isBigEndianSafe())
1248	return;
1249	if (!getReturnType().isVector() || getReturnType().isVoid() ||
1250	getReturnType().getNumElements() == 1)
1251	return;
1252	emitReverseVariable(Dest&: RetVar, Src&: RetVar);
1253	}
1254
1255	void Intrinsic::emitShadowedArgs() {
1256	// Macro arguments are not type-checked like inline function arguments,
1257	// so assign them to local temporaries to get the right type checking.
1258	if (!UseMacro)
1259	return;
1260
1261	for (unsigned I = 0; I < getNumParams(); ++I) {
1262	// Do not create a temporary for an immediate argument.
1263	// That would defeat the whole point of using a macro!
1264	if (getParamType(I).isImmediate())
1265	continue;
1266	// Do not create a temporary for pointer arguments. The input
1267	// pointer may have an alignment hint.
1268	if (getParamType(I).isPointer())
1269	continue;
1270
1271	std::string Name = "p" + utostr(X: I);
1272
1273	assert(Variables.find(Name) != Variables.end());
1274	Variable &V = Variables[Name];
1275
1276	std::string NewName = "s" + utostr(X: I);
1277	Variable V2(V.getType(), NewName + VariablePostfix);
1278
1279	OS << " " << V2.getType().str() << " " << V2.getName() << " = "
1280	<< V.getName() << ";";
1281	emitNewLine();
1282
1283	V = V2;
1284	}
1285	}
1286
1287	bool Intrinsic::protoHasScalar() const {
1288	return llvm::any_of(
1289	Range: Types, P: [](const Type &T) { return T.isScalar() && !T.isImmediate(); });
1290	}
1291
1292	void Intrinsic::emitBodyAsBuiltinCall() {
1293	std::string S;
1294
1295	// If this builtin returns a struct 2, 3, or 4 vectors, pass it as an implicit
1296	// sret-like argument.
1297	bool SRet = getReturnType().getNumVectors() >= 2;
1298
1299	StringRef N = Name;
1300	ClassKind LocalCK = CK;
1301	if (!protoHasScalar())
1302	LocalCK = ClassB;
1303
1304	if (!getReturnType().isVoid() && !SRet)
1305	S += "(" + RetVar.getType().str() + ") ";
1306
1307	S += "__builtin_neon_" + mangleName(Name: std::string(N), LocalCK) + "(";
1308
1309	if (SRet)
1310	S += "&" + RetVar.getName() + ", ";
1311
1312	for (unsigned I = 0; I < getNumParams(); ++I) {
1313	Variable &V = Variables["p" + utostr(X: I)];
1314	Type T = V.getType();
1315
1316	// Handle multiple-vector values specially, emitting each subvector as an
1317	// argument to the builtin.
1318	if (T.getNumVectors() > 1) {
1319	// Check if an explicit cast is needed.
1320	std::string Cast;
1321	if (LocalCK == ClassB) {
1322	Type T2 = T;
1323	T2.makeOneVector();
1324	T2.makeInteger(ElemWidth: 8, /*Sign=*/true);
1325	Cast = "(" + T2.str() + ")";
1326	}
1327
1328	for (unsigned J = 0; J < T.getNumVectors(); ++J)
1329	S += Cast + V.getName() + ".val[" + utostr(X: J) + "], ";
1330	continue;
1331	}
1332
1333	std::string Arg = V.getName();
1334	Type CastToType = T;
1335
1336	// Check if an explicit cast is needed.
1337	if (CastToType.isVector() &&
1338	(LocalCK == ClassB || (T.isHalf() && !T.isScalarForMangling()))) {
1339	CastToType.makeInteger(ElemWidth: 8, Sign: true);
1340	Arg = "(" + CastToType.str() + ")" + Arg;
1341	} else if (CastToType.isVector() && LocalCK == ClassI) {
1342	if (CastToType.isInteger())
1343	CastToType.makeSigned();
1344	Arg = "(" + CastToType.str() + ")" + Arg;
1345	}
1346
1347	S += Arg + ", ";
1348	}
1349
1350	// Extra constant integer to hold type class enum for this function, e.g. s8
1351	if (getClassKind(UseClassBIfScalar: true) == ClassB) {
1352	S += utostr(X: getPolymorphicKeyType().getNeonEnum());
1353	} else {
1354	// Remove extraneous ", ".
1355	S.pop_back();
1356	S.pop_back();
1357	}
1358	S += ");";
1359
1360	std::string RetExpr;
1361	if (!SRet && !RetVar.getType().isVoid())
1362	RetExpr = RetVar.getName() + " = ";
1363
1364	OS << " " << RetExpr << S;
1365	emitNewLine();
1366	}
1367
1368	void Intrinsic::emitBody(StringRef CallPrefix) {
1369	std::vector<std::string> Lines;
1370
1371	if (!Body || Body->getValues().empty()) {
1372	// Nothing specific to output - must output a builtin.
1373	emitBodyAsBuiltinCall();
1374	return;
1375	}
1376
1377	// We have a list of "things to output". The last should be returned.
1378	for (auto *I : Body->getValues()) {
1379	if (StringInit *SI = dyn_cast<StringInit>(Val: I)) {
1380	Lines.push_back(x: replaceParamsIn(S: SI->getAsString()));
1381	} else if (DagInit *DI = dyn_cast<DagInit>(Val: I)) {
1382	DagEmitter DE(*this, CallPrefix);
1383	Lines.push_back(x: DE.emitDag(DI).second + ";");
1384	}
1385	}
1386
1387	assert(!Lines.empty() && "Empty def?");
1388	if (!RetVar.getType().isVoid())
1389	Lines.back().insert(pos1: 0, str: RetVar.getName() + " = ");
1390
1391	for (auto &L : Lines) {
1392	OS << " " << L;
1393	emitNewLine();
1394	}
1395	}
1396
1397	void Intrinsic::emitReturn() {
1398	if (RetVar.getType().isVoid())
1399	return;
1400	if (UseMacro)
1401	OS << " " << RetVar.getName() << ";";
1402	else
1403	OS << " return " << RetVar.getName() << ";";
1404	emitNewLine();
1405	}
1406
1407	std::pair<Type, std::string> Intrinsic::DagEmitter::emitDag(DagInit *DI) {
1408	// At this point we should only be seeing a def.
1409	DefInit *DefI = cast<DefInit>(Val: DI->getOperator());
1410	std::string Op = DefI->getAsString();
1411
1412	if (Op == "cast" || Op == "bitcast")
1413	return emitDagCast(DI, IsBitCast: Op == "bitcast");
1414	if (Op == "shuffle")
1415	return emitDagShuffle(DI);
1416	if (Op == "dup")
1417	return emitDagDup(DI);
1418	if (Op == "dup_typed")
1419	return emitDagDupTyped(DI);
1420	if (Op == "splat")
1421	return emitDagSplat(DI);
1422	if (Op == "save_temp")
1423	return emitDagSaveTemp(DI);
1424	if (Op == "op")
1425	return emitDagOp(DI);
1426	if (Op == "call" || Op == "call_mangled")
1427	return emitDagCall(DI, MatchMangledName: Op == "call_mangled");
1428	if (Op == "name_replace")
1429	return emitDagNameReplace(DI);
1430	if (Op == "literal")
1431	return emitDagLiteral(DI);
1432	assert_with_loc(Assertion: false, Str: "Unknown operation!");
1433	return std::make_pair(x: Type::getVoid(), y: "");
1434	}
1435
1436	std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagOp(DagInit *DI) {
1437	std::string Op = cast<StringInit>(Val: DI->getArg(Num: 0))->getAsUnquotedString();
1438	if (DI->getNumArgs() == 2) {
1439	// Unary op.
1440	std::pair<Type, std::string> R =
1441	emitDagArg(Arg: DI->getArg(Num: 1), ArgName: std::string(DI->getArgNameStr(Num: 1)));
1442	return std::make_pair(x&: R.first, y: Op + R.second);
1443	} else {
1444	assert(DI->getNumArgs() == 3 && "Can only handle unary and binary ops!");
1445	std::pair<Type, std::string> R1 =
1446	emitDagArg(Arg: DI->getArg(Num: 1), ArgName: std::string(DI->getArgNameStr(Num: 1)));
1447	std::pair<Type, std::string> R2 =
1448	emitDagArg(Arg: DI->getArg(Num: 2), ArgName: std::string(DI->getArgNameStr(Num: 2)));
1449	assert_with_loc(Assertion: R1.first == R2.first, Str: "Argument type mismatch!");
1450	return std::make_pair(x&: R1.first, y: R1.second + " " + Op + " " + R2.second);
1451	}
1452	}
1453
1454	std::pair<Type, std::string>
1455	Intrinsic::DagEmitter::emitDagCall(DagInit *DI, bool MatchMangledName) {
1456	std::vector<Type> Types;
1457	std::vector<std::string> Values;
1458	for (unsigned I = 0; I < DI->getNumArgs() - 1; ++I) {
1459	std::pair<Type, std::string> R =
1460	emitDagArg(Arg: DI->getArg(Num: I + 1), ArgName: std::string(DI->getArgNameStr(Num: I + 1)));
1461	Types.push_back(x: R.first);
1462	Values.push_back(x: R.second);
1463	}
1464
1465	// Look up the called intrinsic.
1466	std::string N;
1467	if (StringInit *SI = dyn_cast<StringInit>(Val: DI->getArg(Num: 0)))
1468	N = SI->getAsUnquotedString();
1469	else
1470	N = emitDagArg(Arg: DI->getArg(Num: 0), ArgName: "").second;
1471	std::optional<std::string> MangledName;
1472	if (MatchMangledName) {
1473	if (Intr.getRecord()->getValueAsBit(FieldName: "isLaneQ"))
1474	N += "q";
1475	MangledName = Intr.mangleName(Name: N, LocalCK: ClassS);
1476	}
1477	Intrinsic &Callee = Intr.Emitter.getIntrinsic(Name: N, Types, MangledName);
1478
1479	// Make sure the callee is known as an early def.
1480	Callee.setNeededEarly();
1481	Intr.Dependencies.insert(x: &Callee);
1482
1483	// Now create the call itself.
1484	std::string S;
1485	if (!Callee.isBigEndianSafe())
1486	S += CallPrefix.str();
1487	S += Callee.getMangledName(ForceClassS: true) + "(";
1488	for (unsigned I = 0; I < DI->getNumArgs() - 1; ++I) {
1489	if (I != 0)
1490	S += ", ";
1491	S += Values[I];
1492	}
1493	S += ")";
1494
1495	return std::make_pair(x: Callee.getReturnType(), y&: S);
1496	}
1497
1498	std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagCast(DagInit *DI,
1499	bool IsBitCast){
1500	// (cast MOD* VAL) -> cast VAL to type given by MOD.
1501	std::pair<Type, std::string> R =
1502	emitDagArg(Arg: DI->getArg(Num: DI->getNumArgs() - 1),
1503	ArgName: std::string(DI->getArgNameStr(Num: DI->getNumArgs() - 1)));
1504	Type castToType = R.first;
1505	for (unsigned ArgIdx = 0; ArgIdx < DI->getNumArgs() - 1; ++ArgIdx) {
1506
1507	// MOD can take several forms:
1508	// 1. $X - take the type of parameter / variable X.
1509	// 2. The value "R" - take the type of the return type.
1510	// 3. a type string
1511	// 4. The value "U" or "S" to switch the signedness.
1512	// 5. The value "H" or "D" to half or double the bitwidth.
1513	// 6. The value "8" to convert to 8-bit (signed) integer lanes.
1514	if (!DI->getArgNameStr(Num: ArgIdx).empty()) {
1515	assert_with_loc(Assertion: Intr.Variables.find(x: std::string(
1516	DI->getArgNameStr(Num: ArgIdx))) != Intr.Variables.end(),
1517	Str: "Variable not found");
1518	castToType =
1519	Intr.Variables[std::string(DI->getArgNameStr(Num: ArgIdx))].getType();
1520	} else {
1521	StringInit *SI = dyn_cast<StringInit>(Val: DI->getArg(Num: ArgIdx));
1522	assert_with_loc(Assertion: SI, Str: "Expected string type or $Name for cast type");
1523
1524	if (SI->getAsUnquotedString() == "R") {
1525	castToType = Intr.getReturnType();
1526	} else if (SI->getAsUnquotedString() == "U") {
1527	castToType.makeUnsigned();
1528	} else if (SI->getAsUnquotedString() == "S") {
1529	castToType.makeSigned();
1530	} else if (SI->getAsUnquotedString() == "H") {
1531	castToType.halveLanes();
1532	} else if (SI->getAsUnquotedString() == "D") {
1533	castToType.doubleLanes();
1534	} else if (SI->getAsUnquotedString() == "8") {
1535	castToType.makeInteger(ElemWidth: 8, Sign: true);
1536	} else if (SI->getAsUnquotedString() == "32") {
1537	castToType.make32BitElement();
1538	} else {
1539	castToType = Type::fromTypedefName(Name: SI->getAsUnquotedString());
1540	assert_with_loc(Assertion: !castToType.isVoid(), Str: "Unknown typedef");
1541	}
1542	}
1543	}
1544
1545	std::string S;
1546	if (IsBitCast) {
1547	// Emit a reinterpret cast. The second operand must be an lvalue, so create
1548	// a temporary.
1549	std::string N = "reint";
1550	unsigned I = 0;
1551	while (Intr.Variables.find(x: N) != Intr.Variables.end())
1552	N = "reint" + utostr(X: ++I);
1553	Intr.Variables[N] = Variable(R.first, N + Intr.VariablePostfix);
1554
1555	Intr.OS << R.first.str() << " " << Intr.Variables[N].getName() << " = "
1556	<< R.second << ";";
1557	Intr.emitNewLine();
1558
1559	S = "*(" + castToType.str() + " *) &" + Intr.Variables[N].getName() + "";
1560	} else {
1561	// Emit a normal (static) cast.
1562	S = "(" + castToType.str() + ")(" + R.second + ")";
1563	}
1564
1565	return std::make_pair(x&: castToType, y&: S);
1566	}
1567
1568	std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagShuffle(DagInit *DI){
1569	// See the documentation in arm_neon.td for a description of these operators.
1570	class LowHalf : public SetTheory::Operator {
1571	public:
1572	void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts,
1573	ArrayRef<SMLoc> Loc) override {
1574	SetTheory::RecSet Elts2;
1575	ST.evaluate(begin: Expr->arg_begin(), end: Expr->arg_end(), Elts&: Elts2, Loc);
1576	Elts.insert(Start: Elts2.begin(), End: Elts2.begin() + (Elts2.size() / 2));
1577	}
1578	};
1579
1580	class HighHalf : public SetTheory::Operator {
1581	public:
1582	void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts,
1583	ArrayRef<SMLoc> Loc) override {
1584	SetTheory::RecSet Elts2;
1585	ST.evaluate(begin: Expr->arg_begin(), end: Expr->arg_end(), Elts&: Elts2, Loc);
1586	Elts.insert(Start: Elts2.begin() + (Elts2.size() / 2), End: Elts2.end());
1587	}
1588	};
1589
1590	class Rev : public SetTheory::Operator {
1591	unsigned ElementSize;
1592
1593	public:
1594	Rev(unsigned ElementSize) : ElementSize(ElementSize) {}
1595
1596	void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts,
1597	ArrayRef<SMLoc> Loc) override {
1598	SetTheory::RecSet Elts2;
1599	ST.evaluate(begin: Expr->arg_begin() + 1, end: Expr->arg_end(), Elts&: Elts2, Loc);
1600
1601	int64_t VectorSize = cast<IntInit>(Val: Expr->getArg(Num: 0))->getValue();
1602	VectorSize /= ElementSize;
1603
1604	std::vector<Record *> Revved;
1605	for (unsigned VI = 0; VI < Elts2.size(); VI += VectorSize) {
1606	for (int LI = VectorSize - 1; LI >= 0; --LI) {
1607	Revved.push_back(x: Elts2[VI + LI]);
1608	}
1609	}
1610
1611	Elts.insert(Start: Revved.begin(), End: Revved.end());
1612	}
1613	};
1614
1615	class MaskExpander : public SetTheory::Expander {
1616	unsigned N;
1617
1618	public:
1619	MaskExpander(unsigned N) : N(N) {}
1620
1621	void expand(SetTheory &ST, Record *R, SetTheory::RecSet &Elts) override {
1622	unsigned Addend = 0;
1623	if (R->getName() == "mask0")
1624	Addend = 0;
1625	else if (R->getName() == "mask1")
1626	Addend = N;
1627	else
1628	return;
1629	for (unsigned I = 0; I < N; ++I)
1630	Elts.insert(X: R->getRecords().getDef(Name: "sv" + utostr(X: I + Addend)));
1631	}
1632	};
1633
1634	// (shuffle arg1, arg2, sequence)
1635	std::pair<Type, std::string> Arg1 =
1636	emitDagArg(Arg: DI->getArg(Num: 0), ArgName: std::string(DI->getArgNameStr(Num: 0)));
1637	std::pair<Type, std::string> Arg2 =
1638	emitDagArg(Arg: DI->getArg(Num: 1), ArgName: std::string(DI->getArgNameStr(Num: 1)));
1639	assert_with_loc(Assertion: Arg1.first == Arg2.first,
1640	Str: "Different types in arguments to shuffle!");
1641
1642	SetTheory ST;
1643	SetTheory::RecSet Elts;
1644	ST.addOperator(Name: "lowhalf", std::make_unique<LowHalf>());
1645	ST.addOperator(Name: "highhalf", std::make_unique<HighHalf>());
1646	ST.addOperator(Name: "rev",
1647	std::make_unique<Rev>(args: Arg1.first.getElementSizeInBits()));
1648	ST.addExpander(ClassName: "MaskExpand",
1649	std::make_unique<MaskExpander>(args: Arg1.first.getNumElements()));
1650	ST.evaluate(Expr: DI->getArg(Num: 2), Elts, Loc: std::nullopt);
1651
1652	std::string S = "__builtin_shufflevector(" + Arg1.second + ", " + Arg2.second;
1653	for (auto &E : Elts) {
1654	StringRef Name = E->getName();
1655	assert_with_loc(Assertion: Name.starts_with(Prefix: "sv"),
1656	Str: "Incorrect element kind in shuffle mask!");
1657	S += ", " + Name.drop_front(N: 2).str();
1658	}
1659	S += ")";
1660
1661	// Recalculate the return type - the shuffle may have halved or doubled it.
1662	Type T(Arg1.first);
1663	if (Elts.size() > T.getNumElements()) {
1664	assert_with_loc(
1665	Assertion: Elts.size() == T.getNumElements() * 2,
1666	Str: "Can only double or half the number of elements in a shuffle!");
1667	T.doubleLanes();
1668	} else if (Elts.size() < T.getNumElements()) {
1669	assert_with_loc(
1670	Assertion: Elts.size() == T.getNumElements() / 2,
1671	Str: "Can only double or half the number of elements in a shuffle!");
1672	T.halveLanes();
1673	}
1674
1675	return std::make_pair(x&: T, y&: S);
1676	}
1677
1678	std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagDup(DagInit *DI) {
1679	assert_with_loc(Assertion: DI->getNumArgs() == 1, Str: "dup() expects one argument");
1680	std::pair<Type, std::string> A =
1681	emitDagArg(Arg: DI->getArg(Num: 0), ArgName: std::string(DI->getArgNameStr(Num: 0)));
1682	assert_with_loc(Assertion: A.first.isScalar(), Str: "dup() expects a scalar argument");
1683
1684	Type T = Intr.getBaseType();
1685	assert_with_loc(Assertion: T.isVector(), Str: "dup() used but default type is scalar!");
1686	std::string S = "(" + T.str() + ") {";
1687	for (unsigned I = 0; I < T.getNumElements(); ++I) {
1688	if (I != 0)
1689	S += ", ";
1690	S += A.second;
1691	}
1692	S += "}";
1693
1694	return std::make_pair(x&: T, y&: S);
1695	}
1696
1697	std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagDupTyped(DagInit *DI) {
1698	assert_with_loc(Assertion: DI->getNumArgs() == 2, Str: "dup_typed() expects two arguments");
1699	std::pair<Type, std::string> B =
1700	emitDagArg(Arg: DI->getArg(Num: 1), ArgName: std::string(DI->getArgNameStr(Num: 1)));
1701	assert_with_loc(Assertion: B.first.isScalar(),
1702	Str: "dup_typed() requires a scalar as the second argument");
1703	Type T;
1704	// If the type argument is a constant string, construct the type directly.
1705	if (StringInit *SI = dyn_cast<StringInit>(Val: DI->getArg(Num: 0))) {
1706	T = Type::fromTypedefName(Name: SI->getAsUnquotedString());
1707	assert_with_loc(Assertion: !T.isVoid(), Str: "Unknown typedef");
1708	} else
1709	T = emitDagArg(Arg: DI->getArg(Num: 0), ArgName: std::string(DI->getArgNameStr(Num: 0))).first;
1710
1711	assert_with_loc(Assertion: T.isVector(), Str: "dup_typed() used but target type is scalar!");
1712	std::string S = "(" + T.str() + ") {";
1713	for (unsigned I = 0; I < T.getNumElements(); ++I) {
1714	if (I != 0)
1715	S += ", ";
1716	S += B.second;
1717	}
1718	S += "}";
1719
1720	return std::make_pair(x&: T, y&: S);
1721	}
1722
1723	std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagSplat(DagInit *DI) {
1724	assert_with_loc(Assertion: DI->getNumArgs() == 2, Str: "splat() expects two arguments");
1725	std::pair<Type, std::string> A =
1726	emitDagArg(Arg: DI->getArg(Num: 0), ArgName: std::string(DI->getArgNameStr(Num: 0)));
1727	std::pair<Type, std::string> B =
1728	emitDagArg(Arg: DI->getArg(Num: 1), ArgName: std::string(DI->getArgNameStr(Num: 1)));
1729
1730	assert_with_loc(Assertion: B.first.isScalar(),
1731	Str: "splat() requires a scalar int as the second argument");
1732
1733	std::string S = "__builtin_shufflevector(" + A.second + ", " + A.second;
1734	for (unsigned I = 0; I < Intr.getBaseType().getNumElements(); ++I) {
1735	S += ", " + B.second;
1736	}
1737	S += ")";
1738
1739	return std::make_pair(x: Intr.getBaseType(), y&: S);
1740	}
1741
1742	std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagSaveTemp(DagInit *DI) {
1743	assert_with_loc(Assertion: DI->getNumArgs() == 2, Str: "save_temp() expects two arguments");
1744	std::pair<Type, std::string> A =
1745	emitDagArg(Arg: DI->getArg(Num: 1), ArgName: std::string(DI->getArgNameStr(Num: 1)));
1746
1747	assert_with_loc(Assertion: !A.first.isVoid(),
1748	Str: "Argument to save_temp() must have non-void type!");
1749
1750	std::string N = std::string(DI->getArgNameStr(Num: 0));
1751	assert_with_loc(Assertion: !N.empty(),
1752	Str: "save_temp() expects a name as the first argument");
1753
1754	assert_with_loc(Assertion: Intr.Variables.find(x: N) == Intr.Variables.end(),
1755	Str: "Variable already defined!");
1756	Intr.Variables[N] = Variable(A.first, N + Intr.VariablePostfix);
1757
1758	std::string S =
1759	A.first.str() + " " + Intr.Variables[N].getName() + " = " + A.second;
1760
1761	return std::make_pair(x: Type::getVoid(), y&: S);
1762	}
1763
1764	std::pair<Type, std::string>
1765	Intrinsic::DagEmitter::emitDagNameReplace(DagInit *DI) {
1766	std::string S = Intr.Name;
1767
1768	assert_with_loc(Assertion: DI->getNumArgs() == 2, Str: "name_replace requires 2 arguments!");
1769	std::string ToReplace = cast<StringInit>(Val: DI->getArg(Num: 0))->getAsUnquotedString();
1770	std::string ReplaceWith = cast<StringInit>(Val: DI->getArg(Num: 1))->getAsUnquotedString();
1771
1772	size_t Idx = S.find(str: ToReplace);
1773
1774	assert_with_loc(Assertion: Idx != std::string::npos, Str: "name should contain '" + ToReplace + "'!");
1775	S.replace(pos: Idx, n: ToReplace.size(), str: ReplaceWith);
1776
1777	return std::make_pair(x: Type::getVoid(), y&: S);
1778	}
1779
1780	std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagLiteral(DagInit *DI){
1781	std::string Ty = cast<StringInit>(Val: DI->getArg(Num: 0))->getAsUnquotedString();
1782	std::string Value = cast<StringInit>(Val: DI->getArg(Num: 1))->getAsUnquotedString();
1783	return std::make_pair(x: Type::fromTypedefName(Name: Ty), y&: Value);
1784	}
1785
1786	std::pair<Type, std::string>
1787	Intrinsic::DagEmitter::emitDagArg(Init *Arg, std::string ArgName) {
1788	if (!ArgName.empty()) {
1789	assert_with_loc(Assertion: !Arg->isComplete(),
1790	Str: "Arguments must either be DAGs or names, not both!");
1791	assert_with_loc(Assertion: Intr.Variables.find(x: ArgName) != Intr.Variables.end(),
1792	Str: "Variable not defined!");
1793	Variable &V = Intr.Variables[ArgName];
1794	return std::make_pair(x: V.getType(), y: V.getName());
1795	}
1796
1797	assert(Arg && "Neither ArgName nor Arg?!");
1798	DagInit *DI = dyn_cast<DagInit>(Val: Arg);
1799	assert_with_loc(Assertion: DI, Str: "Arguments must either be DAGs or names!");
1800
1801	return emitDag(DI);
1802	}
1803
1804	std::string Intrinsic::generate() {
1805	// Avoid duplicated code for big and little endian
1806	if (isBigEndianSafe()) {
1807	generateImpl(ReverseArguments: false, NamePrefix: "", CallPrefix: "");
1808	return OS.str();
1809	}
1810	// Little endian intrinsics are simple and don't require any argument
1811	// swapping.
1812	OS << "#ifdef __LITTLE_ENDIAN__\n";
1813
1814	generateImpl(ReverseArguments: false, NamePrefix: "", CallPrefix: "");
1815
1816	OS << "#else\n";
1817
1818	// Big endian intrinsics are more complex. The user intended these
1819	// intrinsics to operate on a vector "as-if" loaded by (V)LDR,
1820	// but we load as-if (V)LD1. So we should swap all arguments and
1821	// swap the return value too.
1822	//
1823	// If we call sub-intrinsics, we should call a version that does
1824	// not re-swap the arguments!
1825	generateImpl(ReverseArguments: true, NamePrefix: "", CallPrefix: "__noswap_");
1826
1827	// If we're needed early, create a non-swapping variant for
1828	// big-endian.
1829	if (NeededEarly) {
1830	generateImpl(ReverseArguments: false, NamePrefix: "__noswap_", CallPrefix: "__noswap_");
1831	}
1832	OS << "#endif\n\n";
1833
1834	return OS.str();
1835	}
1836
1837	void Intrinsic::generateImpl(bool ReverseArguments,
1838	StringRef NamePrefix, StringRef CallPrefix) {
1839	CurrentRecord = R;
1840
1841	// If we call a macro, our local variables may be corrupted due to
1842	// lack of proper lexical scoping. So, add a globally unique postfix
1843	// to every variable.
1844	//
1845	// indexBody() should have set up the Dependencies set by now.
1846	for (auto *I : Dependencies)
1847	if (I->UseMacro) {
1848	VariablePostfix = "_" + utostr(X: Emitter.getUniqueNumber());
1849	break;
1850	}
1851
1852	initVariables();
1853
1854	emitPrototype(NamePrefix);
1855
1856	if (IsUnavailable) {
1857	OS << " __attribute__((unavailable));";
1858	} else {
1859	emitOpeningBrace();
1860	// Emit return variable declaration first as to not trigger
1861	// -Wdeclaration-after-statement.
1862	emitReturnVarDecl();
1863	emitShadowedArgs();
1864	if (ReverseArguments)
1865	emitArgumentReversal();
1866	emitBody(CallPrefix);
1867	if (ReverseArguments)
1868	emitReturnReversal();
1869	emitReturn();
1870	emitClosingBrace();
1871	}
1872	OS << "\n";
1873
1874	CurrentRecord = nullptr;
1875	}
1876
1877	void Intrinsic::indexBody() {
1878	CurrentRecord = R;
1879
1880	initVariables();
1881	// Emit return variable declaration first as to not trigger
1882	// -Wdeclaration-after-statement.
1883	emitReturnVarDecl();
1884	emitBody(CallPrefix: "");
1885	OS.str(s: "");
1886
1887	CurrentRecord = nullptr;
1888	}
1889
1890	//===----------------------------------------------------------------------===//
1891	// NeonEmitter implementation
1892	//===----------------------------------------------------------------------===//
1893
1894	Intrinsic &NeonEmitter::getIntrinsic(StringRef Name, ArrayRef<Type> Types,
1895	std::optional<std::string> MangledName) {
1896	// First, look up the name in the intrinsic map.
1897	assert_with_loc(Assertion: IntrinsicMap.find(x: Name.str()) != IntrinsicMap.end(),
1898	Str: ("Intrinsic '" + Name + "' not found!").str());
1899	auto &V = IntrinsicMap.find(x: Name.str())->second;
1900	std::vector<Intrinsic *> GoodVec;
1901
1902	// Create a string to print if we end up failing.
1903	std::string ErrMsg = "looking up intrinsic '" + Name.str() + "(";
1904	for (unsigned I = 0; I < Types.size(); ++I) {
1905	if (I != 0)
1906	ErrMsg += ", ";
1907	ErrMsg += Types[I].str();
1908	}
1909	ErrMsg += ")'\n";
1910	ErrMsg += "Available overloads:\n";
1911
1912	// Now, look through each intrinsic implementation and see if the types are
1913	// compatible.
1914	for (auto &I : V) {
1915	ErrMsg += " - " + I.getReturnType().str() + " " + I.getMangledName();
1916	ErrMsg += "(";
1917	for (unsigned A = 0; A < I.getNumParams(); ++A) {
1918	if (A != 0)
1919	ErrMsg += ", ";
1920	ErrMsg += I.getParamType(I: A).str();
1921	}
1922	ErrMsg += ")\n";
1923
1924	if (MangledName && MangledName != I.getMangledName(ForceClassS: true))
1925	continue;
1926
1927	if (I.getNumParams() != Types.size())
1928	continue;
1929
1930	unsigned ArgNum = 0;
1931	bool MatchingArgumentTypes = llvm::all_of(Range&: Types, P: [&](const auto &Type) {
1932	return Type == I.getParamType(I: ArgNum++);
1933	});
1934
1935	if (MatchingArgumentTypes)
1936	GoodVec.push_back(x: &I);
1937	}
1938
1939	assert_with_loc(Assertion: !GoodVec.empty(),
1940	Str: "No compatible intrinsic found - " + ErrMsg);
1941	assert_with_loc(Assertion: GoodVec.size() == 1, Str: "Multiple overloads found - " + ErrMsg);
1942
1943	return *GoodVec.front();
1944	}
1945
1946	void NeonEmitter::createIntrinsic(Record *R,
1947	SmallVectorImpl<Intrinsic *> &Out) {
1948	std::string Name = std::string(R->getValueAsString(FieldName: "Name"));
1949	std::string Proto = std::string(R->getValueAsString(FieldName: "Prototype"));
1950	std::string Types = std::string(R->getValueAsString(FieldName: "Types"));
1951	Record *OperationRec = R->getValueAsDef(FieldName: "Operation");
1952	bool BigEndianSafe = R->getValueAsBit(FieldName: "BigEndianSafe");
1953	std::string ArchGuard = std::string(R->getValueAsString(FieldName: "ArchGuard"));
1954	std::string TargetGuard = std::string(R->getValueAsString(FieldName: "TargetGuard"));
1955	bool IsUnavailable = OperationRec->getValueAsBit(FieldName: "Unavailable");
1956	std::string CartesianProductWith = std::string(R->getValueAsString(FieldName: "CartesianProductWith"));
1957
1958	// Set the global current record. This allows assert_with_loc to produce
1959	// decent location information even when highly nested.
1960	CurrentRecord = R;
1961
1962	ListInit *Body = OperationRec->getValueAsListInit(FieldName: "Ops");
1963
1964	std::vector<TypeSpec> TypeSpecs = TypeSpec::fromTypeSpecs(Str: Types);
1965
1966	ClassKind CK = ClassNone;
1967	if (R->getSuperClasses().size() >= 2)
1968	CK = ClassMap[R->getSuperClasses()[1].first];
1969
1970	std::vector<std::pair<TypeSpec, TypeSpec>> NewTypeSpecs;
1971	if (!CartesianProductWith.empty()) {
1972	std::vector<TypeSpec> ProductTypeSpecs = TypeSpec::fromTypeSpecs(Str: CartesianProductWith);
1973	for (auto TS : TypeSpecs) {
1974	Type DefaultT(TS, ".");
1975	for (auto SrcTS : ProductTypeSpecs) {
1976	Type DefaultSrcT(SrcTS, ".");
1977	if (TS == SrcTS ||
1978	DefaultSrcT.getSizeInBits() != DefaultT.getSizeInBits())
1979	continue;
1980	NewTypeSpecs.push_back(x: std::make_pair(x&: TS, y&: SrcTS));
1981	}
1982	}
1983	} else {
1984	for (auto TS : TypeSpecs) {
1985	NewTypeSpecs.push_back(x: std::make_pair(x&: TS, y&: TS));
1986	}
1987	}
1988
1989	llvm::sort(C&: NewTypeSpecs);
1990	NewTypeSpecs.erase(first: std::unique(first: NewTypeSpecs.begin(), last: NewTypeSpecs.end()),
1991	last: NewTypeSpecs.end());
1992	auto &Entry = IntrinsicMap[Name];
1993
1994	for (auto &I : NewTypeSpecs) {
1995	Entry.emplace_back(args&: R, args&: Name, args&: Proto, args&: I.first, args&: I.second, args&: CK, args&: Body, args&: *this,
1996	args&: ArchGuard, args&: TargetGuard, args&: IsUnavailable, args&: BigEndianSafe);
1997	Out.push_back(Elt: &Entry.back());
1998	}
1999
2000	CurrentRecord = nullptr;
2001	}
2002
2003	/// genBuiltinsDef: Generate the BuiltinsARM.def and BuiltinsAArch64.def
2004	/// declaration of builtins, checking for unique builtin declarations.
2005	void NeonEmitter::genBuiltinsDef(raw_ostream &OS,
2006	SmallVectorImpl<Intrinsic *> &Defs) {
2007	OS << "#ifdef GET_NEON_BUILTINS\n";
2008
2009	// We only want to emit a builtin once, and we want to emit them in
2010	// alphabetical order, so use a std::set.
2011	std::set<std::pair<std::string, std::string>> Builtins;
2012
2013	for (auto *Def : Defs) {
2014	if (Def->hasBody())
2015	continue;
2016
2017	std::string S = "__builtin_neon_" + Def->getMangledName() + ", \"";
2018	S += Def->getBuiltinTypeStr();
2019	S += "\", \"n\"";
2020
2021	Builtins.emplace(args&: S, args: Def->getTargetGuard());
2022	}
2023
2024	for (auto &S : Builtins) {
2025	if (S.second == "")
2026	OS << "BUILTIN(";
2027	else
2028	OS << "TARGET_BUILTIN(";
2029	OS << S.first;
2030	if (S.second == "")
2031	OS << ")\n";
2032	else
2033	OS << ", \"" << S.second << "\")\n";
2034	}
2035
2036	OS << "#endif\n\n";
2037	}
2038
2039	void NeonEmitter::genStreamingSVECompatibleList(
2040	raw_ostream &OS, SmallVectorImpl<Intrinsic *> &Defs) {
2041	OS << "#ifdef GET_NEON_STREAMING_COMPAT_FLAG\n";
2042
2043	std::set<std::string> Emitted;
2044	for (auto *Def : Defs) {
2045	// If the def has a body (that is, it has Operation DAGs), it won't call
2046	// __builtin_neon_* so we don't need to generate a definition for it.
2047	if (Def->hasBody())
2048	continue;
2049
2050	std::string Name = Def->getMangledName();
2051	if (Emitted.find(x: Name) != Emitted.end())
2052	continue;
2053
2054	// FIXME: We should make exceptions here for some NEON builtins that are
2055	// permitted in streaming mode.
2056	OS << "case NEON::BI__builtin_neon_" << Name
2057	<< ": BuiltinType = ArmNonStreaming; break;\n";
2058	Emitted.insert(x: Name);
2059	}
2060	OS << "#endif\n\n";
2061	}
2062
2063	/// Generate the ARM and AArch64 overloaded type checking code for
2064	/// SemaChecking.cpp, checking for unique builtin declarations.
2065	void NeonEmitter::genOverloadTypeCheckCode(raw_ostream &OS,
2066	SmallVectorImpl<Intrinsic *> &Defs) {
2067	OS << "#ifdef GET_NEON_OVERLOAD_CHECK\n";
2068
2069	// We record each overload check line before emitting because subsequent Inst
2070	// definitions may extend the number of permitted types (i.e. augment the
2071	// Mask). Use std::map to avoid sorting the table by hash number.
2072	struct OverloadInfo {
2073	uint64_t Mask = 0ULL;
2074	int PtrArgNum = 0;
2075	bool HasConstPtr = false;
2076	OverloadInfo() = default;
2077	};
2078	std::map<std::string, OverloadInfo> OverloadMap;
2079
2080	for (auto *Def : Defs) {
2081	// If the def has a body (that is, it has Operation DAGs), it won't call
2082	// __builtin_neon_* so we don't need to generate a definition for it.
2083	if (Def->hasBody())
2084	continue;
2085	// Functions which have a scalar argument cannot be overloaded, no need to
2086	// check them if we are emitting the type checking code.
2087	if (Def->protoHasScalar())
2088	continue;
2089
2090	uint64_t Mask = 0ULL;
2091	Mask |= 1ULL << Def->getPolymorphicKeyType().getNeonEnum();
2092
2093	// Check if the function has a pointer or const pointer argument.
2094	int PtrArgNum = -1;
2095	bool HasConstPtr = false;
2096	for (unsigned I = 0; I < Def->getNumParams(); ++I) {
2097	const auto &Type = Def->getParamType(I);
2098	if (Type.isPointer()) {
2099	PtrArgNum = I;
2100	HasConstPtr = Type.isConstPointer();
2101	}
2102	}
2103
2104	// For sret builtins, adjust the pointer argument index.
2105	if (PtrArgNum >= 0 && Def->getReturnType().getNumVectors() > 1)
2106	PtrArgNum += 1;
2107
2108	std::string Name = Def->getName();
2109	// Omit type checking for the pointer arguments of vld1_lane, vld1_dup,
2110	// vst1_lane, vldap1_lane, and vstl1_lane intrinsics. Using a pointer to
2111	// the vector element type with one of those operations causes codegen to
2112	// select an aligned load/store instruction. If you want an unaligned
2113	// operation, the pointer argument needs to have less alignment than element
2114	// type, so just accept any pointer type.
2115	if (Name == "vld1_lane" || Name == "vld1_dup" || Name == "vst1_lane" ||
2116	Name == "vldap1_lane" || Name == "vstl1_lane") {
2117	PtrArgNum = -1;
2118	HasConstPtr = false;
2119	}
2120
2121	if (Mask) {
2122	std::string Name = Def->getMangledName();
2123	OverloadMap.insert(x: std::make_pair(x&: Name, y: OverloadInfo()));
2124	OverloadInfo &OI = OverloadMap[Name];
2125	OI.Mask |= Mask;
2126	OI.PtrArgNum |= PtrArgNum;
2127	OI.HasConstPtr = HasConstPtr;
2128	}
2129	}
2130
2131	for (auto &I : OverloadMap) {
2132	OverloadInfo &OI = I.second;
2133
2134	OS << "case NEON::BI__builtin_neon_" << I.first << ": ";
2135	OS << "mask = 0x" << Twine::utohexstr(Val: OI.Mask) << "ULL";
2136	if (OI.PtrArgNum >= 0)
2137	OS << "; PtrArgNum = " << OI.PtrArgNum;
2138	if (OI.HasConstPtr)
2139	OS << "; HasConstPtr = true";
2140	OS << "; break;\n";
2141	}
2142	OS << "#endif\n\n";
2143	}
2144
2145	void NeonEmitter::genIntrinsicRangeCheckCode(raw_ostream &OS,
2146	SmallVectorImpl<Intrinsic *> &Defs) {
2147	OS << "#ifdef GET_NEON_IMMEDIATE_CHECK\n";
2148
2149	std::set<std::string> Emitted;
2150
2151	for (auto *Def : Defs) {
2152	if (Def->hasBody())
2153	continue;
2154	// Functions which do not have an immediate do not need to have range
2155	// checking code emitted.
2156	if (!Def->hasImmediate())
2157	continue;
2158	if (Emitted.find(x: Def->getMangledName()) != Emitted.end())
2159	continue;
2160
2161	std::string LowerBound, UpperBound;
2162
2163	Record *R = Def->getRecord();
2164	if (R->getValueAsBit(FieldName: "isVXAR")) {
2165	//VXAR takes an immediate in the range [0, 63]
2166	LowerBound = "0";
2167	UpperBound = "63";
2168	} else if (R->getValueAsBit(FieldName: "isVCVT_N")) {
2169	// VCVT between floating- and fixed-point values takes an immediate
2170	// in the range [1, 32) for f32 or [1, 64) for f64 or [1, 16) for f16.
2171	LowerBound = "1";
2172	if (Def->getBaseType().getElementSizeInBits() == 16 ||
2173	Def->getName().find(c: 'h') != std::string::npos)
2174	// VCVTh operating on FP16 intrinsics in range [1, 16)
2175	UpperBound = "15";
2176	else if (Def->getBaseType().getElementSizeInBits() == 32)
2177	UpperBound = "31";
2178	else
2179	UpperBound = "63";
2180	} else if (R->getValueAsBit(FieldName: "isScalarShift")) {
2181	// Right shifts have an 'r' in the name, left shifts do not. Convert
2182	// instructions have the same bounds and right shifts.
2183	if (Def->getName().find(c: 'r') != std::string::npos ||
2184	Def->getName().find(s: "cvt") != std::string::npos)
2185	LowerBound = "1";
2186
2187	UpperBound = utostr(X: Def->getReturnType().getElementSizeInBits() - 1);
2188	} else if (R->getValueAsBit(FieldName: "isShift")) {
2189	// Builtins which are overloaded by type will need to have their upper
2190	// bound computed at Sema time based on the type constant.
2191
2192	// Right shifts have an 'r' in the name, left shifts do not.
2193	if (Def->getName().find(c: 'r') != std::string::npos)
2194	LowerBound = "1";
2195	UpperBound = "RFT(TV, true)";
2196	} else if (Def->getClassKind(UseClassBIfScalar: true) == ClassB) {
2197	// ClassB intrinsics have a type (and hence lane number) that is only
2198	// known at runtime.
2199	if (R->getValueAsBit(FieldName: "isLaneQ"))
2200	UpperBound = "RFT(TV, false, true)";
2201	else
2202	UpperBound = "RFT(TV, false, false)";
2203	} else {
2204	// The immediate generally refers to a lane in the preceding argument.
2205	assert(Def->getImmediateIdx() > 0);
2206	Type T = Def->getParamType(I: Def->getImmediateIdx() - 1);
2207	UpperBound = utostr(X: T.getNumElements() - 1);
2208	}
2209
2210	// Calculate the index of the immediate that should be range checked.
2211	unsigned Idx = Def->getNumParams();
2212	if (Def->hasImmediate())
2213	Idx = Def->getGeneratedParamIdx(PIndex: Def->getImmediateIdx());
2214
2215	OS << "case NEON::BI__builtin_neon_" << Def->getMangledName() << ": "
2216	<< "i = " << Idx << ";";
2217	if (!LowerBound.empty())
2218	OS << " l = " << LowerBound << ";";
2219	if (!UpperBound.empty())
2220	OS << " u = " << UpperBound << ";";
2221	OS << " break;\n";
2222
2223	Emitted.insert(x: Def->getMangledName());
2224	}
2225
2226	OS << "#endif\n\n";
2227	}
2228
2229	/// runHeader - Emit a file with sections defining:
2230	/// 1. the NEON section of BuiltinsARM.def and BuiltinsAArch64.def.
2231	/// 2. the SemaChecking code for the type overload checking.
2232	/// 3. the SemaChecking code for validation of intrinsic immediate arguments.
2233	void NeonEmitter::runHeader(raw_ostream &OS) {
2234	std::vector<Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
2235
2236	SmallVector<Intrinsic *, 128> Defs;
2237	for (auto *R : RV)
2238	createIntrinsic(R, Out&: Defs);
2239
2240	// Generate shared BuiltinsXXX.def
2241	genBuiltinsDef(OS, Defs);
2242
2243	// Generate ARM overloaded type checking code for SemaChecking.cpp
2244	genOverloadTypeCheckCode(OS, Defs);
2245
2246	genStreamingSVECompatibleList(OS, Defs);
2247
2248	// Generate ARM range checking code for shift/lane immediates.
2249	genIntrinsicRangeCheckCode(OS, Defs);
2250	}
2251
2252	static void emitNeonTypeDefs(const std::string& types, raw_ostream &OS) {
2253	std::string TypedefTypes(types);
2254	std::vector<TypeSpec> TDTypeVec = TypeSpec::fromTypeSpecs(Str: TypedefTypes);
2255
2256	// Emit vector typedefs.
2257	bool InIfdef = false;
2258	for (auto &TS : TDTypeVec) {
2259	bool IsA64 = false;
2260	Type T(TS, ".");
2261	if (T.isDouble())
2262	IsA64 = true;
2263
2264	if (InIfdef && !IsA64) {
2265	OS << "#endif\n";
2266	InIfdef = false;
2267	}
2268	if (!InIfdef && IsA64) {
2269	OS << "#if defined(__aarch64__) || defined(__arm64ec__)\n";
2270	InIfdef = true;
2271	}
2272
2273	if (T.isPoly())
2274	OS << "typedef __attribute__((neon_polyvector_type(";
2275	else
2276	OS << "typedef __attribute__((neon_vector_type(";
2277
2278	Type T2 = T;
2279	T2.makeScalar();
2280	OS << T.getNumElements() << "))) ";
2281	OS << T2.str();
2282	OS << " " << T.str() << ";\n";
2283	}
2284	if (InIfdef)
2285	OS << "#endif\n";
2286	OS << "\n";
2287
2288	// Emit struct typedefs.
2289	InIfdef = false;
2290	for (unsigned NumMembers = 2; NumMembers <= 4; ++NumMembers) {
2291	for (auto &TS : TDTypeVec) {
2292	bool IsA64 = false;
2293	Type T(TS, ".");
2294	if (T.isDouble())
2295	IsA64 = true;
2296
2297	if (InIfdef && !IsA64) {
2298	OS << "#endif\n";
2299	InIfdef = false;
2300	}
2301	if (!InIfdef && IsA64) {
2302	OS << "#if defined(__aarch64__) || defined(__arm64ec__)\n";
2303	InIfdef = true;
2304	}
2305
2306	const char Mods[] = { static_cast<char>('2' + (NumMembers - 2)), 0};
2307	Type VT(TS, Mods);
2308	OS << "typedef struct " << VT.str() << " {\n";
2309	OS << " " << T.str() << " val";
2310	OS << "[" << NumMembers << "]";
2311	OS << ";\n} ";
2312	OS << VT.str() << ";\n";
2313	OS << "\n";
2314	}
2315	}
2316	if (InIfdef)
2317	OS << "#endif\n";
2318	}
2319
2320	/// run - Read the records in arm_neon.td and output arm_neon.h. arm_neon.h
2321	/// is comprised of type definitions and function declarations.
2322	void NeonEmitter::run(raw_ostream &OS) {
2323	OS << "/*===---- arm_neon.h - ARM Neon intrinsics "
2324	"------------------------------"
2325	"---===\n"
2326	" *\n"
2327	" * Permission is hereby granted, free of charge, to any person "
2328	"obtaining "
2329	"a copy\n"
2330	" * of this software and associated documentation files (the "
2331	"\"Software\"),"
2332	" to deal\n"
2333	" * in the Software without restriction, including without limitation "
2334	"the "
2335	"rights\n"
2336	" * to use, copy, modify, merge, publish, distribute, sublicense, "
2337	"and/or sell\n"
2338	" * copies of the Software, and to permit persons to whom the Software "
2339	"is\n"
2340	" * furnished to do so, subject to the following conditions:\n"
2341	" *\n"
2342	" * The above copyright notice and this permission notice shall be "
2343	"included in\n"
2344	" * all copies or substantial portions of the Software.\n"
2345	" *\n"
2346	" * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, "
2347	"EXPRESS OR\n"
2348	" * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF "
2349	"MERCHANTABILITY,\n"
2350	" * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT "
2351	"SHALL THE\n"
2352	" * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR "
2353	"OTHER\n"
2354	" * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, "
2355	"ARISING FROM,\n"
2356	" * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER "
2357	"DEALINGS IN\n"
2358	" * THE SOFTWARE.\n"
2359	" *\n"
2360	" *===-----------------------------------------------------------------"
2361	"---"
2362	"---===\n"
2363	" */\n\n";
2364
2365	OS << "#ifndef __ARM_NEON_H\n";
2366	OS << "#define __ARM_NEON_H\n\n";
2367
2368	OS << "#ifndef __ARM_FP\n";
2369	OS << "#error \"NEON intrinsics not available with the soft-float ABI. "
2370	"Please use -mfloat-abi=softfp or -mfloat-abi=hard\"\n";
2371	OS << "#else\n\n";
2372
2373	OS << "#if !defined(__ARM_NEON)\n";
2374	OS << "#error \"NEON support not enabled\"\n";
2375	OS << "#else\n\n";
2376
2377	OS << "#include <stdint.h>\n\n";
2378
2379	OS << "#include <arm_bf16.h>\n";
2380
2381	OS << "#include <arm_vector_types.h>\n";
2382
2383	// For now, signedness of polynomial types depends on target
2384	OS << "#if defined(__aarch64__) || defined(__arm64ec__)\n";
2385	OS << "typedef uint8_t poly8_t;\n";
2386	OS << "typedef uint16_t poly16_t;\n";
2387	OS << "typedef uint64_t poly64_t;\n";
2388	OS << "typedef __uint128_t poly128_t;\n";
2389	OS << "#else\n";
2390	OS << "typedef int8_t poly8_t;\n";
2391	OS << "typedef int16_t poly16_t;\n";
2392	OS << "typedef int64_t poly64_t;\n";
2393	OS << "#endif\n";
2394	emitNeonTypeDefs(types: "PcQPcPsQPsPlQPl", OS);
2395
2396	OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
2397	"__nodebug__))\n\n";
2398
2399	SmallVector<Intrinsic *, 128> Defs;
2400	std::vector<Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
2401	for (auto *R : RV)
2402	createIntrinsic(R, Out&: Defs);
2403
2404	for (auto *I : Defs)
2405	I->indexBody();
2406
2407	llvm::stable_sort(Range&: Defs, C: llvm::deref<std::less<>>());
2408
2409	// Only emit a def when its requirements have been met.
2410	// FIXME: This loop could be made faster, but it's fast enough for now.
2411	bool MadeProgress = true;
2412	std::string InGuard;
2413	while (!Defs.empty() && MadeProgress) {
2414	MadeProgress = false;
2415
2416	for (SmallVector<Intrinsic *, 128>::iterator I = Defs.begin();
2417	I != Defs.end(); /*No step*/) {
2418	bool DependenciesSatisfied = true;
2419	for (auto *II : (*I)->getDependencies()) {
2420	if (llvm::is_contained(Range&: Defs, Element: II))
2421	DependenciesSatisfied = false;
2422	}
2423	if (!DependenciesSatisfied) {
2424	// Try the next one.
2425	++I;
2426	continue;
2427	}
2428
2429	// Emit #endif/#if pair if needed.
2430	if ((*I)->getArchGuard() != InGuard) {
2431	if (!InGuard.empty())
2432	OS << "#endif\n";
2433	InGuard = (*I)->getArchGuard();
2434	if (!InGuard.empty())
2435	OS << "#if " << InGuard << "\n";
2436	}
2437
2438	// Actually generate the intrinsic code.
2439	OS << (*I)->generate();
2440
2441	MadeProgress = true;
2442	I = Defs.erase(CI: I);
2443	}
2444	}
2445	assert(Defs.empty() && "Some requirements were not satisfied!");
2446	if (!InGuard.empty())
2447	OS << "#endif\n";
2448
2449	OS << "\n";
2450	OS << "#undef __ai\n\n";
2451	OS << "#endif /* if !defined(__ARM_NEON) */\n";
2452	OS << "#endif /* ifndef __ARM_FP */\n";
2453	OS << "#endif /* __ARM_NEON_H */\n";
2454	}
2455
2456	/// run - Read the records in arm_fp16.td and output arm_fp16.h. arm_fp16.h
2457	/// is comprised of type definitions and function declarations.
2458	void NeonEmitter::runFP16(raw_ostream &OS) {
2459	OS << "/*===---- arm_fp16.h - ARM FP16 intrinsics "
2460	"------------------------------"
2461	"---===\n"
2462	" *\n"
2463	" * Permission is hereby granted, free of charge, to any person "
2464	"obtaining a copy\n"
2465	" * of this software and associated documentation files (the "
2466	"\"Software\"), to deal\n"
2467	" * in the Software without restriction, including without limitation "
2468	"the rights\n"
2469	" * to use, copy, modify, merge, publish, distribute, sublicense, "
2470	"and/or sell\n"
2471	" * copies of the Software, and to permit persons to whom the Software "
2472	"is\n"
2473	" * furnished to do so, subject to the following conditions:\n"
2474	" *\n"
2475	" * The above copyright notice and this permission notice shall be "
2476	"included in\n"
2477	" * all copies or substantial portions of the Software.\n"
2478	" *\n"
2479	" * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, "
2480	"EXPRESS OR\n"
2481	" * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF "
2482	"MERCHANTABILITY,\n"
2483	" * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT "
2484	"SHALL THE\n"
2485	" * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR "
2486	"OTHER\n"
2487	" * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, "
2488	"ARISING FROM,\n"
2489	" * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER "
2490	"DEALINGS IN\n"
2491	" * THE SOFTWARE.\n"
2492	" *\n"
2493	" *===-----------------------------------------------------------------"
2494	"---"
2495	"---===\n"
2496	" */\n\n";
2497
2498	OS << "#ifndef __ARM_FP16_H\n";
2499	OS << "#define __ARM_FP16_H\n\n";
2500
2501	OS << "#include <stdint.h>\n\n";
2502
2503	OS << "typedef __fp16 float16_t;\n";
2504
2505	OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
2506	"__nodebug__))\n\n";
2507
2508	SmallVector<Intrinsic *, 128> Defs;
2509	std::vector<Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
2510	for (auto *R : RV)
2511	createIntrinsic(R, Out&: Defs);
2512
2513	for (auto *I : Defs)
2514	I->indexBody();
2515
2516	llvm::stable_sort(Range&: Defs, C: llvm::deref<std::less<>>());
2517
2518	// Only emit a def when its requirements have been met.
2519	// FIXME: This loop could be made faster, but it's fast enough for now.
2520	bool MadeProgress = true;
2521	std::string InGuard;
2522	while (!Defs.empty() && MadeProgress) {
2523	MadeProgress = false;
2524
2525	for (SmallVector<Intrinsic *, 128>::iterator I = Defs.begin();
2526	I != Defs.end(); /*No step*/) {
2527	bool DependenciesSatisfied = true;
2528	for (auto *II : (*I)->getDependencies()) {
2529	if (llvm::is_contained(Range&: Defs, Element: II))
2530	DependenciesSatisfied = false;
2531	}
2532	if (!DependenciesSatisfied) {
2533	// Try the next one.
2534	++I;
2535	continue;
2536	}
2537
2538	// Emit #endif/#if pair if needed.
2539	if ((*I)->getArchGuard() != InGuard) {
2540	if (!InGuard.empty())
2541	OS << "#endif\n";
2542	InGuard = (*I)->getArchGuard();
2543	if (!InGuard.empty())
2544	OS << "#if " << InGuard << "\n";
2545	}
2546
2547	// Actually generate the intrinsic code.
2548	OS << (*I)->generate();
2549
2550	MadeProgress = true;
2551	I = Defs.erase(CI: I);
2552	}
2553	}
2554	assert(Defs.empty() && "Some requirements were not satisfied!");
2555	if (!InGuard.empty())
2556	OS << "#endif\n";
2557
2558	OS << "\n";
2559	OS << "#undef __ai\n\n";
2560	OS << "#endif /* __ARM_FP16_H */\n";
2561	}
2562
2563	void NeonEmitter::runVectorTypes(raw_ostream &OS) {
2564	OS << "/*===---- arm_vector_types - ARM vector type "
2565	"------===\n"
2566	" *\n"
2567	" *\n"
2568	" * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
2569	"Exceptions.\n"
2570	" * See https://llvm.org/LICENSE.txt for license information.\n"
2571	" * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
2572	" *\n"
2573	" *===-----------------------------------------------------------------"
2574	"------===\n"
2575	" */\n\n";
2576	OS << "#if !defined(__ARM_NEON_H) && !defined(__ARM_SVE_H)\n";
2577	OS << "#error \"This file should not be used standalone. Please include"
2578	" arm_neon.h or arm_sve.h instead\"\n\n";
2579	OS << "#endif\n";
2580	OS << "#ifndef __ARM_NEON_TYPES_H\n";
2581	OS << "#define __ARM_NEON_TYPES_H\n";
2582	OS << "typedef float float32_t;\n";
2583	OS << "typedef __fp16 float16_t;\n";
2584
2585	OS << "#if defined(__aarch64__) || defined(__arm64ec__)\n";
2586	OS << "typedef double float64_t;\n";
2587	OS << "#endif\n\n";
2588
2589	emitNeonTypeDefs(types: "cQcsQsiQilQlUcQUcUsQUsUiQUiUlQUlhQhfQfdQd", OS);
2590
2591	emitNeonTypeDefs(types: "bQb", OS);
2592	OS << "#endif // __ARM_NEON_TYPES_H\n";
2593	}
2594
2595	void NeonEmitter::runBF16(raw_ostream &OS) {
2596	OS << "/*===---- arm_bf16.h - ARM BF16 intrinsics "
2597	"-----------------------------------===\n"
2598	" *\n"
2599	" *\n"
2600	" * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
2601	"Exceptions.\n"
2602	" * See https://llvm.org/LICENSE.txt for license information.\n"
2603	" * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
2604	" *\n"
2605	" *===-----------------------------------------------------------------"
2606	"------===\n"
2607	" */\n\n";
2608
2609	OS << "#ifndef __ARM_BF16_H\n";
2610	OS << "#define __ARM_BF16_H\n\n";
2611
2612	OS << "typedef __bf16 bfloat16_t;\n";
2613
2614	OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
2615	"__nodebug__))\n\n";
2616
2617	SmallVector<Intrinsic *, 128> Defs;
2618	std::vector<Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
2619	for (auto *R : RV)
2620	createIntrinsic(R, Out&: Defs);
2621
2622	for (auto *I : Defs)
2623	I->indexBody();
2624
2625	llvm::stable_sort(Range&: Defs, C: llvm::deref<std::less<>>());
2626
2627	// Only emit a def when its requirements have been met.
2628	// FIXME: This loop could be made faster, but it's fast enough for now.
2629	bool MadeProgress = true;
2630	std::string InGuard;
2631	while (!Defs.empty() && MadeProgress) {
2632	MadeProgress = false;
2633
2634	for (SmallVector<Intrinsic *, 128>::iterator I = Defs.begin();
2635	I != Defs.end(); /*No step*/) {
2636	bool DependenciesSatisfied = true;
2637	for (auto *II : (*I)->getDependencies()) {
2638	if (llvm::is_contained(Range&: Defs, Element: II))
2639	DependenciesSatisfied = false;
2640	}
2641	if (!DependenciesSatisfied) {
2642	// Try the next one.
2643	++I;
2644	continue;
2645	}
2646
2647	// Emit #endif/#if pair if needed.
2648	if ((*I)->getArchGuard() != InGuard) {
2649	if (!InGuard.empty())
2650	OS << "#endif\n";
2651	InGuard = (*I)->getArchGuard();
2652	if (!InGuard.empty())
2653	OS << "#if " << InGuard << "\n";
2654	}
2655
2656	// Actually generate the intrinsic code.
2657	OS << (*I)->generate();
2658
2659	MadeProgress = true;
2660	I = Defs.erase(CI: I);
2661	}
2662	}
2663	assert(Defs.empty() && "Some requirements were not satisfied!");
2664	if (!InGuard.empty())
2665	OS << "#endif\n";
2666
2667	OS << "\n";
2668	OS << "#undef __ai\n\n";
2669
2670	OS << "#endif\n";
2671	}
2672
2673	void clang::EmitNeon(RecordKeeper &Records, raw_ostream &OS) {
2674	NeonEmitter(Records).run(OS);
2675	}
2676
2677	void clang::EmitFP16(RecordKeeper &Records, raw_ostream &OS) {
2678	NeonEmitter(Records).runFP16(OS);
2679	}
2680
2681	void clang::EmitBF16(RecordKeeper &Records, raw_ostream &OS) {
2682	NeonEmitter(Records).runBF16(OS);
2683	}
2684
2685	void clang::EmitNeonSema(RecordKeeper &Records, raw_ostream &OS) {
2686	NeonEmitter(Records).runHeader(OS);
2687	}
2688
2689	void clang::EmitVectorTypes(RecordKeeper &Records, raw_ostream &OS) {
2690	NeonEmitter(Records).runVectorTypes(OS);
2691	}
2692
2693	void clang::EmitNeonTest(RecordKeeper &Records, raw_ostream &OS) {
2694	llvm_unreachable("Neon test generation no longer implemented!");
2695	}
2696