1//===--- RISCVVIntrinsicUtils.h - RISC-V Vector Intrinsic Utils -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
10#define CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
11
12#include "llvm/ADT/ArrayRef.h"
13#include "llvm/ADT/BitmaskEnum.h"
14#include "llvm/ADT/SmallVector.h"
15#include "llvm/ADT/StringRef.h"
16#include <cstdint>
17#include <optional>
18#include <set>
19#include <string>
20#include <unordered_map>
21#include <vector>
22
23namespace llvm {
24class raw_ostream;
25} // end namespace llvm
26
27namespace clang {
28namespace RISCV {
29
30using VScaleVal = std::optional<unsigned>;
31
32// Modifier for vector type.
33enum class VectorTypeModifier : uint8_t {
34 NoModifier,
35 Widening2XVector,
36 Widening4XVector,
37 Widening8XVector,
38 MaskVector,
39 Log2EEW3,
40 Log2EEW4,
41 Log2EEW5,
42 Log2EEW6,
43 FixedSEW8,
44 FixedSEW16,
45 FixedSEW32,
46 FixedSEW64,
47 LFixedLog2LMULN3,
48 LFixedLog2LMULN2,
49 LFixedLog2LMULN1,
50 LFixedLog2LMUL0,
51 LFixedLog2LMUL1,
52 LFixedLog2LMUL2,
53 LFixedLog2LMUL3,
54 SFixedLog2LMULN3,
55 SFixedLog2LMULN2,
56 SFixedLog2LMULN1,
57 SFixedLog2LMUL0,
58 SFixedLog2LMUL1,
59 SFixedLog2LMUL2,
60 SFixedLog2LMUL3,
61 SEFixedLog2LMULN3,
62 SEFixedLog2LMULN2,
63 SEFixedLog2LMULN1,
64 SEFixedLog2LMUL0,
65 SEFixedLog2LMUL1,
66 SEFixedLog2LMUL2,
67 SEFixedLog2LMUL3,
68 Tuple2,
69 Tuple3,
70 Tuple4,
71 Tuple5,
72 Tuple6,
73 Tuple7,
74 Tuple8,
75};
76
77// Similar to basic type but used to describe what's kind of type related to
78// basic vector type, used to compute type info of arguments.
79enum class BaseTypeModifier : uint8_t {
80 Invalid,
81 Scalar,
82 Vector,
83 Void,
84 SizeT,
85 Ptrdiff,
86 UnsignedLong,
87 SignedLong,
88 Float32
89};
90
91// Modifier for type, used for both scalar and vector types.
92enum class TypeModifier : uint8_t {
93 NoModifier = 0,
94 Pointer = 1 << 0,
95 Const = 1 << 1,
96 Immediate = 1 << 2,
97 UnsignedInteger = 1 << 3,
98 SignedInteger = 1 << 4,
99 Float = 1 << 5,
100 BFloat = 1 << 6,
101 // LMUL1 should be kind of VectorTypeModifier, but that might come with
102 // Widening2XVector for widening reduction.
103 // However that might require VectorTypeModifier become bitmask rather than
104 // simple enum, so we decide keek LMUL1 in TypeModifier for code size
105 // optimization of clang binary size.
106 LMUL1 = 1 << 7,
107 MaxOffset = 7,
108 LLVM_MARK_AS_BITMASK_ENUM(LMUL1),
109};
110
111class Policy {
112public:
113 enum PolicyType {
114 Undisturbed,
115 Agnostic,
116 };
117
118private:
119 // The default assumption for an RVV instruction is TAMA, as an undisturbed
120 // policy generally will affect the performance of an out-of-order core.
121 const PolicyType TailPolicy = Agnostic;
122 const PolicyType MaskPolicy = Agnostic;
123
124public:
125 Policy() = default;
126 Policy(PolicyType TailPolicy) : TailPolicy(TailPolicy) {}
127 Policy(PolicyType TailPolicy, PolicyType MaskPolicy)
128 : TailPolicy(TailPolicy), MaskPolicy(MaskPolicy) {}
129
130 bool isTAMAPolicy() const {
131 return TailPolicy == Agnostic && MaskPolicy == Agnostic;
132 }
133
134 bool isTAMUPolicy() const {
135 return TailPolicy == Agnostic && MaskPolicy == Undisturbed;
136 }
137
138 bool isTUMAPolicy() const {
139 return TailPolicy == Undisturbed && MaskPolicy == Agnostic;
140 }
141
142 bool isTUMUPolicy() const {
143 return TailPolicy == Undisturbed && MaskPolicy == Undisturbed;
144 }
145
146 bool isTAPolicy() const { return TailPolicy == Agnostic; }
147
148 bool isTUPolicy() const { return TailPolicy == Undisturbed; }
149
150 bool isMAPolicy() const { return MaskPolicy == Agnostic; }
151
152 bool isMUPolicy() const { return MaskPolicy == Undisturbed; }
153
154 bool operator==(const Policy &Other) const {
155 return TailPolicy == Other.TailPolicy && MaskPolicy == Other.MaskPolicy;
156 }
157
158 bool operator!=(const Policy &Other) const { return !(*this == Other); }
159
160 bool operator<(const Policy &Other) const {
161 // Just for maintain the old order for quick test.
162 if (MaskPolicy != Other.MaskPolicy)
163 return Other.MaskPolicy < MaskPolicy;
164 return TailPolicy < Other.TailPolicy;
165 }
166};
167
168// PrototypeDescriptor is used to compute type info of arguments or return
169// value.
170struct PrototypeDescriptor {
171 constexpr PrototypeDescriptor() = default;
172 constexpr PrototypeDescriptor(
173 BaseTypeModifier PT,
174 VectorTypeModifier VTM = VectorTypeModifier::NoModifier,
175 TypeModifier TM = TypeModifier::NoModifier)
176 : PT(static_cast<uint8_t>(PT)), VTM(static_cast<uint8_t>(VTM)),
177 TM(static_cast<uint8_t>(TM)) {}
178 constexpr PrototypeDescriptor(uint8_t PT, uint8_t VTM, uint8_t TM)
179 : PT(PT), VTM(VTM), TM(TM) {}
180
181 uint8_t PT = static_cast<uint8_t>(BaseTypeModifier::Invalid);
182 uint8_t VTM = static_cast<uint8_t>(VectorTypeModifier::NoModifier);
183 uint8_t TM = static_cast<uint8_t>(TypeModifier::NoModifier);
184
185 bool operator!=(const PrototypeDescriptor &PD) const {
186 return !(*this == PD);
187 }
188 bool operator==(const PrototypeDescriptor &PD) const {
189 return PD.PT == PT && PD.VTM == VTM && PD.TM == TM;
190 }
191 bool operator<(const PrototypeDescriptor &PD) const {
192 return std::tie(args: PT, args: VTM, args: TM) < std::tie(args: PD.PT, args: PD.VTM, args: PD.TM);
193 }
194 static const PrototypeDescriptor Mask;
195 static const PrototypeDescriptor Vector;
196 static const PrototypeDescriptor VL;
197 static std::optional<PrototypeDescriptor>
198 parsePrototypeDescriptor(llvm::StringRef PrototypeStr);
199};
200
201llvm::SmallVector<PrototypeDescriptor>
202parsePrototypes(llvm::StringRef Prototypes);
203
204// Basic type of vector type.
205enum class BasicType : uint8_t {
206 Unknown = 0,
207 Int8 = 1 << 0,
208 Int16 = 1 << 1,
209 Int32 = 1 << 2,
210 Int64 = 1 << 3,
211 BFloat16 = 1 << 4,
212 Float16 = 1 << 5,
213 Float32 = 1 << 6,
214 Float64 = 1 << 7,
215 MaxOffset = 7,
216 LLVM_MARK_AS_BITMASK_ENUM(Float64),
217};
218
219// Type of vector type.
220enum ScalarTypeKind : uint8_t {
221 Void,
222 Size_t,
223 Ptrdiff_t,
224 UnsignedLong,
225 SignedLong,
226 Boolean,
227 SignedInteger,
228 UnsignedInteger,
229 Float,
230 BFloat,
231 Invalid,
232 Undefined,
233};
234
235// Exponential LMUL
236struct LMULType {
237 int Log2LMUL;
238 LMULType(int Log2LMUL);
239 // Return the C/C++ string representation of LMUL
240 std::string str() const;
241 std::optional<unsigned> getScale(unsigned ElementBitwidth) const;
242 void MulLog2LMUL(int Log2LMUL);
243};
244
245class RVVType;
246using RVVTypePtr = RVVType *;
247using RVVTypes = std::vector<RVVTypePtr>;
248class RVVTypeCache;
249
250// This class is compact representation of a valid and invalid RVVType.
251class RVVType {
252 friend class RVVTypeCache;
253
254 BasicType BT;
255 ScalarTypeKind ScalarType = Undefined;
256 LMULType LMUL;
257 bool IsPointer = false;
258 // IsConstant indices are "int", but have the constant expression.
259 bool IsImmediate = false;
260 // Const qualifier for pointer to const object or object of const type.
261 bool IsConstant = false;
262 unsigned ElementBitwidth = 0;
263 VScaleVal Scale = 0;
264 bool Valid;
265 bool IsTuple = false;
266 unsigned NF = 0;
267
268 std::string BuiltinStr;
269 std::string ClangBuiltinStr;
270 std::string Str;
271 std::string ShortStr;
272
273 enum class FixedLMULType { LargerThan, SmallerThan, SmallerOrEqual };
274
275 RVVType(BasicType BT, int Log2LMUL, const PrototypeDescriptor &Profile);
276
277public:
278 // Return the string representation of a type, which is an encoded string for
279 // passing to the BUILTIN() macro in Builtins.def.
280 const std::string &getBuiltinStr() const { return BuiltinStr; }
281
282 // Return the clang builtin type for RVV vector type which are used in the
283 // riscv_vector.h header file.
284 const std::string &getClangBuiltinStr() const { return ClangBuiltinStr; }
285
286 // Return the C/C++ string representation of a type for use in the
287 // riscv_vector.h header file.
288 const std::string &getTypeStr() const { return Str; }
289
290 // Return the short name of a type for C/C++ name suffix.
291 const std::string &getShortStr() {
292 // Not all types are used in short name, so compute the short name by
293 // demanded.
294 if (ShortStr.empty())
295 initShortStr();
296 return ShortStr;
297 }
298
299 bool isValid() const { return Valid; }
300 bool isScalar() const { return Scale && *Scale == 0; }
301 bool isVector() const { return Scale && *Scale != 0; }
302 bool isVector(unsigned Width) const {
303 return isVector() && ElementBitwidth == Width;
304 }
305 bool isFloat() const { return ScalarType == ScalarTypeKind::Float; }
306 bool isBFloat() const { return ScalarType == ScalarTypeKind::BFloat; }
307 bool isSignedInteger() const {
308 return ScalarType == ScalarTypeKind::SignedInteger;
309 }
310 bool isFloatVector(unsigned Width) const {
311 return isVector() && isFloat() && ElementBitwidth == Width;
312 }
313 bool isFloat(unsigned Width) const {
314 return isFloat() && ElementBitwidth == Width;
315 }
316 bool isConstant() const { return IsConstant; }
317 bool isPointer() const { return IsPointer; }
318 bool isTuple() const { return IsTuple; }
319 unsigned getElementBitwidth() const { return ElementBitwidth; }
320
321 ScalarTypeKind getScalarType() const { return ScalarType; }
322 VScaleVal getScale() const { return Scale; }
323 unsigned getNF() const {
324 assert(NF > 1 && NF <= 8 && "Only legal NF should be fetched");
325 return NF;
326 }
327
328private:
329 // Verify RVV vector type and set Valid.
330 bool verifyType() const;
331
332 // Creates a type based on basic types of TypeRange
333 void applyBasicType();
334
335 // Applies a prototype modifier to the current type. The result maybe an
336 // invalid type.
337 void applyModifier(const PrototypeDescriptor &prototype);
338
339 void applyLog2EEW(unsigned Log2EEW);
340 void applyFixedSEW(unsigned NewSEW);
341 void applyFixedLog2LMUL(int Log2LMUL, enum FixedLMULType Type);
342
343 // Compute and record a string for legal type.
344 void initBuiltinStr();
345 // Compute and record a builtin RVV vector type string.
346 void initClangBuiltinStr();
347 // Compute and record a type string for used in the header.
348 void initTypeStr();
349 // Compute and record a short name of a type for C/C++ name suffix.
350 void initShortStr();
351};
352
353// This class is used to manage RVVType, RVVType should only created by this
354// class, also provided thread-safe cache capability.
355class RVVTypeCache {
356private:
357 std::unordered_map<uint64_t, RVVType> LegalTypes;
358 std::set<uint64_t> IllegalTypes;
359
360public:
361 /// Compute output and input types by applying different config (basic type
362 /// and LMUL with type transformers). It also record result of type in legal
363 /// or illegal set to avoid compute the same config again. The result maybe
364 /// have illegal RVVType.
365 std::optional<RVVTypes>
366 computeTypes(BasicType BT, int Log2LMUL, unsigned NF,
367 llvm::ArrayRef<PrototypeDescriptor> Prototype);
368 std::optional<RVVTypePtr> computeType(BasicType BT, int Log2LMUL,
369 PrototypeDescriptor Proto);
370};
371
372enum PolicyScheme : uint8_t {
373 SchemeNone,
374 // Passthru operand is at first parameter in C builtin.
375 HasPassthruOperand,
376 HasPolicyOperand,
377};
378
379// TODO refactor RVVIntrinsic class design after support all intrinsic
380// combination. This represents an instantiation of an intrinsic with a
381// particular type and prototype
382class RVVIntrinsic {
383
384private:
385 std::string BuiltinName; // Builtin name
386 std::string Name; // C intrinsic name.
387 std::string OverloadedName;
388 std::string IRName;
389 bool IsMasked;
390 bool HasMaskedOffOperand;
391 bool HasVL;
392 PolicyScheme Scheme;
393 bool SupportOverloading;
394 bool HasBuiltinAlias;
395 std::string ManualCodegen;
396 RVVTypePtr OutputType; // Builtin output type
397 RVVTypes InputTypes; // Builtin input types
398 // The types we use to obtain the specific LLVM intrinsic. They are index of
399 // InputTypes. -1 means the return type.
400 std::vector<int64_t> IntrinsicTypes;
401 unsigned NF = 1;
402 Policy PolicyAttrs;
403
404public:
405 RVVIntrinsic(llvm::StringRef Name, llvm::StringRef Suffix,
406 llvm::StringRef OverloadedName, llvm::StringRef OverloadedSuffix,
407 llvm::StringRef IRName, bool IsMasked, bool HasMaskedOffOperand,
408 bool HasVL, PolicyScheme Scheme, bool SupportOverloading,
409 bool HasBuiltinAlias, llvm::StringRef ManualCodegen,
410 const RVVTypes &Types,
411 const std::vector<int64_t> &IntrinsicTypes,
412 const std::vector<llvm::StringRef> &RequiredFeatures,
413 unsigned NF, Policy PolicyAttrs, bool HasFRMRoundModeOp);
414 ~RVVIntrinsic() = default;
415
416 RVVTypePtr getOutputType() const { return OutputType; }
417 const RVVTypes &getInputTypes() const { return InputTypes; }
418 llvm::StringRef getBuiltinName() const { return BuiltinName; }
419 bool hasMaskedOffOperand() const { return HasMaskedOffOperand; }
420 bool hasVL() const { return HasVL; }
421 bool hasPolicy() const { return Scheme != PolicyScheme::SchemeNone; }
422 bool hasPassthruOperand() const {
423 return Scheme == PolicyScheme::HasPassthruOperand;
424 }
425 bool hasPolicyOperand() const {
426 return Scheme == PolicyScheme::HasPolicyOperand;
427 }
428 bool supportOverloading() const { return SupportOverloading; }
429 bool hasBuiltinAlias() const { return HasBuiltinAlias; }
430 bool hasManualCodegen() const { return !ManualCodegen.empty(); }
431 bool isMasked() const { return IsMasked; }
432 llvm::StringRef getIRName() const { return IRName; }
433 llvm::StringRef getManualCodegen() const { return ManualCodegen; }
434 PolicyScheme getPolicyScheme() const { return Scheme; }
435 unsigned getNF() const { return NF; }
436 const std::vector<int64_t> &getIntrinsicTypes() const {
437 return IntrinsicTypes;
438 }
439 Policy getPolicyAttrs() const {
440 return PolicyAttrs;
441 }
442 unsigned getPolicyAttrsBits() const {
443 // CGBuiltin.cpp
444 // The 0th bit simulates the `vta` of RVV
445 // The 1st bit simulates the `vma` of RVV
446 // int PolicyAttrs = 0;
447
448 if (PolicyAttrs.isTUMAPolicy())
449 return 2;
450 if (PolicyAttrs.isTAMAPolicy())
451 return 3;
452 if (PolicyAttrs.isTUMUPolicy())
453 return 0;
454 if (PolicyAttrs.isTAMUPolicy())
455 return 1;
456
457 llvm_unreachable("unsupport policy");
458 return 0;
459 }
460
461 // Return the type string for a BUILTIN() macro in Builtins.def.
462 std::string getBuiltinTypeStr() const;
463
464 static std::string
465 getSuffixStr(RVVTypeCache &TypeCache, BasicType Type, int Log2LMUL,
466 llvm::ArrayRef<PrototypeDescriptor> PrototypeDescriptors);
467
468 static llvm::SmallVector<PrototypeDescriptor>
469 computeBuiltinTypes(llvm::ArrayRef<PrototypeDescriptor> Prototype,
470 bool IsMasked, bool HasMaskedOffOperand, bool HasVL,
471 unsigned NF, PolicyScheme DefaultScheme,
472 Policy PolicyAttrs, bool IsTuple);
473
474 static llvm::SmallVector<Policy> getSupportedUnMaskedPolicies();
475 static llvm::SmallVector<Policy>
476 getSupportedMaskedPolicies(bool HasTailPolicy, bool HasMaskPolicy);
477
478 static void updateNamesAndPolicy(bool IsMasked, bool HasPolicy,
479 std::string &Name, std::string &BuiltinName,
480 std::string &OverloadedName,
481 Policy &PolicyAttrs, bool HasFRMRoundModeOp);
482};
483
484// RVVRequire should be sync'ed with target features, but only
485// required features used in riscv_vector.td.
486enum RVVRequire : uint32_t {
487 RVV_REQ_None = 0,
488 RVV_REQ_RV64 = 1 << 0,
489 RVV_REQ_Zvfhmin = 1 << 1,
490 RVV_REQ_Xsfvcp = 1 << 2,
491 RVV_REQ_Xsfvfnrclipxfqf = 1 << 3,
492 RVV_REQ_Xsfvfwmaccqqq = 1 << 4,
493 RVV_REQ_Xsfvqmaccdod = 1 << 5,
494 RVV_REQ_Xsfvqmaccqoq = 1 << 6,
495 RVV_REQ_Zvbb = 1 << 7,
496 RVV_REQ_Zvbc = 1 << 8,
497 RVV_REQ_Zvkb = 1 << 9,
498 RVV_REQ_Zvkg = 1 << 10,
499 RVV_REQ_Zvkned = 1 << 11,
500 RVV_REQ_Zvknha = 1 << 12,
501 RVV_REQ_Zvknhb = 1 << 13,
502 RVV_REQ_Zvksed = 1 << 14,
503 RVV_REQ_Zvksh = 1 << 15,
504 RVV_REQ_Zvfbfwma = 1 << 16,
505 RVV_REQ_Zvfbfmin = 1 << 17,
506 RVV_REQ_Experimental = 1 << 18,
507
508 LLVM_MARK_AS_BITMASK_ENUM(RVV_REQ_Experimental)
509};
510
511// Raw RVV intrinsic info, used to expand later.
512// This struct is highly compact for minimized code size.
513struct RVVIntrinsicRecord {
514 // Intrinsic name, e.g. vadd_vv
515 const char *Name;
516
517 // Overloaded intrinsic name, could be empty if it can be computed from Name.
518 // e.g. vadd
519 const char *OverloadedName;
520
521 // Prototype for this intrinsic, index of RVVSignatureTable.
522 uint16_t PrototypeIndex;
523
524 // Suffix of intrinsic name, index of RVVSignatureTable.
525 uint16_t SuffixIndex;
526
527 // Suffix of overloaded intrinsic name, index of RVVSignatureTable.
528 uint16_t OverloadedSuffixIndex;
529
530 // Length of the prototype.
531 uint8_t PrototypeLength;
532
533 // Length of intrinsic name suffix.
534 uint8_t SuffixLength;
535
536 // Length of overloaded intrinsic suffix.
537 uint8_t OverloadedSuffixSize;
538
539 // Required target features for this intrinsic.
540 uint32_t RequiredExtensions;
541
542 // Supported type, mask of BasicType.
543 uint8_t TypeRangeMask;
544
545 // Supported LMUL.
546 uint8_t Log2LMULMask;
547
548 // Number of fields, greater than 1 if it's segment load/store.
549 uint8_t NF;
550
551 bool HasMasked : 1;
552 bool HasVL : 1;
553 bool HasMaskedOffOperand : 1;
554 bool HasTailPolicy : 1;
555 bool HasMaskPolicy : 1;
556 bool HasFRMRoundModeOp : 1;
557 bool IsTuple : 1;
558 LLVM_PREFERRED_TYPE(PolicyScheme)
559 uint8_t UnMaskedPolicyScheme : 2;
560 LLVM_PREFERRED_TYPE(PolicyScheme)
561 uint8_t MaskedPolicyScheme : 2;
562};
563
564llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
565 const RVVIntrinsicRecord &RVVInstrRecord);
566
567LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
568} // end namespace RISCV
569
570} // end namespace clang
571
572#endif // CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
573

source code of clang/include/clang/Support/RISCVVIntrinsicUtils.h