1//===- llvm/CodeGen/GlobalISel/LegalizerInfo.h ------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// Interface for Targets to specify which operations they can successfully
10/// select and how the others should be expanded most efficiently.
11///
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H
15#define LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H
16
17#include "llvm/ADT/SmallBitVector.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h"
20#include "llvm/CodeGen/MachineMemOperand.h"
21#include "llvm/CodeGen/TargetOpcodes.h"
22#include "llvm/CodeGenTypes/LowLevelType.h"
23#include "llvm/MC/MCInstrDesc.h"
24#include "llvm/Support/AtomicOrdering.h"
25#include "llvm/Support/CommandLine.h"
26#include <cassert>
27#include <cstdint>
28#include <tuple>
29#include <utility>
30
31namespace llvm {
32
33extern cl::opt<bool> DisableGISelLegalityCheck;
34
35class MachineFunction;
36class raw_ostream;
37class LegalizerHelper;
38class LostDebugLocObserver;
39class MachineInstr;
40class MachineRegisterInfo;
41class MCInstrInfo;
42
43namespace LegalizeActions {
44enum LegalizeAction : std::uint8_t {
45 /// The operation is expected to be selectable directly by the target, and
46 /// no transformation is necessary.
47 Legal,
48
49 /// The operation should be synthesized from multiple instructions acting on
50 /// a narrower scalar base-type. For example a 64-bit add might be
51 /// implemented in terms of 32-bit add-with-carry.
52 NarrowScalar,
53
54 /// The operation should be implemented in terms of a wider scalar
55 /// base-type. For example a <2 x s8> add could be implemented as a <2
56 /// x s32> add (ignoring the high bits).
57 WidenScalar,
58
59 /// The (vector) operation should be implemented by splitting it into
60 /// sub-vectors where the operation is legal. For example a <8 x s64> add
61 /// might be implemented as 4 separate <2 x s64> adds. There can be a leftover
62 /// if there are not enough elements for last sub-vector e.g. <7 x s64> add
63 /// will be implemented as 3 separate <2 x s64> adds and one s64 add. Leftover
64 /// types can be avoided by doing MoreElements first.
65 FewerElements,
66
67 /// The (vector) operation should be implemented by widening the input
68 /// vector and ignoring the lanes added by doing so. For example <2 x i8> is
69 /// rarely legal, but you might perform an <8 x i8> and then only look at
70 /// the first two results.
71 MoreElements,
72
73 /// Perform the operation on a different, but equivalently sized type.
74 Bitcast,
75
76 /// The operation itself must be expressed in terms of simpler actions on
77 /// this target. E.g. a SREM replaced by an SDIV and subtraction.
78 Lower,
79
80 /// The operation should be implemented as a call to some kind of runtime
81 /// support library. For example this usually happens on machines that don't
82 /// support floating-point operations natively.
83 Libcall,
84
85 /// The target wants to do something special with this combination of
86 /// operand and type. A callback will be issued when it is needed.
87 Custom,
88
89 /// This operation is completely unsupported on the target. A programming
90 /// error has occurred.
91 Unsupported,
92
93 /// Sentinel value for when no action was found in the specified table.
94 NotFound,
95
96 /// Fall back onto the old rules.
97 /// TODO: Remove this once we've migrated
98 UseLegacyRules,
99};
100} // end namespace LegalizeActions
101raw_ostream &operator<<(raw_ostream &OS, LegalizeActions::LegalizeAction Action);
102
103using LegalizeActions::LegalizeAction;
104
105/// The LegalityQuery object bundles together all the information that's needed
106/// to decide whether a given operation is legal or not.
107/// For efficiency, it doesn't make a copy of Types so care must be taken not
108/// to free it before using the query.
109struct LegalityQuery {
110 unsigned Opcode;
111 ArrayRef<LLT> Types;
112
113 struct MemDesc {
114 LLT MemoryTy;
115 uint64_t AlignInBits;
116 AtomicOrdering Ordering;
117
118 MemDesc() = default;
119 MemDesc(LLT MemoryTy, uint64_t AlignInBits, AtomicOrdering Ordering)
120 : MemoryTy(MemoryTy), AlignInBits(AlignInBits), Ordering(Ordering) {}
121 MemDesc(const MachineMemOperand &MMO)
122 : MemoryTy(MMO.getMemoryType()),
123 AlignInBits(MMO.getAlign().value() * 8),
124 Ordering(MMO.getSuccessOrdering()) {}
125 };
126
127 /// Operations which require memory can use this to place requirements on the
128 /// memory type for each MMO.
129 ArrayRef<MemDesc> MMODescrs;
130
131 constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types,
132 const ArrayRef<MemDesc> MMODescrs)
133 : Opcode(Opcode), Types(Types), MMODescrs(MMODescrs) {}
134 constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types)
135 : LegalityQuery(Opcode, Types, {}) {}
136
137 raw_ostream &print(raw_ostream &OS) const;
138};
139
140/// The result of a query. It either indicates a final answer of Legal or
141/// Unsupported or describes an action that must be taken to make an operation
142/// more legal.
143struct LegalizeActionStep {
144 /// The action to take or the final answer.
145 LegalizeAction Action;
146 /// If describing an action, the type index to change. Otherwise zero.
147 unsigned TypeIdx;
148 /// If describing an action, the new type for TypeIdx. Otherwise LLT{}.
149 LLT NewType;
150
151 LegalizeActionStep(LegalizeAction Action, unsigned TypeIdx,
152 const LLT NewType)
153 : Action(Action), TypeIdx(TypeIdx), NewType(NewType) {}
154
155 LegalizeActionStep(LegacyLegalizeActionStep Step)
156 : TypeIdx(Step.TypeIdx), NewType(Step.NewType) {
157 switch (Step.Action) {
158 case LegacyLegalizeActions::Legal:
159 Action = LegalizeActions::Legal;
160 break;
161 case LegacyLegalizeActions::NarrowScalar:
162 Action = LegalizeActions::NarrowScalar;
163 break;
164 case LegacyLegalizeActions::WidenScalar:
165 Action = LegalizeActions::WidenScalar;
166 break;
167 case LegacyLegalizeActions::FewerElements:
168 Action = LegalizeActions::FewerElements;
169 break;
170 case LegacyLegalizeActions::MoreElements:
171 Action = LegalizeActions::MoreElements;
172 break;
173 case LegacyLegalizeActions::Bitcast:
174 Action = LegalizeActions::Bitcast;
175 break;
176 case LegacyLegalizeActions::Lower:
177 Action = LegalizeActions::Lower;
178 break;
179 case LegacyLegalizeActions::Libcall:
180 Action = LegalizeActions::Libcall;
181 break;
182 case LegacyLegalizeActions::Custom:
183 Action = LegalizeActions::Custom;
184 break;
185 case LegacyLegalizeActions::Unsupported:
186 Action = LegalizeActions::Unsupported;
187 break;
188 case LegacyLegalizeActions::NotFound:
189 Action = LegalizeActions::NotFound;
190 break;
191 }
192 }
193
194 bool operator==(const LegalizeActionStep &RHS) const {
195 return std::tie(args: Action, args: TypeIdx, args: NewType) ==
196 std::tie(args: RHS.Action, args: RHS.TypeIdx, args: RHS.NewType);
197 }
198};
199
200using LegalityPredicate = std::function<bool (const LegalityQuery &)>;
201using LegalizeMutation =
202 std::function<std::pair<unsigned, LLT>(const LegalityQuery &)>;
203
204namespace LegalityPredicates {
205struct TypePairAndMemDesc {
206 LLT Type0;
207 LLT Type1;
208 LLT MemTy;
209 uint64_t Align;
210
211 bool operator==(const TypePairAndMemDesc &Other) const {
212 return Type0 == Other.Type0 && Type1 == Other.Type1 &&
213 Align == Other.Align && MemTy == Other.MemTy;
214 }
215
216 /// \returns true if this memory access is legal with for the access described
217 /// by \p Other (The alignment is sufficient for the size and result type).
218 bool isCompatible(const TypePairAndMemDesc &Other) const {
219 return Type0 == Other.Type0 && Type1 == Other.Type1 &&
220 Align >= Other.Align &&
221 // FIXME: This perhaps should be stricter, but the current legality
222 // rules are written only considering the size.
223 MemTy.getSizeInBits() == Other.MemTy.getSizeInBits();
224 }
225};
226
227/// True iff P is false.
228template <typename Predicate> Predicate predNot(Predicate P) {
229 return [=](const LegalityQuery &Query) { return !P(Query); };
230}
231
232/// True iff P0 and P1 are true.
233template<typename Predicate>
234Predicate all(Predicate P0, Predicate P1) {
235 return [=](const LegalityQuery &Query) {
236 return P0(Query) && P1(Query);
237 };
238}
239/// True iff all given predicates are true.
240template<typename Predicate, typename... Args>
241Predicate all(Predicate P0, Predicate P1, Args... args) {
242 return all(all(P0, P1), args...);
243}
244
245/// True iff P0 or P1 are true.
246template<typename Predicate>
247Predicate any(Predicate P0, Predicate P1) {
248 return [=](const LegalityQuery &Query) {
249 return P0(Query) || P1(Query);
250 };
251}
252/// True iff any given predicates are true.
253template<typename Predicate, typename... Args>
254Predicate any(Predicate P0, Predicate P1, Args... args) {
255 return any(any(P0, P1), args...);
256}
257
258/// True iff the given type index is the specified type.
259LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit);
260/// True iff the given type index is one of the specified types.
261LegalityPredicate typeInSet(unsigned TypeIdx,
262 std::initializer_list<LLT> TypesInit);
263
264/// True iff the given type index is not the specified type.
265inline LegalityPredicate typeIsNot(unsigned TypeIdx, LLT Type) {
266 return [=](const LegalityQuery &Query) {
267 return Query.Types[TypeIdx] != Type;
268 };
269}
270
271/// True iff the given types for the given pair of type indexes is one of the
272/// specified type pairs.
273LegalityPredicate
274typePairInSet(unsigned TypeIdx0, unsigned TypeIdx1,
275 std::initializer_list<std::pair<LLT, LLT>> TypesInit);
276/// True iff the given types for the given pair of type indexes is one of the
277/// specified type pairs.
278LegalityPredicate typePairAndMemDescInSet(
279 unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx,
280 std::initializer_list<TypePairAndMemDesc> TypesAndMemDescInit);
281/// True iff the specified type index is a scalar.
282LegalityPredicate isScalar(unsigned TypeIdx);
283/// True iff the specified type index is a vector.
284LegalityPredicate isVector(unsigned TypeIdx);
285/// True iff the specified type index is a pointer (with any address space).
286LegalityPredicate isPointer(unsigned TypeIdx);
287/// True iff the specified type index is a pointer with the specified address
288/// space.
289LegalityPredicate isPointer(unsigned TypeIdx, unsigned AddrSpace);
290
291/// True if the type index is a vector with element type \p EltTy
292LegalityPredicate elementTypeIs(unsigned TypeIdx, LLT EltTy);
293
294/// True iff the specified type index is a scalar that's narrower than the given
295/// size.
296LegalityPredicate scalarNarrowerThan(unsigned TypeIdx, unsigned Size);
297
298/// True iff the specified type index is a scalar that's wider than the given
299/// size.
300LegalityPredicate scalarWiderThan(unsigned TypeIdx, unsigned Size);
301
302/// True iff the specified type index is a scalar or vector with an element type
303/// that's narrower than the given size.
304LegalityPredicate scalarOrEltNarrowerThan(unsigned TypeIdx, unsigned Size);
305
306/// True iff the specified type index is a scalar or a vector with an element
307/// type that's wider than the given size.
308LegalityPredicate scalarOrEltWiderThan(unsigned TypeIdx, unsigned Size);
309
310/// True iff the specified type index is a scalar whose size is not a multiple
311/// of Size.
312LegalityPredicate sizeNotMultipleOf(unsigned TypeIdx, unsigned Size);
313
314/// True iff the specified type index is a scalar whose size is not a power of
315/// 2.
316LegalityPredicate sizeNotPow2(unsigned TypeIdx);
317
318/// True iff the specified type index is a scalar or vector whose element size
319/// is not a power of 2.
320LegalityPredicate scalarOrEltSizeNotPow2(unsigned TypeIdx);
321
322/// True if the total bitwidth of the specified type index is \p Size bits.
323LegalityPredicate sizeIs(unsigned TypeIdx, unsigned Size);
324
325/// True iff the specified type indices are both the same bit size.
326LegalityPredicate sameSize(unsigned TypeIdx0, unsigned TypeIdx1);
327
328/// True iff the first type index has a larger total bit size than second type
329/// index.
330LegalityPredicate largerThan(unsigned TypeIdx0, unsigned TypeIdx1);
331
332/// True iff the first type index has a smaller total bit size than second type
333/// index.
334LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1);
335
336/// True iff the specified MMO index has a size (rounded to bytes) that is not a
337/// power of 2.
338LegalityPredicate memSizeInBytesNotPow2(unsigned MMOIdx);
339
340/// True iff the specified MMO index has a size that is not an even byte size,
341/// or that even byte size is not a power of 2.
342LegalityPredicate memSizeNotByteSizePow2(unsigned MMOIdx);
343
344/// True iff the specified type index is a vector whose element count is not a
345/// power of 2.
346LegalityPredicate numElementsNotPow2(unsigned TypeIdx);
347/// True iff the specified MMO index has at an atomic ordering of at Ordering or
348/// stronger.
349LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx,
350 AtomicOrdering Ordering);
351} // end namespace LegalityPredicates
352
353namespace LegalizeMutations {
354/// Select this specific type for the given type index.
355LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty);
356
357/// Keep the same type as the given type index.
358LegalizeMutation changeTo(unsigned TypeIdx, unsigned FromTypeIdx);
359
360/// Keep the same scalar or element type as the given type index.
361LegalizeMutation changeElementTo(unsigned TypeIdx, unsigned FromTypeIdx);
362
363/// Keep the same scalar or element type as the given type.
364LegalizeMutation changeElementTo(unsigned TypeIdx, LLT Ty);
365
366/// Keep the same scalar or element type as \p TypeIdx, but take the number of
367/// elements from \p FromTypeIdx.
368LegalizeMutation changeElementCountTo(unsigned TypeIdx, unsigned FromTypeIdx);
369
370/// Keep the same scalar or element type as \p TypeIdx, but take the number of
371/// elements from \p Ty.
372LegalizeMutation changeElementCountTo(unsigned TypeIdx, LLT Ty);
373
374/// Change the scalar size or element size to have the same scalar size as type
375/// index \p FromIndex. Unlike changeElementTo, this discards pointer types and
376/// only changes the size.
377LegalizeMutation changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx);
378
379/// Widen the scalar type or vector element type for the given type index to the
380/// next power of 2.
381LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min = 0);
382
383/// Widen the scalar type or vector element type for the given type index to
384/// next multiple of \p Size.
385LegalizeMutation widenScalarOrEltToNextMultipleOf(unsigned TypeIdx,
386 unsigned Size);
387
388/// Add more elements to the type for the given type index to the next power of
389/// 2.
390LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min = 0);
391/// Break up the vector type for the given type index into the element type.
392LegalizeMutation scalarize(unsigned TypeIdx);
393} // end namespace LegalizeMutations
394
395/// A single rule in a legalizer info ruleset.
396/// The specified action is chosen when the predicate is true. Where appropriate
397/// for the action (e.g. for WidenScalar) the new type is selected using the
398/// given mutator.
399class LegalizeRule {
400 LegalityPredicate Predicate;
401 LegalizeAction Action;
402 LegalizeMutation Mutation;
403
404public:
405 LegalizeRule(LegalityPredicate Predicate, LegalizeAction Action,
406 LegalizeMutation Mutation = nullptr)
407 : Predicate(Predicate), Action(Action), Mutation(Mutation) {}
408
409 /// Test whether the LegalityQuery matches.
410 bool match(const LegalityQuery &Query) const {
411 return Predicate(Query);
412 }
413
414 LegalizeAction getAction() const { return Action; }
415
416 /// Determine the change to make.
417 std::pair<unsigned, LLT> determineMutation(const LegalityQuery &Query) const {
418 if (Mutation)
419 return Mutation(Query);
420 return std::make_pair(x: 0, y: LLT{});
421 }
422};
423
424class LegalizeRuleSet {
425 /// When non-zero, the opcode we are an alias of
426 unsigned AliasOf = 0;
427 /// If true, there is another opcode that aliases this one
428 bool IsAliasedByAnother = false;
429 SmallVector<LegalizeRule, 2> Rules;
430
431#ifndef NDEBUG
432 /// If bit I is set, this rule set contains a rule that may handle (predicate
433 /// or perform an action upon (or both)) the type index I. The uncertainty
434 /// comes from free-form rules executing user-provided lambda functions. We
435 /// conservatively assume such rules do the right thing and cover all type
436 /// indices. The bitset is intentionally 1 bit wider than it absolutely needs
437 /// to be to distinguish such cases from the cases where all type indices are
438 /// individually handled.
439 SmallBitVector TypeIdxsCovered{MCOI::OPERAND_LAST_GENERIC -
440 MCOI::OPERAND_FIRST_GENERIC + 2};
441 SmallBitVector ImmIdxsCovered{MCOI::OPERAND_LAST_GENERIC_IMM -
442 MCOI::OPERAND_FIRST_GENERIC_IMM + 2};
443#endif
444
445 unsigned typeIdx(unsigned TypeIdx) {
446 assert(TypeIdx <=
447 (MCOI::OPERAND_LAST_GENERIC - MCOI::OPERAND_FIRST_GENERIC) &&
448 "Type Index is out of bounds");
449#ifndef NDEBUG
450 TypeIdxsCovered.set(TypeIdx);
451#endif
452 return TypeIdx;
453 }
454
455 void markAllIdxsAsCovered() {
456#ifndef NDEBUG
457 TypeIdxsCovered.set();
458 ImmIdxsCovered.set();
459#endif
460 }
461
462 void add(const LegalizeRule &Rule) {
463 assert(AliasOf == 0 &&
464 "RuleSet is aliased, change the representative opcode instead");
465 Rules.push_back(Elt: Rule);
466 }
467
468 static bool always(const LegalityQuery &) { return true; }
469
470 /// Use the given action when the predicate is true.
471 /// Action should not be an action that requires mutation.
472 LegalizeRuleSet &actionIf(LegalizeAction Action,
473 LegalityPredicate Predicate) {
474 add(Rule: {Predicate, Action});
475 return *this;
476 }
477 /// Use the given action when the predicate is true.
478 /// Action should be an action that requires mutation.
479 LegalizeRuleSet &actionIf(LegalizeAction Action, LegalityPredicate Predicate,
480 LegalizeMutation Mutation) {
481 add(Rule: {Predicate, Action, Mutation});
482 return *this;
483 }
484 /// Use the given action when type index 0 is any type in the given list.
485 /// Action should not be an action that requires mutation.
486 LegalizeRuleSet &actionFor(LegalizeAction Action,
487 std::initializer_list<LLT> Types) {
488 using namespace LegalityPredicates;
489 return actionIf(Action, Predicate: typeInSet(TypeIdx: typeIdx(TypeIdx: 0), TypesInit: Types));
490 }
491 /// Use the given action when type index 0 is any type in the given list.
492 /// Action should be an action that requires mutation.
493 LegalizeRuleSet &actionFor(LegalizeAction Action,
494 std::initializer_list<LLT> Types,
495 LegalizeMutation Mutation) {
496 using namespace LegalityPredicates;
497 return actionIf(Action, Predicate: typeInSet(TypeIdx: typeIdx(TypeIdx: 0), TypesInit: Types), Mutation);
498 }
499 /// Use the given action when type indexes 0 and 1 is any type pair in the
500 /// given list.
501 /// Action should not be an action that requires mutation.
502 LegalizeRuleSet &actionFor(LegalizeAction Action,
503 std::initializer_list<std::pair<LLT, LLT>> Types) {
504 using namespace LegalityPredicates;
505 return actionIf(Action, Predicate: typePairInSet(TypeIdx0: typeIdx(TypeIdx: 0), TypeIdx1: typeIdx(TypeIdx: 1), TypesInit: Types));
506 }
507 /// Use the given action when type indexes 0 and 1 is any type pair in the
508 /// given list.
509 /// Action should be an action that requires mutation.
510 LegalizeRuleSet &actionFor(LegalizeAction Action,
511 std::initializer_list<std::pair<LLT, LLT>> Types,
512 LegalizeMutation Mutation) {
513 using namespace LegalityPredicates;
514 return actionIf(Action, Predicate: typePairInSet(TypeIdx0: typeIdx(TypeIdx: 0), TypeIdx1: typeIdx(TypeIdx: 1), TypesInit: Types),
515 Mutation);
516 }
517 /// Use the given action when type index 0 is any type in the given list and
518 /// imm index 0 is anything. Action should not be an action that requires
519 /// mutation.
520 LegalizeRuleSet &actionForTypeWithAnyImm(LegalizeAction Action,
521 std::initializer_list<LLT> Types) {
522 using namespace LegalityPredicates;
523 immIdx(ImmIdx: 0); // Inform verifier imm idx 0 is handled.
524 return actionIf(Action, Predicate: typeInSet(TypeIdx: typeIdx(TypeIdx: 0), TypesInit: Types));
525 }
526
527 LegalizeRuleSet &actionForTypeWithAnyImm(
528 LegalizeAction Action, std::initializer_list<std::pair<LLT, LLT>> Types) {
529 using namespace LegalityPredicates;
530 immIdx(ImmIdx: 0); // Inform verifier imm idx 0 is handled.
531 return actionIf(Action, Predicate: typePairInSet(TypeIdx0: typeIdx(TypeIdx: 0), TypeIdx1: typeIdx(TypeIdx: 1), TypesInit: Types));
532 }
533
534 /// Use the given action when type indexes 0 and 1 are both in the given list.
535 /// That is, the type pair is in the cartesian product of the list.
536 /// Action should not be an action that requires mutation.
537 LegalizeRuleSet &actionForCartesianProduct(LegalizeAction Action,
538 std::initializer_list<LLT> Types) {
539 using namespace LegalityPredicates;
540 return actionIf(Action, Predicate: all(P0: typeInSet(TypeIdx: typeIdx(TypeIdx: 0), TypesInit: Types),
541 P1: typeInSet(TypeIdx: typeIdx(TypeIdx: 1), TypesInit: Types)));
542 }
543 /// Use the given action when type indexes 0 and 1 are both in their
544 /// respective lists.
545 /// That is, the type pair is in the cartesian product of the lists
546 /// Action should not be an action that requires mutation.
547 LegalizeRuleSet &
548 actionForCartesianProduct(LegalizeAction Action,
549 std::initializer_list<LLT> Types0,
550 std::initializer_list<LLT> Types1) {
551 using namespace LegalityPredicates;
552 return actionIf(Action, Predicate: all(P0: typeInSet(TypeIdx: typeIdx(TypeIdx: 0), TypesInit: Types0),
553 P1: typeInSet(TypeIdx: typeIdx(TypeIdx: 1), TypesInit: Types1)));
554 }
555 /// Use the given action when type indexes 0, 1, and 2 are all in their
556 /// respective lists.
557 /// That is, the type triple is in the cartesian product of the lists
558 /// Action should not be an action that requires mutation.
559 LegalizeRuleSet &actionForCartesianProduct(
560 LegalizeAction Action, std::initializer_list<LLT> Types0,
561 std::initializer_list<LLT> Types1, std::initializer_list<LLT> Types2) {
562 using namespace LegalityPredicates;
563 return actionIf(Action, Predicate: all(P0: typeInSet(TypeIdx: typeIdx(TypeIdx: 0), TypesInit: Types0),
564 P1: all(P0: typeInSet(TypeIdx: typeIdx(TypeIdx: 1), TypesInit: Types1),
565 P1: typeInSet(TypeIdx: typeIdx(TypeIdx: 2), TypesInit: Types2))));
566 }
567
568public:
569 LegalizeRuleSet() = default;
570
571 bool isAliasedByAnother() { return IsAliasedByAnother; }
572 void setIsAliasedByAnother() { IsAliasedByAnother = true; }
573 void aliasTo(unsigned Opcode) {
574 assert((AliasOf == 0 || AliasOf == Opcode) &&
575 "Opcode is already aliased to another opcode");
576 assert(Rules.empty() && "Aliasing will discard rules");
577 AliasOf = Opcode;
578 }
579 unsigned getAlias() const { return AliasOf; }
580
581 unsigned immIdx(unsigned ImmIdx) {
582 assert(ImmIdx <= (MCOI::OPERAND_LAST_GENERIC_IMM -
583 MCOI::OPERAND_FIRST_GENERIC_IMM) &&
584 "Imm Index is out of bounds");
585#ifndef NDEBUG
586 ImmIdxsCovered.set(ImmIdx);
587#endif
588 return ImmIdx;
589 }
590
591 /// The instruction is legal if predicate is true.
592 LegalizeRuleSet &legalIf(LegalityPredicate Predicate) {
593 // We have no choice but conservatively assume that the free-form
594 // user-provided Predicate properly handles all type indices:
595 markAllIdxsAsCovered();
596 return actionIf(Action: LegalizeAction::Legal, Predicate);
597 }
598 /// The instruction is legal when type index 0 is any type in the given list.
599 LegalizeRuleSet &legalFor(std::initializer_list<LLT> Types) {
600 return actionFor(Action: LegalizeAction::Legal, Types);
601 }
602 /// The instruction is legal when type indexes 0 and 1 is any type pair in the
603 /// given list.
604 LegalizeRuleSet &legalFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
605 return actionFor(Action: LegalizeAction::Legal, Types);
606 }
607 /// The instruction is legal when type index 0 is any type in the given list
608 /// and imm index 0 is anything.
609 LegalizeRuleSet &legalForTypeWithAnyImm(std::initializer_list<LLT> Types) {
610 markAllIdxsAsCovered();
611 return actionForTypeWithAnyImm(Action: LegalizeAction::Legal, Types);
612 }
613
614 LegalizeRuleSet &legalForTypeWithAnyImm(
615 std::initializer_list<std::pair<LLT, LLT>> Types) {
616 markAllIdxsAsCovered();
617 return actionForTypeWithAnyImm(Action: LegalizeAction::Legal, Types);
618 }
619
620 /// The instruction is legal when type indexes 0 and 1 along with the memory
621 /// size and minimum alignment is any type and size tuple in the given list.
622 LegalizeRuleSet &legalForTypesWithMemDesc(
623 std::initializer_list<LegalityPredicates::TypePairAndMemDesc>
624 TypesAndMemDesc) {
625 return actionIf(Action: LegalizeAction::Legal,
626 Predicate: LegalityPredicates::typePairAndMemDescInSet(
627 TypeIdx0: typeIdx(TypeIdx: 0), TypeIdx1: typeIdx(TypeIdx: 1), /*MMOIdx*/ MMOIdx: 0, TypesAndMemDescInit: TypesAndMemDesc));
628 }
629 /// The instruction is legal when type indexes 0 and 1 are both in the given
630 /// list. That is, the type pair is in the cartesian product of the list.
631 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types) {
632 return actionForCartesianProduct(Action: LegalizeAction::Legal, Types);
633 }
634 /// The instruction is legal when type indexes 0 and 1 are both their
635 /// respective lists.
636 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0,
637 std::initializer_list<LLT> Types1) {
638 return actionForCartesianProduct(Action: LegalizeAction::Legal, Types0, Types1);
639 }
640 /// The instruction is legal when type indexes 0, 1, and 2 are both their
641 /// respective lists.
642 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0,
643 std::initializer_list<LLT> Types1,
644 std::initializer_list<LLT> Types2) {
645 return actionForCartesianProduct(Action: LegalizeAction::Legal, Types0, Types1,
646 Types2);
647 }
648
649 LegalizeRuleSet &alwaysLegal() {
650 using namespace LegalizeMutations;
651 markAllIdxsAsCovered();
652 return actionIf(Action: LegalizeAction::Legal, Predicate: always);
653 }
654
655 /// The specified type index is coerced if predicate is true.
656 LegalizeRuleSet &bitcastIf(LegalityPredicate Predicate,
657 LegalizeMutation Mutation) {
658 // We have no choice but conservatively assume that lowering with a
659 // free-form user provided Predicate properly handles all type indices:
660 markAllIdxsAsCovered();
661 return actionIf(Action: LegalizeAction::Bitcast, Predicate, Mutation);
662 }
663
664 /// The instruction is lowered.
665 LegalizeRuleSet &lower() {
666 using namespace LegalizeMutations;
667 // We have no choice but conservatively assume that predicate-less lowering
668 // properly handles all type indices by design:
669 markAllIdxsAsCovered();
670 return actionIf(Action: LegalizeAction::Lower, Predicate: always);
671 }
672 /// The instruction is lowered if predicate is true. Keep type index 0 as the
673 /// same type.
674 LegalizeRuleSet &lowerIf(LegalityPredicate Predicate) {
675 using namespace LegalizeMutations;
676 // We have no choice but conservatively assume that lowering with a
677 // free-form user provided Predicate properly handles all type indices:
678 markAllIdxsAsCovered();
679 return actionIf(Action: LegalizeAction::Lower, Predicate);
680 }
681 /// The instruction is lowered if predicate is true.
682 LegalizeRuleSet &lowerIf(LegalityPredicate Predicate,
683 LegalizeMutation Mutation) {
684 // We have no choice but conservatively assume that lowering with a
685 // free-form user provided Predicate properly handles all type indices:
686 markAllIdxsAsCovered();
687 return actionIf(Action: LegalizeAction::Lower, Predicate, Mutation);
688 }
689 /// The instruction is lowered when type index 0 is any type in the given
690 /// list. Keep type index 0 as the same type.
691 LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types) {
692 return actionFor(Action: LegalizeAction::Lower, Types);
693 }
694 /// The instruction is lowered when type index 0 is any type in the given
695 /// list.
696 LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types,
697 LegalizeMutation Mutation) {
698 return actionFor(Action: LegalizeAction::Lower, Types, Mutation);
699 }
700 /// The instruction is lowered when type indexes 0 and 1 is any type pair in
701 /// the given list. Keep type index 0 as the same type.
702 LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
703 return actionFor(Action: LegalizeAction::Lower, Types);
704 }
705 /// The instruction is lowered when type indexes 0 and 1 is any type pair in
706 /// the given list.
707 LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types,
708 LegalizeMutation Mutation) {
709 return actionFor(Action: LegalizeAction::Lower, Types, Mutation);
710 }
711 /// The instruction is lowered when type indexes 0 and 1 are both in their
712 /// respective lists.
713 LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0,
714 std::initializer_list<LLT> Types1) {
715 using namespace LegalityPredicates;
716 return actionForCartesianProduct(Action: LegalizeAction::Lower, Types0, Types1);
717 }
718 /// The instruction is lowered when type indexes 0, 1, and 2 are all in
719 /// their respective lists.
720 LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0,
721 std::initializer_list<LLT> Types1,
722 std::initializer_list<LLT> Types2) {
723 using namespace LegalityPredicates;
724 return actionForCartesianProduct(Action: LegalizeAction::Lower, Types0, Types1,
725 Types2);
726 }
727
728 /// The instruction is emitted as a library call.
729 LegalizeRuleSet &libcall() {
730 using namespace LegalizeMutations;
731 // We have no choice but conservatively assume that predicate-less lowering
732 // properly handles all type indices by design:
733 markAllIdxsAsCovered();
734 return actionIf(Action: LegalizeAction::Libcall, Predicate: always);
735 }
736
737 /// Like legalIf, but for the Libcall action.
738 LegalizeRuleSet &libcallIf(LegalityPredicate Predicate) {
739 // We have no choice but conservatively assume that a libcall with a
740 // free-form user provided Predicate properly handles all type indices:
741 markAllIdxsAsCovered();
742 return actionIf(Action: LegalizeAction::Libcall, Predicate);
743 }
744 LegalizeRuleSet &libcallFor(std::initializer_list<LLT> Types) {
745 return actionFor(Action: LegalizeAction::Libcall, Types);
746 }
747 LegalizeRuleSet &
748 libcallFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
749 return actionFor(Action: LegalizeAction::Libcall, Types);
750 }
751 LegalizeRuleSet &
752 libcallForCartesianProduct(std::initializer_list<LLT> Types) {
753 return actionForCartesianProduct(Action: LegalizeAction::Libcall, Types);
754 }
755 LegalizeRuleSet &
756 libcallForCartesianProduct(std::initializer_list<LLT> Types0,
757 std::initializer_list<LLT> Types1) {
758 return actionForCartesianProduct(Action: LegalizeAction::Libcall, Types0, Types1);
759 }
760
761 /// Widen the scalar to the one selected by the mutation if the predicate is
762 /// true.
763 LegalizeRuleSet &widenScalarIf(LegalityPredicate Predicate,
764 LegalizeMutation Mutation) {
765 // We have no choice but conservatively assume that an action with a
766 // free-form user provided Predicate properly handles all type indices:
767 markAllIdxsAsCovered();
768 return actionIf(Action: LegalizeAction::WidenScalar, Predicate, Mutation);
769 }
770 /// Narrow the scalar to the one selected by the mutation if the predicate is
771 /// true.
772 LegalizeRuleSet &narrowScalarIf(LegalityPredicate Predicate,
773 LegalizeMutation Mutation) {
774 // We have no choice but conservatively assume that an action with a
775 // free-form user provided Predicate properly handles all type indices:
776 markAllIdxsAsCovered();
777 return actionIf(Action: LegalizeAction::NarrowScalar, Predicate, Mutation);
778 }
779 /// Narrow the scalar, specified in mutation, when type indexes 0 and 1 is any
780 /// type pair in the given list.
781 LegalizeRuleSet &
782 narrowScalarFor(std::initializer_list<std::pair<LLT, LLT>> Types,
783 LegalizeMutation Mutation) {
784 return actionFor(Action: LegalizeAction::NarrowScalar, Types, Mutation);
785 }
786
787 /// Add more elements to reach the type selected by the mutation if the
788 /// predicate is true.
789 LegalizeRuleSet &moreElementsIf(LegalityPredicate Predicate,
790 LegalizeMutation Mutation) {
791 // We have no choice but conservatively assume that an action with a
792 // free-form user provided Predicate properly handles all type indices:
793 markAllIdxsAsCovered();
794 return actionIf(Action: LegalizeAction::MoreElements, Predicate, Mutation);
795 }
796 /// Remove elements to reach the type selected by the mutation if the
797 /// predicate is true.
798 LegalizeRuleSet &fewerElementsIf(LegalityPredicate Predicate,
799 LegalizeMutation Mutation) {
800 // We have no choice but conservatively assume that an action with a
801 // free-form user provided Predicate properly handles all type indices:
802 markAllIdxsAsCovered();
803 return actionIf(Action: LegalizeAction::FewerElements, Predicate, Mutation);
804 }
805
806 /// The instruction is unsupported.
807 LegalizeRuleSet &unsupported() {
808 markAllIdxsAsCovered();
809 return actionIf(Action: LegalizeAction::Unsupported, Predicate: always);
810 }
811 LegalizeRuleSet &unsupportedIf(LegalityPredicate Predicate) {
812 return actionIf(Action: LegalizeAction::Unsupported, Predicate);
813 }
814
815 LegalizeRuleSet &unsupportedFor(std::initializer_list<LLT> Types) {
816 return actionFor(Action: LegalizeAction::Unsupported, Types);
817 }
818
819 LegalizeRuleSet &unsupportedIfMemSizeNotPow2() {
820 return actionIf(Action: LegalizeAction::Unsupported,
821 Predicate: LegalityPredicates::memSizeInBytesNotPow2(MMOIdx: 0));
822 }
823
824 /// Lower a memory operation if the memory size, rounded to bytes, is not a
825 /// power of 2. For example, this will not trigger for s1 or s7, but will for
826 /// s24.
827 LegalizeRuleSet &lowerIfMemSizeNotPow2() {
828 return actionIf(Action: LegalizeAction::Lower,
829 Predicate: LegalityPredicates::memSizeInBytesNotPow2(MMOIdx: 0));
830 }
831
832 /// Lower a memory operation if the memory access size is not a round power of
833 /// 2 byte size. This is stricter than lowerIfMemSizeNotPow2, and more likely
834 /// what you want (e.g. this will lower s1, s7 and s24).
835 LegalizeRuleSet &lowerIfMemSizeNotByteSizePow2() {
836 return actionIf(Action: LegalizeAction::Lower,
837 Predicate: LegalityPredicates::memSizeNotByteSizePow2(MMOIdx: 0));
838 }
839
840 LegalizeRuleSet &customIf(LegalityPredicate Predicate) {
841 // We have no choice but conservatively assume that a custom action with a
842 // free-form user provided Predicate properly handles all type indices:
843 markAllIdxsAsCovered();
844 return actionIf(Action: LegalizeAction::Custom, Predicate);
845 }
846 LegalizeRuleSet &customFor(std::initializer_list<LLT> Types) {
847 return actionFor(Action: LegalizeAction::Custom, Types);
848 }
849
850 /// The instruction is custom when type indexes 0 and 1 is any type pair in the
851 /// given list.
852 LegalizeRuleSet &customFor(std::initializer_list<std::pair<LLT, LLT>> Types) {
853 return actionFor(Action: LegalizeAction::Custom, Types);
854 }
855
856 LegalizeRuleSet &customForCartesianProduct(std::initializer_list<LLT> Types) {
857 return actionForCartesianProduct(Action: LegalizeAction::Custom, Types);
858 }
859 /// The instruction is custom when type indexes 0 and 1 are both in their
860 /// respective lists.
861 LegalizeRuleSet &
862 customForCartesianProduct(std::initializer_list<LLT> Types0,
863 std::initializer_list<LLT> Types1) {
864 return actionForCartesianProduct(Action: LegalizeAction::Custom, Types0, Types1);
865 }
866 /// The instruction is custom when type indexes 0, 1, and 2 are all in
867 /// their respective lists.
868 LegalizeRuleSet &
869 customForCartesianProduct(std::initializer_list<LLT> Types0,
870 std::initializer_list<LLT> Types1,
871 std::initializer_list<LLT> Types2) {
872 return actionForCartesianProduct(Action: LegalizeAction::Custom, Types0, Types1,
873 Types2);
874 }
875
876 /// Unconditionally custom lower.
877 LegalizeRuleSet &custom() {
878 return customIf(Predicate: always);
879 }
880
881 /// Widen the scalar to the next power of two that is at least MinSize.
882 /// No effect if the type is a power of two, except if the type is smaller
883 /// than MinSize, or if the type is a vector type.
884 LegalizeRuleSet &widenScalarToNextPow2(unsigned TypeIdx,
885 unsigned MinSize = 0) {
886 using namespace LegalityPredicates;
887 return actionIf(
888 Action: LegalizeAction::WidenScalar, Predicate: sizeNotPow2(TypeIdx: typeIdx(TypeIdx)),
889 Mutation: LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, Min: MinSize));
890 }
891
892 /// Widen the scalar to the next multiple of Size. No effect if the
893 /// type is not a scalar or is a multiple of Size.
894 LegalizeRuleSet &widenScalarToNextMultipleOf(unsigned TypeIdx,
895 unsigned Size) {
896 using namespace LegalityPredicates;
897 return actionIf(
898 Action: LegalizeAction::WidenScalar, Predicate: sizeNotMultipleOf(TypeIdx: typeIdx(TypeIdx), Size),
899 Mutation: LegalizeMutations::widenScalarOrEltToNextMultipleOf(TypeIdx, Size));
900 }
901
902 /// Widen the scalar or vector element type to the next power of two that is
903 /// at least MinSize. No effect if the scalar size is a power of two.
904 LegalizeRuleSet &widenScalarOrEltToNextPow2(unsigned TypeIdx,
905 unsigned MinSize = 0) {
906 using namespace LegalityPredicates;
907 return actionIf(
908 Action: LegalizeAction::WidenScalar, Predicate: scalarOrEltSizeNotPow2(TypeIdx: typeIdx(TypeIdx)),
909 Mutation: LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, Min: MinSize));
910 }
911
912 /// Widen the scalar or vector element type to the next power of two that is
913 /// at least MinSize. No effect if the scalar size is a power of two.
914 LegalizeRuleSet &widenScalarOrEltToNextPow2OrMinSize(unsigned TypeIdx,
915 unsigned MinSize = 0) {
916 using namespace LegalityPredicates;
917 return actionIf(
918 Action: LegalizeAction::WidenScalar,
919 Predicate: any(P0: scalarOrEltNarrowerThan(TypeIdx, Size: MinSize),
920 P1: scalarOrEltSizeNotPow2(TypeIdx: typeIdx(TypeIdx))),
921 Mutation: LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, Min: MinSize));
922 }
923
924 LegalizeRuleSet &narrowScalar(unsigned TypeIdx, LegalizeMutation Mutation) {
925 using namespace LegalityPredicates;
926 return actionIf(Action: LegalizeAction::NarrowScalar, Predicate: isScalar(TypeIdx: typeIdx(TypeIdx)),
927 Mutation);
928 }
929
930 LegalizeRuleSet &scalarize(unsigned TypeIdx) {
931 using namespace LegalityPredicates;
932 return actionIf(Action: LegalizeAction::FewerElements, Predicate: isVector(TypeIdx: typeIdx(TypeIdx)),
933 Mutation: LegalizeMutations::scalarize(TypeIdx));
934 }
935
936 LegalizeRuleSet &scalarizeIf(LegalityPredicate Predicate, unsigned TypeIdx) {
937 using namespace LegalityPredicates;
938 return actionIf(Action: LegalizeAction::FewerElements,
939 Predicate: all(P0: Predicate, P1: isVector(TypeIdx: typeIdx(TypeIdx))),
940 Mutation: LegalizeMutations::scalarize(TypeIdx));
941 }
942
943 /// Ensure the scalar or element is at least as wide as Ty.
944 LegalizeRuleSet &minScalarOrElt(unsigned TypeIdx, const LLT Ty) {
945 using namespace LegalityPredicates;
946 using namespace LegalizeMutations;
947 return actionIf(Action: LegalizeAction::WidenScalar,
948 Predicate: scalarOrEltNarrowerThan(TypeIdx, Size: Ty.getScalarSizeInBits()),
949 Mutation: changeElementTo(TypeIdx: typeIdx(TypeIdx), Ty));
950 }
951
952 /// Ensure the scalar or element is at least as wide as Ty.
953 LegalizeRuleSet &minScalarOrEltIf(LegalityPredicate Predicate,
954 unsigned TypeIdx, const LLT Ty) {
955 using namespace LegalityPredicates;
956 using namespace LegalizeMutations;
957 return actionIf(Action: LegalizeAction::WidenScalar,
958 Predicate: all(P0: Predicate, P1: scalarOrEltNarrowerThan(
959 TypeIdx, Size: Ty.getScalarSizeInBits())),
960 Mutation: changeElementTo(TypeIdx: typeIdx(TypeIdx), Ty));
961 }
962
963 /// Ensure the vector size is at least as wide as VectorSize by promoting the
964 /// element.
965 LegalizeRuleSet &widenVectorEltsToVectorMinSize(unsigned TypeIdx,
966 unsigned VectorSize) {
967 using namespace LegalityPredicates;
968 using namespace LegalizeMutations;
969 return actionIf(
970 Action: LegalizeAction::WidenScalar,
971 Predicate: [=](const LegalityQuery &Query) {
972 const LLT VecTy = Query.Types[TypeIdx];
973 return VecTy.isVector() && !VecTy.isScalable() &&
974 VecTy.getSizeInBits() < VectorSize;
975 },
976 Mutation: [=](const LegalityQuery &Query) {
977 const LLT VecTy = Query.Types[TypeIdx];
978 unsigned NumElts = VecTy.getNumElements();
979 unsigned MinSize = VectorSize / NumElts;
980 LLT NewTy = LLT::fixed_vector(NumElements: NumElts, ScalarTy: LLT::scalar(SizeInBits: MinSize));
981 return std::make_pair(x: TypeIdx, y&: NewTy);
982 });
983 }
984
985 /// Ensure the scalar is at least as wide as Ty.
986 LegalizeRuleSet &minScalar(unsigned TypeIdx, const LLT Ty) {
987 using namespace LegalityPredicates;
988 using namespace LegalizeMutations;
989 return actionIf(Action: LegalizeAction::WidenScalar,
990 Predicate: scalarNarrowerThan(TypeIdx, Size: Ty.getSizeInBits()),
991 Mutation: changeTo(TypeIdx: typeIdx(TypeIdx), Ty));
992 }
993
994 /// Ensure the scalar is at least as wide as Ty if condition is met.
995 LegalizeRuleSet &minScalarIf(LegalityPredicate Predicate, unsigned TypeIdx,
996 const LLT Ty) {
997 using namespace LegalityPredicates;
998 using namespace LegalizeMutations;
999 return actionIf(
1000 Action: LegalizeAction::WidenScalar,
1001 Predicate: [=](const LegalityQuery &Query) {
1002 const LLT QueryTy = Query.Types[TypeIdx];
1003 return QueryTy.isScalar() &&
1004 QueryTy.getSizeInBits() < Ty.getSizeInBits() &&
1005 Predicate(Query);
1006 },
1007 Mutation: changeTo(TypeIdx: typeIdx(TypeIdx), Ty));
1008 }
1009
1010 /// Ensure the scalar is at most as wide as Ty.
1011 LegalizeRuleSet &maxScalarOrElt(unsigned TypeIdx, const LLT Ty) {
1012 using namespace LegalityPredicates;
1013 using namespace LegalizeMutations;
1014 return actionIf(Action: LegalizeAction::NarrowScalar,
1015 Predicate: scalarOrEltWiderThan(TypeIdx, Size: Ty.getScalarSizeInBits()),
1016 Mutation: changeElementTo(TypeIdx: typeIdx(TypeIdx), Ty));
1017 }
1018
1019 /// Ensure the scalar is at most as wide as Ty.
1020 LegalizeRuleSet &maxScalar(unsigned TypeIdx, const LLT Ty) {
1021 using namespace LegalityPredicates;
1022 using namespace LegalizeMutations;
1023 return actionIf(Action: LegalizeAction::NarrowScalar,
1024 Predicate: scalarWiderThan(TypeIdx, Size: Ty.getSizeInBits()),
1025 Mutation: changeTo(TypeIdx: typeIdx(TypeIdx), Ty));
1026 }
1027
1028 /// Conditionally limit the maximum size of the scalar.
1029 /// For example, when the maximum size of one type depends on the size of
1030 /// another such as extracting N bits from an M bit container.
1031 LegalizeRuleSet &maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx,
1032 const LLT Ty) {
1033 using namespace LegalityPredicates;
1034 using namespace LegalizeMutations;
1035 return actionIf(
1036 Action: LegalizeAction::NarrowScalar,
1037 Predicate: [=](const LegalityQuery &Query) {
1038 const LLT QueryTy = Query.Types[TypeIdx];
1039 return QueryTy.isScalar() &&
1040 QueryTy.getSizeInBits() > Ty.getSizeInBits() &&
1041 Predicate(Query);
1042 },
1043 Mutation: changeElementTo(TypeIdx: typeIdx(TypeIdx), Ty));
1044 }
1045
1046 /// Limit the range of scalar sizes to MinTy and MaxTy.
1047 LegalizeRuleSet &clampScalar(unsigned TypeIdx, const LLT MinTy,
1048 const LLT MaxTy) {
1049 assert(MinTy.isScalar() && MaxTy.isScalar() && "Expected scalar types");
1050 return minScalar(TypeIdx, Ty: MinTy).maxScalar(TypeIdx, Ty: MaxTy);
1051 }
1052
1053 /// Limit the range of scalar sizes to MinTy and MaxTy.
1054 LegalizeRuleSet &clampScalarOrElt(unsigned TypeIdx, const LLT MinTy,
1055 const LLT MaxTy) {
1056 return minScalarOrElt(TypeIdx, Ty: MinTy).maxScalarOrElt(TypeIdx, Ty: MaxTy);
1057 }
1058
1059 /// Widen the scalar to match the size of another.
1060 LegalizeRuleSet &minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx) {
1061 typeIdx(TypeIdx);
1062 return widenScalarIf(
1063 Predicate: [=](const LegalityQuery &Query) {
1064 return Query.Types[LargeTypeIdx].getScalarSizeInBits() >
1065 Query.Types[TypeIdx].getSizeInBits();
1066 },
1067 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx, FromTypeIdx: LargeTypeIdx));
1068 }
1069
1070 /// Narrow the scalar to match the size of another.
1071 LegalizeRuleSet &maxScalarSameAs(unsigned TypeIdx, unsigned NarrowTypeIdx) {
1072 typeIdx(TypeIdx);
1073 return narrowScalarIf(
1074 Predicate: [=](const LegalityQuery &Query) {
1075 return Query.Types[NarrowTypeIdx].getScalarSizeInBits() <
1076 Query.Types[TypeIdx].getSizeInBits();
1077 },
1078 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx, FromTypeIdx: NarrowTypeIdx));
1079 }
1080
1081 /// Change the type \p TypeIdx to have the same scalar size as type \p
1082 /// SameSizeIdx.
1083 LegalizeRuleSet &scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx) {
1084 return minScalarSameAs(TypeIdx, LargeTypeIdx: SameSizeIdx)
1085 .maxScalarSameAs(TypeIdx, NarrowTypeIdx: SameSizeIdx);
1086 }
1087
1088 /// Conditionally widen the scalar or elt to match the size of another.
1089 LegalizeRuleSet &minScalarEltSameAsIf(LegalityPredicate Predicate,
1090 unsigned TypeIdx, unsigned LargeTypeIdx) {
1091 typeIdx(TypeIdx);
1092 return widenScalarIf(
1093 Predicate: [=](const LegalityQuery &Query) {
1094 return Query.Types[LargeTypeIdx].getScalarSizeInBits() >
1095 Query.Types[TypeIdx].getScalarSizeInBits() &&
1096 Predicate(Query);
1097 },
1098 Mutation: [=](const LegalityQuery &Query) {
1099 LLT T = Query.Types[LargeTypeIdx];
1100 if (T.isPointerVector())
1101 T = T.changeElementType(NewEltTy: LLT::scalar(SizeInBits: T.getScalarSizeInBits()));
1102 return std::make_pair(x: TypeIdx, y&: T);
1103 });
1104 }
1105
1106 /// Conditionally narrow the scalar or elt to match the size of another.
1107 LegalizeRuleSet &maxScalarEltSameAsIf(LegalityPredicate Predicate,
1108 unsigned TypeIdx,
1109 unsigned SmallTypeIdx) {
1110 typeIdx(TypeIdx);
1111 return narrowScalarIf(
1112 Predicate: [=](const LegalityQuery &Query) {
1113 return Query.Types[SmallTypeIdx].getScalarSizeInBits() <
1114 Query.Types[TypeIdx].getScalarSizeInBits() &&
1115 Predicate(Query);
1116 },
1117 Mutation: [=](const LegalityQuery &Query) {
1118 LLT T = Query.Types[SmallTypeIdx];
1119 return std::make_pair(x: TypeIdx, y&: T);
1120 });
1121 }
1122
1123 /// Add more elements to the vector to reach the next power of two.
1124 /// No effect if the type is not a vector or the element count is a power of
1125 /// two.
1126 LegalizeRuleSet &moreElementsToNextPow2(unsigned TypeIdx) {
1127 using namespace LegalityPredicates;
1128 return actionIf(Action: LegalizeAction::MoreElements,
1129 Predicate: numElementsNotPow2(TypeIdx: typeIdx(TypeIdx)),
1130 Mutation: LegalizeMutations::moreElementsToNextPow2(TypeIdx));
1131 }
1132
1133 /// Limit the number of elements in EltTy vectors to at least MinElements.
1134 LegalizeRuleSet &clampMinNumElements(unsigned TypeIdx, const LLT EltTy,
1135 unsigned MinElements) {
1136 // Mark the type index as covered:
1137 typeIdx(TypeIdx);
1138 return actionIf(
1139 Action: LegalizeAction::MoreElements,
1140 Predicate: [=](const LegalityQuery &Query) {
1141 LLT VecTy = Query.Types[TypeIdx];
1142 return VecTy.isVector() && VecTy.getElementType() == EltTy &&
1143 VecTy.getNumElements() < MinElements;
1144 },
1145 Mutation: [=](const LegalityQuery &Query) {
1146 LLT VecTy = Query.Types[TypeIdx];
1147 return std::make_pair(
1148 x: TypeIdx, y: LLT::fixed_vector(NumElements: MinElements, ScalarTy: VecTy.getElementType()));
1149 });
1150 }
1151
1152 /// Set number of elements to nearest larger multiple of NumElts.
1153 LegalizeRuleSet &alignNumElementsTo(unsigned TypeIdx, const LLT EltTy,
1154 unsigned NumElts) {
1155 typeIdx(TypeIdx);
1156 return actionIf(
1157 Action: LegalizeAction::MoreElements,
1158 Predicate: [=](const LegalityQuery &Query) {
1159 LLT VecTy = Query.Types[TypeIdx];
1160 return VecTy.isVector() && VecTy.getElementType() == EltTy &&
1161 (VecTy.getNumElements() % NumElts != 0);
1162 },
1163 Mutation: [=](const LegalityQuery &Query) {
1164 LLT VecTy = Query.Types[TypeIdx];
1165 unsigned NewSize = alignTo(Value: VecTy.getNumElements(), Align: NumElts);
1166 return std::make_pair(
1167 x: TypeIdx, y: LLT::fixed_vector(NumElements: NewSize, ScalarTy: VecTy.getElementType()));
1168 });
1169 }
1170
1171 /// Limit the number of elements in EltTy vectors to at most MaxElements.
1172 LegalizeRuleSet &clampMaxNumElements(unsigned TypeIdx, const LLT EltTy,
1173 unsigned MaxElements) {
1174 // Mark the type index as covered:
1175 typeIdx(TypeIdx);
1176 return actionIf(
1177 Action: LegalizeAction::FewerElements,
1178 Predicate: [=](const LegalityQuery &Query) {
1179 LLT VecTy = Query.Types[TypeIdx];
1180 return VecTy.isVector() && VecTy.getElementType() == EltTy &&
1181 VecTy.getNumElements() > MaxElements;
1182 },
1183 Mutation: [=](const LegalityQuery &Query) {
1184 LLT VecTy = Query.Types[TypeIdx];
1185 LLT NewTy = LLT::scalarOrVector(EC: ElementCount::getFixed(MinVal: MaxElements),
1186 ScalarTy: VecTy.getElementType());
1187 return std::make_pair(x: TypeIdx, y&: NewTy);
1188 });
1189 }
1190 /// Limit the number of elements for the given vectors to at least MinTy's
1191 /// number of elements and at most MaxTy's number of elements.
1192 ///
1193 /// No effect if the type is not a vector or does not have the same element
1194 /// type as the constraints.
1195 /// The element type of MinTy and MaxTy must match.
1196 LegalizeRuleSet &clampNumElements(unsigned TypeIdx, const LLT MinTy,
1197 const LLT MaxTy) {
1198 assert(MinTy.getElementType() == MaxTy.getElementType() &&
1199 "Expected element types to agree");
1200
1201 const LLT EltTy = MinTy.getElementType();
1202 return clampMinNumElements(TypeIdx, EltTy, MinElements: MinTy.getNumElements())
1203 .clampMaxNumElements(TypeIdx, EltTy, MaxElements: MaxTy.getNumElements());
1204 }
1205
1206 /// Express \p EltTy vectors strictly using vectors with \p NumElts elements
1207 /// (or scalars when \p NumElts equals 1).
1208 /// First pad with undef elements to nearest larger multiple of \p NumElts.
1209 /// Then perform split with all sub-instructions having the same type.
1210 /// Using clampMaxNumElements (non-strict) can result in leftover instruction
1211 /// with different type (fewer elements then \p NumElts or scalar).
1212 /// No effect if the type is not a vector.
1213 LegalizeRuleSet &clampMaxNumElementsStrict(unsigned TypeIdx, const LLT EltTy,
1214 unsigned NumElts) {
1215 return alignNumElementsTo(TypeIdx, EltTy, NumElts)
1216 .clampMaxNumElements(TypeIdx, EltTy, MaxElements: NumElts);
1217 }
1218
1219 /// Fallback on the previous implementation. This should only be used while
1220 /// porting a rule.
1221 LegalizeRuleSet &fallback() {
1222 add(Rule: {always, LegalizeAction::UseLegacyRules});
1223 return *this;
1224 }
1225
1226 /// Check if there is no type index which is obviously not handled by the
1227 /// LegalizeRuleSet in any way at all.
1228 /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set.
1229 bool verifyTypeIdxsCoverage(unsigned NumTypeIdxs) const;
1230 /// Check if there is no imm index which is obviously not handled by the
1231 /// LegalizeRuleSet in any way at all.
1232 /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set.
1233 bool verifyImmIdxsCoverage(unsigned NumImmIdxs) const;
1234
1235 /// Apply the ruleset to the given LegalityQuery.
1236 LegalizeActionStep apply(const LegalityQuery &Query) const;
1237};
1238
1239class LegalizerInfo {
1240public:
1241 virtual ~LegalizerInfo() = default;
1242
1243 const LegacyLegalizerInfo &getLegacyLegalizerInfo() const {
1244 return LegacyInfo;
1245 }
1246 LegacyLegalizerInfo &getLegacyLegalizerInfo() { return LegacyInfo; }
1247
1248 unsigned getOpcodeIdxForOpcode(unsigned Opcode) const;
1249 unsigned getActionDefinitionsIdx(unsigned Opcode) const;
1250
1251 /// Perform simple self-diagnostic and assert if there is anything obviously
1252 /// wrong with the actions set up.
1253 void verify(const MCInstrInfo &MII) const;
1254
1255 /// Get the action definitions for the given opcode. Use this to run a
1256 /// LegalityQuery through the definitions.
1257 const LegalizeRuleSet &getActionDefinitions(unsigned Opcode) const;
1258
1259 /// Get the action definition builder for the given opcode. Use this to define
1260 /// the action definitions.
1261 ///
1262 /// It is an error to request an opcode that has already been requested by the
1263 /// multiple-opcode variant.
1264 LegalizeRuleSet &getActionDefinitionsBuilder(unsigned Opcode);
1265
1266 /// Get the action definition builder for the given set of opcodes. Use this
1267 /// to define the action definitions for multiple opcodes at once. The first
1268 /// opcode given will be considered the representative opcode and will hold
1269 /// the definitions whereas the other opcodes will be configured to refer to
1270 /// the representative opcode. This lowers memory requirements and very
1271 /// slightly improves performance.
1272 ///
1273 /// It would be very easy to introduce unexpected side-effects as a result of
1274 /// this aliasing if it were permitted to request different but intersecting
1275 /// sets of opcodes but that is difficult to keep track of. It is therefore an
1276 /// error to request the same opcode twice using this API, to request an
1277 /// opcode that already has definitions, or to use the single-opcode API on an
1278 /// opcode that has already been requested by this API.
1279 LegalizeRuleSet &
1280 getActionDefinitionsBuilder(std::initializer_list<unsigned> Opcodes);
1281 void aliasActionDefinitions(unsigned OpcodeTo, unsigned OpcodeFrom);
1282
1283 /// Determine what action should be taken to legalize the described
1284 /// instruction. Requires computeTables to have been called.
1285 ///
1286 /// \returns a description of the next legalization step to perform.
1287 LegalizeActionStep getAction(const LegalityQuery &Query) const;
1288
1289 /// Determine what action should be taken to legalize the given generic
1290 /// instruction.
1291 ///
1292 /// \returns a description of the next legalization step to perform.
1293 LegalizeActionStep getAction(const MachineInstr &MI,
1294 const MachineRegisterInfo &MRI) const;
1295
1296 bool isLegal(const LegalityQuery &Query) const {
1297 return getAction(Query).Action == LegalizeAction::Legal;
1298 }
1299
1300 bool isLegalOrCustom(const LegalityQuery &Query) const {
1301 auto Action = getAction(Query).Action;
1302 return Action == LegalizeAction::Legal || Action == LegalizeAction::Custom;
1303 }
1304
1305 bool isLegal(const MachineInstr &MI, const MachineRegisterInfo &MRI) const;
1306 bool isLegalOrCustom(const MachineInstr &MI,
1307 const MachineRegisterInfo &MRI) const;
1308
1309 /// Called for instructions with the Custom LegalizationAction.
1310 virtual bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
1311 LostDebugLocObserver &LocObserver) const {
1312 llvm_unreachable("must implement this if custom action is used");
1313 }
1314
1315 /// \returns true if MI is either legal or has been legalized and false if not
1316 /// legal.
1317 /// Return true if MI is either legal or has been legalized and false
1318 /// if not legal.
1319 virtual bool legalizeIntrinsic(LegalizerHelper &Helper,
1320 MachineInstr &MI) const {
1321 return true;
1322 }
1323
1324 /// Return the opcode (SEXT/ZEXT/ANYEXT) that should be performed while
1325 /// widening a constant of type SmallTy which targets can override.
1326 /// For eg, the DAG does (SmallTy.isByteSized() ? G_SEXT : G_ZEXT) which
1327 /// will be the default.
1328 virtual unsigned getExtOpcodeForWideningConstant(LLT SmallTy) const;
1329
1330private:
1331 static const int FirstOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_START;
1332 static const int LastOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_END;
1333
1334 LegalizeRuleSet RulesForOpcode[LastOp - FirstOp + 1];
1335 LegacyLegalizerInfo LegacyInfo;
1336};
1337
1338#ifndef NDEBUG
1339/// Checks that MIR is fully legal, returns an illegal instruction if it's not,
1340/// nullptr otherwise
1341const MachineInstr *machineFunctionIsIllegal(const MachineFunction &MF);
1342#endif
1343
1344} // end namespace llvm.
1345
1346#endif // LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H
1347

source code of llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h