1//===- llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h -------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file implements GIMatchTableExecutor's `executeMatchTable`
10/// function. This is implemented in a separate file because the function is
11/// quite large.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
16#define LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
17
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"
20#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
21#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
22#include "llvm/CodeGen/GlobalISel/Utils.h"
23#include "llvm/CodeGen/MachineInstrBuilder.h"
24#include "llvm/CodeGen/MachineOperand.h"
25#include "llvm/CodeGen/MachineRegisterInfo.h"
26#include "llvm/CodeGen/RegisterBankInfo.h"
27#include "llvm/CodeGen/TargetInstrInfo.h"
28#include "llvm/CodeGen/TargetOpcodes.h"
29#include "llvm/CodeGen/TargetRegisterInfo.h"
30#include "llvm/IR/Constants.h"
31#include "llvm/IR/DataLayout.h"
32#include "llvm/IR/Type.h"
33#include "llvm/Support/CodeGenCoverage.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/LEB128.h"
37#include "llvm/Support/raw_ostream.h"
38#include <cassert>
39#include <cstddef>
40#include <cstdint>
41
42namespace llvm {
43
44template <class TgtExecutor, class PredicateBitset, class ComplexMatcherMemFn,
45 class CustomRendererFn>
46bool GIMatchTableExecutor::executeMatchTable(
47 TgtExecutor &Exec, MatcherState &State,
48 const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
49 &ExecInfo,
50 MachineIRBuilder &Builder, const uint8_t *MatchTable,
51 const TargetInstrInfo &TII, MachineRegisterInfo &MRI,
52 const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI,
53 const PredicateBitset &AvailableFeatures,
54 CodeGenCoverage *CoverageInfo) const {
55
56 uint64_t CurrentIdx = 0;
57 SmallVector<uint64_t, 4> OnFailResumeAt;
58 NewMIVector OutMIs;
59
60 GISelChangeObserver *Observer = Builder.getObserver();
61 // Bypass the flag check on the instruction, and only look at the MCInstrDesc.
62 bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException();
63
64 const uint16_t Flags = State.MIs[0]->getFlags();
65
66 enum RejectAction { RejectAndGiveUp, RejectAndResume };
67 auto handleReject = [&]() -> RejectAction {
68 DEBUG_WITH_TYPE(TgtExecutor::getName(),
69 dbgs() << CurrentIdx << ": Rejected\n");
70 if (OnFailResumeAt.empty())
71 return RejectAndGiveUp;
72 CurrentIdx = OnFailResumeAt.pop_back_val();
73 DEBUG_WITH_TYPE(TgtExecutor::getName(),
74 dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
75 << OnFailResumeAt.size() << " try-blocks remain)\n");
76 return RejectAndResume;
77 };
78
79 const auto propagateFlags = [&]() {
80 for (auto MIB : OutMIs) {
81 // Set the NoFPExcept flag when no original matched instruction could
82 // raise an FP exception, but the new instruction potentially might.
83 uint16_t MIBFlags = Flags;
84 if (NoFPException && MIB->mayRaiseFPException())
85 MIBFlags |= MachineInstr::NoFPExcept;
86 if (Observer)
87 Observer->changingInstr(MI&: *MIB);
88 MIB.setMIFlags(MIBFlags);
89 if (Observer)
90 Observer->changedInstr(MI&: *MIB);
91 }
92 };
93
94 // If the index is >= 0, it's an index in the type objects generated by
95 // TableGen. If the index is <0, it's an index in the recorded types object.
96 const auto getTypeFromIdx = [&](int64_t Idx) -> LLT {
97 if (Idx >= 0)
98 return ExecInfo.TypeObjects[Idx];
99 return State.RecordedTypes[1 - Idx];
100 };
101
102 const auto readULEB = [&]() {
103 unsigned N = 0;
104 uint64_t Val = decodeULEB128(p: MatchTable + CurrentIdx, n: &N);
105 CurrentIdx += N;
106 return Val;
107 };
108
109 // Convenience function to return a signed value. This avoids
110 // us forgetting to first cast to int8_t before casting to a
111 // wider signed int type.
112 // if we casted uint8 directly to a wider type we'd lose
113 // negative values.
114 const auto readS8 = [&]() { return (int8_t)MatchTable[CurrentIdx++]; };
115
116 const auto readU16 = [&]() {
117 auto V = readBytesAs<uint16_t>(MatchTable: MatchTable + CurrentIdx);
118 CurrentIdx += 2;
119 return V;
120 };
121
122 const auto readU32 = [&]() {
123 auto V = readBytesAs<uint32_t>(MatchTable: MatchTable + CurrentIdx);
124 CurrentIdx += 4;
125 return V;
126 };
127
128 const auto readU64 = [&]() {
129 auto V = readBytesAs<uint64_t>(MatchTable: MatchTable + CurrentIdx);
130 CurrentIdx += 8;
131 return V;
132 };
133
134 const auto eraseImpl = [&](MachineInstr *MI) {
135 // If we're erasing the insertion point, ensure we don't leave a dangling
136 // pointer in the builder.
137 if (Builder.getInsertPt() == MI)
138 Builder.setInsertPt(MBB&: *MI->getParent(), II: ++MI->getIterator());
139 if (Observer)
140 Observer->erasingInstr(MI&: *MI);
141 MI->eraseFromParent();
142 };
143
144 while (true) {
145 assert(CurrentIdx != ~0u && "Invalid MatchTable index");
146 uint8_t MatcherOpcode = MatchTable[CurrentIdx++];
147 switch (MatcherOpcode) {
148 case GIM_Try: {
149 DEBUG_WITH_TYPE(TgtExecutor::getName(),
150 dbgs() << CurrentIdx << ": Begin try-block\n");
151 OnFailResumeAt.push_back(Elt: readU32());
152 break;
153 }
154
155 case GIM_RecordInsn:
156 case GIM_RecordInsnIgnoreCopies: {
157 uint64_t NewInsnID = readULEB();
158 uint64_t InsnID = readULEB();
159 uint64_t OpIdx = readULEB();
160
161 // As an optimisation we require that MIs[0] is always the root. Refuse
162 // any attempt to modify it.
163 assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
164
165 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
166 if (!MO.isReg()) {
167 DEBUG_WITH_TYPE(TgtExecutor::getName(),
168 dbgs() << CurrentIdx << ": Not a register\n");
169 if (handleReject() == RejectAndGiveUp)
170 return false;
171 break;
172 }
173 if (MO.getReg().isPhysical()) {
174 DEBUG_WITH_TYPE(TgtExecutor::getName(),
175 dbgs() << CurrentIdx << ": Is a physical register\n");
176 if (handleReject() == RejectAndGiveUp)
177 return false;
178 break;
179 }
180
181 MachineInstr *NewMI;
182 if (MatcherOpcode == GIM_RecordInsnIgnoreCopies)
183 NewMI = getDefIgnoringCopies(Reg: MO.getReg(), MRI);
184 else
185 NewMI = MRI.getVRegDef(Reg: MO.getReg());
186
187 if ((size_t)NewInsnID < State.MIs.size())
188 State.MIs[NewInsnID] = NewMI;
189 else {
190 assert((size_t)NewInsnID == State.MIs.size() &&
191 "Expected to store MIs in order");
192 State.MIs.push_back(Elt: NewMI);
193 }
194 DEBUG_WITH_TYPE(TgtExecutor::getName(),
195 dbgs() << CurrentIdx << ": MIs[" << NewInsnID
196 << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
197 << ")\n");
198 break;
199 }
200
201 case GIM_CheckFeatures: {
202 uint16_t ExpectedBitsetID = readU16();
203 DEBUG_WITH_TYPE(TgtExecutor::getName(),
204 dbgs() << CurrentIdx
205 << ": GIM_CheckFeatures(ExpectedBitsetID="
206 << ExpectedBitsetID << ")\n");
207 if ((AvailableFeatures & ExecInfo.FeatureBitsets[ExpectedBitsetID]) !=
208 ExecInfo.FeatureBitsets[ExpectedBitsetID]) {
209 if (handleReject() == RejectAndGiveUp)
210 return false;
211 }
212 break;
213 }
214 case GIM_CheckOpcode:
215 case GIM_CheckOpcodeIsEither: {
216 uint64_t InsnID = readULEB();
217 uint16_t Expected0 = readU16();
218 uint16_t Expected1 = -1;
219 if (MatcherOpcode == GIM_CheckOpcodeIsEither)
220 Expected1 = readU16();
221
222 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
223 unsigned Opcode = State.MIs[InsnID]->getOpcode();
224
225 DEBUG_WITH_TYPE(TgtExecutor::getName(),
226 dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
227 << "], ExpectedOpcode=" << Expected0;
228 if (MatcherOpcode == GIM_CheckOpcodeIsEither) dbgs()
229 << " || " << Expected1;
230 dbgs() << ") // Got=" << Opcode << "\n";);
231
232 if (Opcode != Expected0 && Opcode != Expected1) {
233 if (handleReject() == RejectAndGiveUp)
234 return false;
235 }
236 break;
237 }
238 case GIM_SwitchOpcode: {
239 uint64_t InsnID = readULEB();
240 uint16_t LowerBound = readU16();
241 uint16_t UpperBound = readU16();
242 uint32_t Default = readU32();
243
244 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
245 const int64_t Opcode = State.MIs[InsnID]->getOpcode();
246
247 DEBUG_WITH_TYPE(TgtExecutor::getName(), {
248 dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
249 << LowerBound << ", " << UpperBound << "), Default=" << Default
250 << ", JumpTable...) // Got=" << Opcode << "\n";
251 });
252 if (Opcode < LowerBound || UpperBound <= Opcode) {
253 CurrentIdx = Default;
254 break;
255 }
256 const auto EntryIdx = (Opcode - LowerBound);
257 // Each entry is 4 bytes
258 CurrentIdx =
259 readBytesAs<uint32_t>(MatchTable: MatchTable + CurrentIdx + (EntryIdx * 4));
260 if (!CurrentIdx) {
261 CurrentIdx = Default;
262 break;
263 }
264 OnFailResumeAt.push_back(Elt: Default);
265 break;
266 }
267
268 case GIM_SwitchType: {
269 uint64_t InsnID = readULEB();
270 uint64_t OpIdx = readULEB();
271 uint16_t LowerBound = readU16();
272 uint16_t UpperBound = readU16();
273 int64_t Default = readU32();
274
275 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
276 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
277
278 DEBUG_WITH_TYPE(TgtExecutor::getName(), {
279 dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
280 << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
281 << UpperBound << "), Default=" << Default
282 << ", JumpTable...) // Got=";
283 if (!MO.isReg())
284 dbgs() << "Not a VReg\n";
285 else
286 dbgs() << MRI.getType(MO.getReg()) << "\n";
287 });
288 if (!MO.isReg()) {
289 CurrentIdx = Default;
290 break;
291 }
292 const LLT Ty = MRI.getType(Reg: MO.getReg());
293 const auto TyI = ExecInfo.TypeIDMap.find(Ty);
294 if (TyI == ExecInfo.TypeIDMap.end()) {
295 CurrentIdx = Default;
296 break;
297 }
298 const int64_t TypeID = TyI->second;
299 if (TypeID < LowerBound || UpperBound <= TypeID) {
300 CurrentIdx = Default;
301 break;
302 }
303 const auto NumEntry = (TypeID - LowerBound);
304 // Each entry is 4 bytes
305 CurrentIdx =
306 readBytesAs<uint32_t>(MatchTable: MatchTable + CurrentIdx + (NumEntry * 4));
307 if (!CurrentIdx) {
308 CurrentIdx = Default;
309 break;
310 }
311 OnFailResumeAt.push_back(Elt: Default);
312 break;
313 }
314
315 case GIM_CheckNumOperands: {
316 uint64_t InsnID = readULEB();
317 uint64_t Expected = readULEB();
318 DEBUG_WITH_TYPE(TgtExecutor::getName(),
319 dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
320 << InsnID << "], Expected=" << Expected << ")\n");
321 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
322 if (State.MIs[InsnID]->getNumOperands() != Expected) {
323 if (handleReject() == RejectAndGiveUp)
324 return false;
325 }
326 break;
327 }
328 case GIM_CheckI64ImmPredicate:
329 case GIM_CheckImmOperandPredicate: {
330 uint64_t InsnID = readULEB();
331 unsigned OpIdx =
332 MatcherOpcode == GIM_CheckImmOperandPredicate ? readULEB() : 1;
333 uint16_t Predicate = readU16();
334 DEBUG_WITH_TYPE(TgtExecutor::getName(),
335 dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs["
336 << InsnID << "]->getOperand(" << OpIdx
337 << "), Predicate=" << Predicate << ")\n");
338 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
339 assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() ||
340 State.MIs[InsnID]->getOperand(OpIdx).isCImm()) &&
341 "Expected immediate operand");
342 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
343 int64_t Value = 0;
344 if (State.MIs[InsnID]->getOperand(i: OpIdx).isCImm())
345 Value = State.MIs[InsnID]->getOperand(i: OpIdx).getCImm()->getSExtValue();
346 else if (State.MIs[InsnID]->getOperand(i: OpIdx).isImm())
347 Value = State.MIs[InsnID]->getOperand(i: OpIdx).getImm();
348 else
349 llvm_unreachable("Expected Imm or CImm operand");
350
351 if (!testImmPredicate_I64(Predicate, Value))
352 if (handleReject() == RejectAndGiveUp)
353 return false;
354 break;
355 }
356 case GIM_CheckAPIntImmPredicate: {
357 uint64_t InsnID = readULEB();
358 uint16_t Predicate = readU16();
359 DEBUG_WITH_TYPE(TgtExecutor::getName(),
360 dbgs()
361 << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
362 << InsnID << "], Predicate=" << Predicate << ")\n");
363 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
364 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
365 "Expected G_CONSTANT");
366 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
367 if (!State.MIs[InsnID]->getOperand(i: 1).isCImm())
368 llvm_unreachable("Expected Imm or CImm operand");
369
370 const APInt &Value =
371 State.MIs[InsnID]->getOperand(i: 1).getCImm()->getValue();
372 if (!testImmPredicate_APInt(Predicate, Value))
373 if (handleReject() == RejectAndGiveUp)
374 return false;
375 break;
376 }
377 case GIM_CheckAPFloatImmPredicate: {
378 uint64_t InsnID = readULEB();
379 uint16_t Predicate = readU16();
380 DEBUG_WITH_TYPE(TgtExecutor::getName(),
381 dbgs()
382 << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
383 << InsnID << "], Predicate=" << Predicate << ")\n");
384 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
385 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
386 "Expected G_FCONSTANT");
387 assert(State.MIs[InsnID]->getOperand(1).isFPImm() &&
388 "Expected FPImm operand");
389 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
390 const APFloat &Value =
391 State.MIs[InsnID]->getOperand(i: 1).getFPImm()->getValueAPF();
392
393 if (!testImmPredicate_APFloat(Predicate, Value))
394 if (handleReject() == RejectAndGiveUp)
395 return false;
396 break;
397 }
398 case GIM_CheckIsBuildVectorAllOnes:
399 case GIM_CheckIsBuildVectorAllZeros: {
400 uint64_t InsnID = readULEB();
401
402 DEBUG_WITH_TYPE(TgtExecutor::getName(),
403 dbgs() << CurrentIdx
404 << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs["
405 << InsnID << "])\n");
406 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
407
408 const MachineInstr *MI = State.MIs[InsnID];
409 assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
410 MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) &&
411 "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC");
412
413 if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) {
414 if (!isBuildVectorAllOnes(MI: *MI, MRI)) {
415 if (handleReject() == RejectAndGiveUp)
416 return false;
417 }
418 } else {
419 if (!isBuildVectorAllZeros(MI: *MI, MRI)) {
420 if (handleReject() == RejectAndGiveUp)
421 return false;
422 }
423 }
424
425 break;
426 }
427 case GIM_CheckSimplePredicate: {
428 // Note: we don't check for invalid here because this is purely a hook to
429 // allow some executors (such as the combiner) to check arbitrary,
430 // contextless predicates, such as whether a rule is enabled or not.
431 uint16_t Predicate = readU16();
432 DEBUG_WITH_TYPE(TgtExecutor::getName(),
433 dbgs() << CurrentIdx
434 << ": GIM_CheckSimplePredicate(Predicate="
435 << Predicate << ")\n");
436 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
437 if (!testSimplePredicate(Predicate)) {
438 if (handleReject() == RejectAndGiveUp)
439 return false;
440 }
441 break;
442 }
443 case GIM_CheckCxxInsnPredicate: {
444 uint64_t InsnID = readULEB();
445 uint16_t Predicate = readU16();
446 DEBUG_WITH_TYPE(TgtExecutor::getName(),
447 dbgs()
448 << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
449 << InsnID << "], Predicate=" << Predicate << ")\n");
450 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
451 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
452
453 if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID], State))
454 if (handleReject() == RejectAndGiveUp)
455 return false;
456 break;
457 }
458 case GIM_CheckHasNoUse: {
459 uint64_t InsnID = readULEB();
460
461 DEBUG_WITH_TYPE(TgtExecutor::getName(),
462 dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs["
463 << InsnID << "]\n");
464
465 const MachineInstr *MI = State.MIs[InsnID];
466 assert(MI && "Used insn before defined");
467 assert(MI->getNumDefs() > 0 && "No defs");
468 const Register Res = MI->getOperand(i: 0).getReg();
469
470 if (!MRI.use_nodbg_empty(RegNo: Res)) {
471 if (handleReject() == RejectAndGiveUp)
472 return false;
473 }
474
475 break;
476 }
477 case GIM_CheckAtomicOrdering: {
478 uint64_t InsnID = readULEB();
479 auto Ordering = (AtomicOrdering)readULEB();
480 DEBUG_WITH_TYPE(TgtExecutor::getName(),
481 dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
482 << InsnID << "], " << (uint64_t)Ordering << ")\n");
483 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
484 if (!State.MIs[InsnID]->hasOneMemOperand())
485 if (handleReject() == RejectAndGiveUp)
486 return false;
487
488 for (const auto &MMO : State.MIs[InsnID]->memoperands())
489 if (MMO->getMergedOrdering() != Ordering)
490 if (handleReject() == RejectAndGiveUp)
491 return false;
492 break;
493 }
494 case GIM_CheckAtomicOrderingOrStrongerThan: {
495 uint64_t InsnID = readULEB();
496 auto Ordering = (AtomicOrdering)readULEB();
497 DEBUG_WITH_TYPE(TgtExecutor::getName(),
498 dbgs() << CurrentIdx
499 << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
500 << InsnID << "], " << (uint64_t)Ordering << ")\n");
501 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
502 if (!State.MIs[InsnID]->hasOneMemOperand())
503 if (handleReject() == RejectAndGiveUp)
504 return false;
505
506 for (const auto &MMO : State.MIs[InsnID]->memoperands())
507 if (!isAtLeastOrStrongerThan(AO: MMO->getMergedOrdering(), Other: Ordering))
508 if (handleReject() == RejectAndGiveUp)
509 return false;
510 break;
511 }
512 case GIM_CheckAtomicOrderingWeakerThan: {
513 uint64_t InsnID = readULEB();
514 auto Ordering = (AtomicOrdering)readULEB();
515 DEBUG_WITH_TYPE(TgtExecutor::getName(),
516 dbgs() << CurrentIdx
517 << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
518 << InsnID << "], " << (uint64_t)Ordering << ")\n");
519 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
520 if (!State.MIs[InsnID]->hasOneMemOperand())
521 if (handleReject() == RejectAndGiveUp)
522 return false;
523
524 for (const auto &MMO : State.MIs[InsnID]->memoperands())
525 if (!isStrongerThan(AO: Ordering, Other: MMO->getMergedOrdering()))
526 if (handleReject() == RejectAndGiveUp)
527 return false;
528 break;
529 }
530 case GIM_CheckMemoryAddressSpace: {
531 uint64_t InsnID = readULEB();
532 uint64_t MMOIdx = readULEB();
533 // This accepts a list of possible address spaces.
534 const uint64_t NumAddrSpace = readULEB();
535
536 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
537 if (handleReject() == RejectAndGiveUp)
538 return false;
539 break;
540 }
541
542 // Need to still jump to the end of the list of address spaces if we find
543 // a match earlier.
544 const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
545
546 const MachineMemOperand *MMO =
547 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
548 const unsigned MMOAddrSpace = MMO->getAddrSpace();
549
550 bool Success = false;
551 for (unsigned I = 0; I != NumAddrSpace; ++I) {
552 uint64_t AddrSpace = readULEB();
553 DEBUG_WITH_TYPE(TgtExecutor::getName(),
554 dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
555 << AddrSpace << '\n');
556
557 if (AddrSpace == MMOAddrSpace) {
558 Success = true;
559 break;
560 }
561 }
562
563 CurrentIdx = LastIdx;
564 if (!Success && handleReject() == RejectAndGiveUp)
565 return false;
566 break;
567 }
568 case GIM_CheckMemoryAlignment: {
569 uint64_t InsnID = readULEB();
570 uint64_t MMOIdx = readULEB();
571 uint64_t MinAlign = readULEB();
572
573 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
574
575 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
576 if (handleReject() == RejectAndGiveUp)
577 return false;
578 break;
579 }
580
581 MachineMemOperand *MMO =
582 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
583 DEBUG_WITH_TYPE(TgtExecutor::getName(),
584 dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
585 << "(MIs[" << InsnID << "]->memoperands() + "
586 << MMOIdx << ")->getAlignment() >= " << MinAlign
587 << ")\n");
588 if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp)
589 return false;
590
591 break;
592 }
593 case GIM_CheckMemorySizeEqualTo: {
594 uint64_t InsnID = readULEB();
595 uint64_t MMOIdx = readULEB();
596 uint32_t Size = readU32();
597
598 DEBUG_WITH_TYPE(TgtExecutor::getName(),
599 dbgs() << CurrentIdx << ": GIM_CheckMemorySizeEqual(MIs["
600 << InsnID << "]->memoperands() + " << MMOIdx
601 << ", Size=" << Size << ")\n");
602 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
603
604 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
605 if (handleReject() == RejectAndGiveUp)
606 return false;
607 break;
608 }
609
610 MachineMemOperand *MMO =
611 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
612
613 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << MMO->getSize()
614 << " bytes vs " << Size
615 << " bytes\n");
616 if (MMO->getSize() != Size)
617 if (handleReject() == RejectAndGiveUp)
618 return false;
619
620 break;
621 }
622 case GIM_CheckMemorySizeEqualToLLT:
623 case GIM_CheckMemorySizeLessThanLLT:
624 case GIM_CheckMemorySizeGreaterThanLLT: {
625 uint64_t InsnID = readULEB();
626 uint64_t MMOIdx = readULEB();
627 uint64_t OpIdx = readULEB();
628
629 DEBUG_WITH_TYPE(
630 TgtExecutor::getName(),
631 dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
632 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT ? "EqualTo"
633 : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
634 ? "GreaterThan"
635 : "LessThan")
636 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
637 << ", OpIdx=" << OpIdx << ")\n");
638 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
639
640 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
641 if (!MO.isReg()) {
642 DEBUG_WITH_TYPE(TgtExecutor::getName(),
643 dbgs() << CurrentIdx << ": Not a register\n");
644 if (handleReject() == RejectAndGiveUp)
645 return false;
646 break;
647 }
648
649 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
650 if (handleReject() == RejectAndGiveUp)
651 return false;
652 break;
653 }
654
655 MachineMemOperand *MMO =
656 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
657
658 unsigned Size = MRI.getType(Reg: MO.getReg()).getSizeInBits();
659 if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
660 MMO->getSizeInBits().getValue() != Size) {
661 if (handleReject() == RejectAndGiveUp)
662 return false;
663 } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
664 MMO->getSizeInBits().getValue() >= Size) {
665 if (handleReject() == RejectAndGiveUp)
666 return false;
667 } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
668 MMO->getSizeInBits().getValue() <= Size)
669 if (handleReject() == RejectAndGiveUp)
670 return false;
671
672 break;
673 }
674 case GIM_RootCheckType:
675 case GIM_CheckType: {
676 uint64_t InsnID = (MatcherOpcode == GIM_RootCheckType) ? 0 : readULEB();
677 uint64_t OpIdx = readULEB();
678 int TypeID = readS8();
679 DEBUG_WITH_TYPE(TgtExecutor::getName(),
680 dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
681 << "]->getOperand(" << OpIdx
682 << "), TypeID=" << TypeID << ")\n");
683 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
684 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
685 if (!MO.isReg() || MRI.getType(Reg: MO.getReg()) != getTypeFromIdx(TypeID)) {
686 if (handleReject() == RejectAndGiveUp)
687 return false;
688 }
689 break;
690 }
691 case GIM_CheckPointerToAny: {
692 uint64_t InsnID = readULEB();
693 uint64_t OpIdx = readULEB();
694 uint64_t SizeInBits = readULEB();
695
696 DEBUG_WITH_TYPE(TgtExecutor::getName(),
697 dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
698 << InsnID << "]->getOperand(" << OpIdx
699 << "), SizeInBits=" << SizeInBits << ")\n");
700 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
701 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
702 const LLT Ty = MRI.getType(Reg: MO.getReg());
703
704 // iPTR must be looked up in the target.
705 if (SizeInBits == 0) {
706 MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
707 const unsigned AddrSpace = Ty.getAddressSpace();
708 SizeInBits = MF->getDataLayout().getPointerSizeInBits(AS: AddrSpace);
709 }
710
711 assert(SizeInBits != 0 && "Pointer size must be known");
712
713 if (MO.isReg()) {
714 if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
715 if (handleReject() == RejectAndGiveUp)
716 return false;
717 } else if (handleReject() == RejectAndGiveUp)
718 return false;
719
720 break;
721 }
722 case GIM_RecordNamedOperand: {
723 uint64_t InsnID = readULEB();
724 uint64_t OpIdx = readULEB();
725 uint64_t StoreIdx = readULEB();
726
727 DEBUG_WITH_TYPE(TgtExecutor::getName(),
728 dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs["
729 << InsnID << "]->getOperand(" << OpIdx
730 << "), StoreIdx=" << StoreIdx << ")\n");
731 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
732 assert(StoreIdx < State.RecordedOperands.size() && "Index out of range");
733 State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(i: OpIdx);
734 break;
735 }
736 case GIM_RecordRegType: {
737 uint64_t InsnID = readULEB();
738 uint64_t OpIdx = readULEB();
739 int TypeIdx = readS8();
740
741 DEBUG_WITH_TYPE(TgtExecutor::getName(),
742 dbgs() << CurrentIdx << ": GIM_RecordRegType(MIs["
743 << InsnID << "]->getOperand(" << OpIdx
744 << "), TypeIdx=" << TypeIdx << ")\n");
745 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
746 assert(TypeIdx < 0 && "Temp types always have negative indexes!");
747 // Indexes start at -1.
748 TypeIdx = 1 - TypeIdx;
749 const auto &Op = State.MIs[InsnID]->getOperand(i: OpIdx);
750 if (State.RecordedTypes.size() <= (uint64_t)TypeIdx)
751 State.RecordedTypes.resize(N: TypeIdx + 1, NV: LLT());
752 State.RecordedTypes[TypeIdx] = MRI.getType(Reg: Op.getReg());
753 break;
754 }
755
756 case GIM_RootCheckRegBankForClass:
757 case GIM_CheckRegBankForClass: {
758 uint64_t InsnID =
759 (MatcherOpcode == GIM_RootCheckRegBankForClass) ? 0 : readULEB();
760 uint64_t OpIdx = readULEB();
761 uint16_t RCEnum = readU16();
762 DEBUG_WITH_TYPE(TgtExecutor::getName(),
763 dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
764 << InsnID << "]->getOperand(" << OpIdx
765 << "), RCEnum=" << RCEnum << ")\n");
766 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
767 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
768 if (!MO.isReg() ||
769 &RBI.getRegBankFromRegClass(RC: *TRI.getRegClass(i: RCEnum),
770 Ty: MRI.getType(Reg: MO.getReg())) !=
771 RBI.getRegBank(Reg: MO.getReg(), MRI, TRI)) {
772 if (handleReject() == RejectAndGiveUp)
773 return false;
774 }
775 break;
776 }
777
778 case GIM_CheckComplexPattern: {
779 uint64_t InsnID = readULEB();
780 uint64_t OpIdx = readULEB();
781 uint16_t RendererID = readU16();
782 uint16_t ComplexPredicateID = readU16();
783 DEBUG_WITH_TYPE(TgtExecutor::getName(),
784 dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
785 << "] = GIM_CheckComplexPattern(MIs[" << InsnID
786 << "]->getOperand(" << OpIdx
787 << "), ComplexPredicateID=" << ComplexPredicateID
788 << ")\n");
789 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
790 // FIXME: Use std::invoke() when it's available.
791 ComplexRendererFns Renderer =
792 (Exec.*ExecInfo.ComplexPredicates[ComplexPredicateID])(
793 State.MIs[InsnID]->getOperand(i: OpIdx));
794 if (Renderer)
795 State.Renderers[RendererID] = *Renderer;
796 else if (handleReject() == RejectAndGiveUp)
797 return false;
798 break;
799 }
800
801 case GIM_CheckConstantInt:
802 case GIM_CheckConstantInt8: {
803 const bool IsInt8 = (MatcherOpcode == GIM_CheckConstantInt8);
804
805 uint64_t InsnID = readULEB();
806 uint64_t OpIdx = readULEB();
807 uint64_t Value = IsInt8 ? (int64_t)readS8() : readU64();
808 DEBUG_WITH_TYPE(TgtExecutor::getName(),
809 dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
810 << InsnID << "]->getOperand(" << OpIdx
811 << "), Value=" << Value << ")\n");
812 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
813 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
814 if (MO.isReg()) {
815 // isOperandImmEqual() will sign-extend to 64-bits, so should we.
816 LLT Ty = MRI.getType(Reg: MO.getReg());
817 // If the type is > 64 bits, it can't be a constant int, so we bail
818 // early because SignExtend64 will assert otherwise.
819 if (Ty.getScalarSizeInBits() > 64) {
820 if (handleReject() == RejectAndGiveUp)
821 return false;
822 break;
823 }
824
825 Value = SignExtend64(X: Value, B: Ty.getScalarSizeInBits());
826 if (!isOperandImmEqual(MO, Value, MRI, /*Splat=*/Splat: true)) {
827 if (handleReject() == RejectAndGiveUp)
828 return false;
829 }
830 } else if (handleReject() == RejectAndGiveUp)
831 return false;
832
833 break;
834 }
835
836 case GIM_CheckLiteralInt: {
837 uint64_t InsnID = readULEB();
838 uint64_t OpIdx = readULEB();
839 int64_t Value = readU64();
840 DEBUG_WITH_TYPE(TgtExecutor::getName(),
841 dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
842 << InsnID << "]->getOperand(" << OpIdx
843 << "), Value=" << Value << ")\n");
844 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
845 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
846 if (MO.isImm() && MO.getImm() == Value)
847 break;
848
849 if (MO.isCImm() && MO.getCImm()->equalsInt(V: Value))
850 break;
851
852 if (handleReject() == RejectAndGiveUp)
853 return false;
854
855 break;
856 }
857
858 case GIM_CheckIntrinsicID: {
859 uint64_t InsnID = readULEB();
860 uint64_t OpIdx = readULEB();
861 uint16_t Value = readU16();
862 DEBUG_WITH_TYPE(TgtExecutor::getName(),
863 dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
864 << InsnID << "]->getOperand(" << OpIdx
865 << "), Value=" << Value << ")\n");
866 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
867 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
868 if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
869 if (handleReject() == RejectAndGiveUp)
870 return false;
871 break;
872 }
873 case GIM_CheckCmpPredicate: {
874 uint64_t InsnID = readULEB();
875 uint64_t OpIdx = readULEB();
876 uint16_t Value = readU16();
877 DEBUG_WITH_TYPE(TgtExecutor::getName(),
878 dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
879 << InsnID << "]->getOperand(" << OpIdx
880 << "), Value=" << Value << ")\n");
881 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
882 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
883 if (!MO.isPredicate() || MO.getPredicate() != Value)
884 if (handleReject() == RejectAndGiveUp)
885 return false;
886 break;
887 }
888 case GIM_CheckIsMBB: {
889 uint64_t InsnID = readULEB();
890 uint64_t OpIdx = readULEB();
891 DEBUG_WITH_TYPE(TgtExecutor::getName(),
892 dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
893 << "]->getOperand(" << OpIdx << "))\n");
894 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
895 if (!State.MIs[InsnID]->getOperand(i: OpIdx).isMBB()) {
896 if (handleReject() == RejectAndGiveUp)
897 return false;
898 }
899 break;
900 }
901 case GIM_CheckIsImm: {
902 uint64_t InsnID = readULEB();
903 uint64_t OpIdx = readULEB();
904 DEBUG_WITH_TYPE(TgtExecutor::getName(),
905 dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
906 << "]->getOperand(" << OpIdx << "))\n");
907 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
908 if (!State.MIs[InsnID]->getOperand(i: OpIdx).isImm()) {
909 if (handleReject() == RejectAndGiveUp)
910 return false;
911 }
912 break;
913 }
914 case GIM_CheckIsSafeToFold: {
915 uint64_t NumInsn = MatchTable[CurrentIdx++];
916 DEBUG_WITH_TYPE(TgtExecutor::getName(),
917 dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(N = "
918 << NumInsn << ")\n");
919 MachineInstr &Root = *State.MIs[0];
920 for (unsigned K = 1, E = NumInsn + 1; K < E; ++K) {
921 if (!isObviouslySafeToFold(MI&: *State.MIs[K], IntoMI&: Root)) {
922 if (handleReject() == RejectAndGiveUp)
923 return false;
924 }
925 }
926 break;
927 }
928 case GIM_CheckIsSameOperand:
929 case GIM_CheckIsSameOperandIgnoreCopies: {
930 uint64_t InsnID = readULEB();
931 uint64_t OpIdx = readULEB();
932 uint64_t OtherInsnID = readULEB();
933 uint64_t OtherOpIdx = readULEB();
934 DEBUG_WITH_TYPE(TgtExecutor::getName(),
935 dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
936 << InsnID << "][" << OpIdx << "], MIs["
937 << OtherInsnID << "][" << OtherOpIdx << "])\n");
938 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
939 assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
940
941 MachineOperand &Op = State.MIs[InsnID]->getOperand(i: OpIdx);
942 MachineOperand &OtherOp = State.MIs[OtherInsnID]->getOperand(i: OtherOpIdx);
943
944 if (MatcherOpcode == GIM_CheckIsSameOperandIgnoreCopies) {
945 if (Op.isReg() && OtherOp.isReg()) {
946 if (getSrcRegIgnoringCopies(Reg: Op.getReg(), MRI) ==
947 getSrcRegIgnoringCopies(Reg: OtherOp.getReg(), MRI))
948 break;
949 }
950 }
951
952 if (!Op.isIdenticalTo(Other: OtherOp)) {
953 if (handleReject() == RejectAndGiveUp)
954 return false;
955 }
956 break;
957 }
958 case GIM_CheckCanReplaceReg: {
959 uint64_t OldInsnID = readULEB();
960 uint64_t OldOpIdx = readULEB();
961 uint64_t NewInsnID = readULEB();
962 uint64_t NewOpIdx = readULEB();
963
964 DEBUG_WITH_TYPE(TgtExecutor::getName(),
965 dbgs() << CurrentIdx << ": GIM_CheckCanReplaceReg(MIs["
966 << OldInsnID << "][" << OldOpIdx << "] = MIs["
967 << NewInsnID << "][" << NewOpIdx << "])\n");
968
969 Register Old = State.MIs[OldInsnID]->getOperand(i: OldOpIdx).getReg();
970 Register New = State.MIs[NewInsnID]->getOperand(i: NewOpIdx).getReg();
971 if (!canReplaceReg(DstReg: Old, SrcReg: New, MRI)) {
972 if (handleReject() == RejectAndGiveUp)
973 return false;
974 }
975 break;
976 }
977 case GIM_MIFlags: {
978 uint64_t InsnID = readULEB();
979 uint32_t Flags = readU32();
980
981 DEBUG_WITH_TYPE(TgtExecutor::getName(),
982 dbgs() << CurrentIdx << ": GIM_MIFlags(MIs[" << InsnID
983 << "], " << Flags << ")\n");
984 if ((State.MIs[InsnID]->getFlags() & Flags) != Flags) {
985 if (handleReject() == RejectAndGiveUp)
986 return false;
987 }
988 break;
989 }
990 case GIM_MIFlagsNot: {
991 uint64_t InsnID = readULEB();
992 uint32_t Flags = readU32();
993
994 DEBUG_WITH_TYPE(TgtExecutor::getName(),
995 dbgs() << CurrentIdx << ": GIM_MIFlagsNot(MIs[" << InsnID
996 << "], " << Flags << ")\n");
997 if ((State.MIs[InsnID]->getFlags() & Flags)) {
998 if (handleReject() == RejectAndGiveUp)
999 return false;
1000 }
1001 break;
1002 }
1003 case GIM_Reject:
1004 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1005 dbgs() << CurrentIdx << ": GIM_Reject\n");
1006 if (handleReject() == RejectAndGiveUp)
1007 return false;
1008 break;
1009 case GIR_MutateOpcode: {
1010 uint64_t OldInsnID = readULEB();
1011 uint64_t NewInsnID = readULEB();
1012 uint16_t NewOpcode = readU16();
1013 if (NewInsnID >= OutMIs.size())
1014 OutMIs.resize(N: NewInsnID + 1);
1015
1016 MachineInstr *OldMI = State.MIs[OldInsnID];
1017 if (Observer)
1018 Observer->changingInstr(MI&: *OldMI);
1019 OutMIs[NewInsnID] = MachineInstrBuilder(*OldMI->getMF(), OldMI);
1020 OutMIs[NewInsnID]->setDesc(TII.get(Opcode: NewOpcode));
1021 if (Observer)
1022 Observer->changedInstr(MI&: *OldMI);
1023 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1024 dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
1025 << NewInsnID << "], MIs[" << OldInsnID << "], "
1026 << NewOpcode << ")\n");
1027 break;
1028 }
1029
1030 case GIR_BuildRootMI:
1031 case GIR_BuildMI: {
1032 uint64_t NewInsnID = (MatcherOpcode == GIR_BuildRootMI) ? 0 : readULEB();
1033 uint16_t Opcode = readU16();
1034 if (NewInsnID >= OutMIs.size())
1035 OutMIs.resize(N: NewInsnID + 1);
1036
1037 OutMIs[NewInsnID] = Builder.buildInstr(Opcode);
1038 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1039 dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
1040 << NewInsnID << "], " << Opcode << ")\n");
1041 break;
1042 }
1043
1044 case GIR_BuildConstant: {
1045 uint64_t TempRegID = readULEB();
1046 uint64_t Imm = readU64();
1047 Builder.buildConstant(Res: State.TempRegisters[TempRegID], Val: Imm);
1048 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1049 dbgs() << CurrentIdx << ": GIR_BuildConstant(TempReg["
1050 << TempRegID << "], Imm=" << Imm << ")\n");
1051 break;
1052 }
1053
1054 case GIR_RootToRootCopy:
1055 case GIR_Copy: {
1056 uint64_t NewInsnID =
1057 (MatcherOpcode == GIR_RootToRootCopy) ? 0 : readULEB();
1058 uint64_t OldInsnID =
1059 (MatcherOpcode == GIR_RootToRootCopy) ? 0 : readULEB();
1060 uint64_t OpIdx = readULEB();
1061 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1062 OutMIs[NewInsnID].add(MO: State.MIs[OldInsnID]->getOperand(i: OpIdx));
1063 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1064 dbgs()
1065 << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
1066 << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
1067 break;
1068 }
1069
1070 case GIR_CopyOrAddZeroReg: {
1071 uint64_t NewInsnID = readULEB();
1072 uint64_t OldInsnID = readULEB();
1073 uint64_t OpIdx = readULEB();
1074 uint16_t ZeroReg = readU16();
1075 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1076 MachineOperand &MO = State.MIs[OldInsnID]->getOperand(i: OpIdx);
1077 if (isOperandImmEqual(MO, Value: 0, MRI))
1078 OutMIs[NewInsnID].addReg(RegNo: ZeroReg);
1079 else
1080 OutMIs[NewInsnID].add(MO);
1081 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1082 dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
1083 << NewInsnID << "], MIs[" << OldInsnID << "], "
1084 << OpIdx << ", " << ZeroReg << ")\n");
1085 break;
1086 }
1087
1088 case GIR_CopySubReg: {
1089 uint64_t NewInsnID = readULEB();
1090 uint64_t OldInsnID = readULEB();
1091 uint64_t OpIdx = readULEB();
1092 uint16_t SubRegIdx = readU16();
1093 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1094 OutMIs[NewInsnID].addReg(RegNo: State.MIs[OldInsnID]->getOperand(i: OpIdx).getReg(),
1095 flags: 0, SubReg: SubRegIdx);
1096 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1097 dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
1098 << NewInsnID << "], MIs[" << OldInsnID << "], "
1099 << OpIdx << ", " << SubRegIdx << ")\n");
1100 break;
1101 }
1102
1103 case GIR_AddImplicitDef: {
1104 uint64_t InsnID = readULEB();
1105 uint16_t RegNum = readU16();
1106 uint16_t Flags = readU16();
1107 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1108 Flags |= RegState::Implicit;
1109 OutMIs[InsnID].addDef(RegNo: RegNum, Flags);
1110 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1111 dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
1112 << InsnID << "], " << RegNum << ")\n");
1113 break;
1114 }
1115
1116 case GIR_AddImplicitUse: {
1117 uint64_t InsnID = readULEB();
1118 uint16_t RegNum = readU16();
1119 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1120 OutMIs[InsnID].addUse(RegNo: RegNum, Flags: RegState::Implicit);
1121 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1122 dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
1123 << InsnID << "], " << RegNum << ")\n");
1124 break;
1125 }
1126
1127 case GIR_AddRegister: {
1128 uint64_t InsnID = readULEB();
1129 uint16_t RegNum = readU16();
1130 uint16_t RegFlags = readU16();
1131 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1132 OutMIs[InsnID].addReg(RegNo: RegNum, flags: RegFlags);
1133 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1134 dbgs()
1135 << CurrentIdx << ": GIR_AddRegister(OutMIs[" << InsnID
1136 << "], " << RegNum << ", " << RegFlags << ")\n");
1137 break;
1138 }
1139 case GIR_AddIntrinsicID: {
1140 uint64_t InsnID = readULEB();
1141 uint16_t Value = readU16();
1142 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1143 OutMIs[InsnID].addIntrinsicID(ID: (Intrinsic::ID)Value);
1144 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1145 dbgs() << CurrentIdx << ": GIR_AddIntrinsicID(OutMIs["
1146 << InsnID << "], " << Value << ")\n");
1147 break;
1148 }
1149 case GIR_SetImplicitDefDead: {
1150 uint64_t InsnID = readULEB();
1151 uint64_t OpIdx = readULEB();
1152 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1153 dbgs() << CurrentIdx << ": GIR_SetImplicitDefDead(OutMIs["
1154 << InsnID << "], OpIdx=" << OpIdx << ")\n");
1155 MachineInstr *MI = OutMIs[InsnID];
1156 assert(MI && "Modifying undefined instruction");
1157 MI->getOperand(i: MI->getNumExplicitOperands() + OpIdx).setIsDead();
1158 break;
1159 }
1160 case GIR_SetMIFlags: {
1161 uint64_t InsnID = readULEB();
1162 uint32_t Flags = readU32();
1163
1164 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1165 dbgs() << CurrentIdx << ": GIR_SetMIFlags(OutMIs["
1166 << InsnID << "], " << Flags << ")\n");
1167 MachineInstr *MI = OutMIs[InsnID];
1168 MI->setFlags(MI->getFlags() | Flags);
1169 break;
1170 }
1171 case GIR_UnsetMIFlags: {
1172 uint64_t InsnID = readULEB();
1173 uint32_t Flags = readU32();
1174
1175 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1176 dbgs() << CurrentIdx << ": GIR_UnsetMIFlags(OutMIs["
1177 << InsnID << "], " << Flags << ")\n");
1178 MachineInstr *MI = OutMIs[InsnID];
1179 MI->setFlags(MI->getFlags() & ~Flags);
1180 break;
1181 }
1182 case GIR_CopyMIFlags: {
1183 uint64_t InsnID = readULEB();
1184 uint64_t OldInsnID = readULEB();
1185
1186 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1187 dbgs() << CurrentIdx << ": GIR_CopyMIFlags(OutMIs["
1188 << InsnID << "], MIs[" << OldInsnID << "])\n");
1189 MachineInstr *MI = OutMIs[InsnID];
1190 MI->setFlags(MI->getFlags() | State.MIs[OldInsnID]->getFlags());
1191 break;
1192 }
1193 case GIR_AddSimpleTempRegister:
1194 case GIR_AddTempRegister:
1195 case GIR_AddTempSubRegister: {
1196 uint64_t InsnID = readULEB();
1197 uint64_t TempRegID = readULEB();
1198 uint16_t TempRegFlags = 0;
1199 if (MatcherOpcode != GIR_AddSimpleTempRegister)
1200 TempRegFlags = readU16();
1201 uint16_t SubReg = 0;
1202 if (MatcherOpcode == GIR_AddTempSubRegister)
1203 SubReg = readU16();
1204
1205 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1206
1207 OutMIs[InsnID].addReg(RegNo: State.TempRegisters[TempRegID], flags: TempRegFlags,
1208 SubReg);
1209 DEBUG_WITH_TYPE(
1210 TgtExecutor::getName(),
1211 dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" << InsnID
1212 << "], TempRegisters[" << TempRegID << "]";
1213 if (SubReg) dbgs() << '.' << TRI.getSubRegIndexName(SubReg);
1214 dbgs() << ", " << TempRegFlags << ")\n");
1215 break;
1216 }
1217
1218 case GIR_AddImm8:
1219 case GIR_AddImm: {
1220 const bool IsAdd8 = (MatcherOpcode == GIR_AddImm8);
1221 uint64_t InsnID = readULEB();
1222 uint64_t Imm = IsAdd8 ? (int64_t)readS8() : readU64();
1223 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1224 OutMIs[InsnID].addImm(Val: Imm);
1225 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1226 dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
1227 << "], " << Imm << ")\n");
1228 break;
1229 }
1230
1231 case GIR_AddCImm: {
1232 uint64_t InsnID = readULEB();
1233 int TypeID = readS8();
1234 uint64_t Imm = readU64();
1235 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1236
1237 unsigned Width = ExecInfo.TypeObjects[TypeID].getScalarSizeInBits();
1238 LLVMContext &Ctx = MF->getFunction().getContext();
1239 OutMIs[InsnID].addCImm(
1240 Val: ConstantInt::get(Ty: IntegerType::get(C&: Ctx, NumBits: Width), V: Imm, /*signed*/ IsSigned: true));
1241 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1242 dbgs() << CurrentIdx << ": GIR_AddCImm(OutMIs[" << InsnID
1243 << "], TypeID=" << TypeID << ", Imm=" << Imm
1244 << ")\n");
1245 break;
1246 }
1247
1248 case GIR_ComplexRenderer: {
1249 uint64_t InsnID = readULEB();
1250 uint16_t RendererID = readU16();
1251 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1252 for (const auto &RenderOpFn : State.Renderers[RendererID])
1253 RenderOpFn(OutMIs[InsnID]);
1254 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1255 dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
1256 << InsnID << "], " << RendererID << ")\n");
1257 break;
1258 }
1259 case GIR_ComplexSubOperandRenderer: {
1260 uint64_t InsnID = readULEB();
1261 uint16_t RendererID = readU16();
1262 uint64_t RenderOpID = readULEB();
1263 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1264 State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
1265 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1266 dbgs() << CurrentIdx
1267 << ": GIR_ComplexSubOperandRenderer(OutMIs["
1268 << InsnID << "], " << RendererID << ", "
1269 << RenderOpID << ")\n");
1270 break;
1271 }
1272 case GIR_ComplexSubOperandSubRegRenderer: {
1273 uint64_t InsnID = readULEB();
1274 uint16_t RendererID = readU16();
1275 uint64_t RenderOpID = readULEB();
1276 uint16_t SubRegIdx = readU16();
1277 MachineInstrBuilder &MI = OutMIs[InsnID];
1278 assert(MI && "Attempted to add to undefined instruction");
1279 State.Renderers[RendererID][RenderOpID](MI);
1280 MI->getOperand(i: MI->getNumOperands() - 1).setSubReg(SubRegIdx);
1281 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1282 dbgs() << CurrentIdx
1283 << ": GIR_ComplexSubOperandSubRegRenderer(OutMIs["
1284 << InsnID << "], " << RendererID << ", "
1285 << RenderOpID << ", " << SubRegIdx << ")\n");
1286 break;
1287 }
1288
1289 case GIR_CopyConstantAsSImm: {
1290 uint64_t NewInsnID = readULEB();
1291 uint64_t OldInsnID = readULEB();
1292 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1293 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
1294 "Expected G_CONSTANT");
1295 if (State.MIs[OldInsnID]->getOperand(i: 1).isCImm()) {
1296 OutMIs[NewInsnID].addImm(
1297 Val: State.MIs[OldInsnID]->getOperand(i: 1).getCImm()->getSExtValue());
1298 } else if (State.MIs[OldInsnID]->getOperand(i: 1).isImm())
1299 OutMIs[NewInsnID].add(MO: State.MIs[OldInsnID]->getOperand(i: 1));
1300 else
1301 llvm_unreachable("Expected Imm or CImm operand");
1302 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1303 dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
1304 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1305 break;
1306 }
1307
1308 // TODO: Needs a test case once we have a pattern that uses this.
1309 case GIR_CopyFConstantAsFPImm: {
1310 uint64_t NewInsnID = readULEB();
1311 uint64_t OldInsnID = readULEB();
1312 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1313 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
1314 "Expected G_FCONSTANT");
1315 if (State.MIs[OldInsnID]->getOperand(i: 1).isFPImm())
1316 OutMIs[NewInsnID].addFPImm(
1317 Val: State.MIs[OldInsnID]->getOperand(i: 1).getFPImm());
1318 else
1319 llvm_unreachable("Expected FPImm operand");
1320 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1321 dbgs()
1322 << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
1323 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1324 break;
1325 }
1326
1327 case GIR_CustomRenderer: {
1328 uint64_t InsnID = readULEB();
1329 uint64_t OldInsnID = readULEB();
1330 uint16_t RendererFnID = readU16();
1331 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1332 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1333 dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
1334 << InsnID << "], MIs[" << OldInsnID << "], "
1335 << RendererFnID << ")\n");
1336 (Exec.*ExecInfo.CustomRenderers[RendererFnID])(
1337 OutMIs[InsnID], *State.MIs[OldInsnID],
1338 -1); // Not a source operand of the old instruction.
1339 break;
1340 }
1341 case GIR_CustomAction: {
1342 uint16_t FnID = readU16();
1343 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1344 dbgs() << CurrentIdx << ": GIR_CustomAction(FnID=" << FnID
1345 << ")\n");
1346 assert(FnID > GICXXCustomAction_Invalid && "Expected a valid FnID");
1347 runCustomAction(FnID, State, OutMIs);
1348 break;
1349 }
1350 case GIR_CustomOperandRenderer: {
1351 uint64_t InsnID = readULEB();
1352 uint64_t OldInsnID = readULEB();
1353 uint64_t OpIdx = readULEB();
1354 uint16_t RendererFnID = readU16();
1355 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1356
1357 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1358 dbgs() << CurrentIdx
1359 << ": GIR_CustomOperandRenderer(OutMIs[" << InsnID
1360 << "], MIs[" << OldInsnID << "]->getOperand("
1361 << OpIdx << "), " << RendererFnID << ")\n");
1362 (Exec.*ExecInfo.CustomRenderers[RendererFnID])(
1363 OutMIs[InsnID], *State.MIs[OldInsnID], OpIdx);
1364 break;
1365 }
1366 case GIR_ConstrainOperandRC: {
1367 uint64_t InsnID = readULEB();
1368 uint64_t OpIdx = readULEB();
1369 uint16_t RCEnum = readU16();
1370 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1371 MachineInstr &I = *OutMIs[InsnID].getInstr();
1372 MachineFunction &MF = *I.getParent()->getParent();
1373 MachineRegisterInfo &MRI = MF.getRegInfo();
1374 const TargetRegisterClass &RC = *TRI.getRegClass(i: RCEnum);
1375 MachineOperand &MO = I.getOperand(i: OpIdx);
1376 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, InsertPt&: I, RegClass: RC, RegMO&: MO);
1377 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1378 dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
1379 << InsnID << "], " << OpIdx << ", " << RCEnum
1380 << ")\n");
1381 break;
1382 }
1383
1384 case GIR_RootConstrainSelectedInstOperands:
1385 case GIR_ConstrainSelectedInstOperands: {
1386 uint64_t InsnID = (MatcherOpcode == GIR_RootConstrainSelectedInstOperands)
1387 ? 0
1388 : readULEB();
1389 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1390 constrainSelectedInstRegOperands(I&: *OutMIs[InsnID].getInstr(), TII, TRI,
1391 RBI);
1392 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1393 dbgs() << CurrentIdx
1394 << ": GIR_ConstrainSelectedInstOperands(OutMIs["
1395 << InsnID << "])\n");
1396 break;
1397 }
1398 case GIR_MergeMemOperands: {
1399 uint64_t InsnID = readULEB();
1400 uint64_t NumInsn = MatchTable[CurrentIdx++];
1401 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1402
1403 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1404 dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
1405 << InsnID << "]");
1406 for (unsigned K = 0; K < NumInsn; ++K) {
1407 uint64_t NextID = readULEB();
1408 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1409 dbgs() << ", MIs[" << NextID << "]");
1410 for (const auto &MMO : State.MIs[NextID]->memoperands())
1411 OutMIs[InsnID].addMemOperand(MMO);
1412 }
1413 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << ")\n");
1414 break;
1415 }
1416 case GIR_EraseFromParent: {
1417 uint64_t InsnID = readULEB();
1418 MachineInstr *MI = State.MIs[InsnID];
1419 assert(MI && "Attempted to erase an undefined instruction");
1420 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1421 dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
1422 << InsnID << "])\n");
1423 eraseImpl(MI);
1424 break;
1425 }
1426 case GIR_EraseRootFromParent_Done: {
1427 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1428 dbgs()
1429 << CurrentIdx << ": GIR_EraseRootFromParent_Done\n");
1430 eraseImpl(State.MIs[0]);
1431 propagateFlags();
1432 return true;
1433 }
1434 case GIR_MakeTempReg: {
1435 uint64_t TempRegID = readULEB();
1436 int TypeID = readS8();
1437
1438 State.TempRegisters[TempRegID] =
1439 MRI.createGenericVirtualRegister(Ty: getTypeFromIdx(TypeID));
1440 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1441 dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
1442 << "] = GIR_MakeTempReg(" << TypeID << ")\n");
1443 break;
1444 }
1445 case GIR_ReplaceReg: {
1446 uint64_t OldInsnID = readULEB();
1447 uint64_t OldOpIdx = readULEB();
1448 uint64_t NewInsnID = readULEB();
1449 uint64_t NewOpIdx = readULEB();
1450
1451 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1452 dbgs() << CurrentIdx << ": GIR_ReplaceReg(MIs["
1453 << OldInsnID << "][" << OldOpIdx << "] = MIs["
1454 << NewInsnID << "][" << NewOpIdx << "])\n");
1455
1456 Register Old = State.MIs[OldInsnID]->getOperand(i: OldOpIdx).getReg();
1457 Register New = State.MIs[NewInsnID]->getOperand(i: NewOpIdx).getReg();
1458 if (Observer)
1459 Observer->changingAllUsesOfReg(MRI, Reg: Old);
1460 MRI.replaceRegWith(FromReg: Old, ToReg: New);
1461 if (Observer)
1462 Observer->finishedChangingAllUsesOfReg();
1463 break;
1464 }
1465 case GIR_ReplaceRegWithTempReg: {
1466 uint64_t OldInsnID = readULEB();
1467 uint64_t OldOpIdx = readULEB();
1468 uint64_t TempRegID = readULEB();
1469
1470 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1471 dbgs() << CurrentIdx << ": GIR_ReplaceRegWithTempReg(MIs["
1472 << OldInsnID << "][" << OldOpIdx << "] = TempRegs["
1473 << TempRegID << "])\n");
1474
1475 Register Old = State.MIs[OldInsnID]->getOperand(i: OldOpIdx).getReg();
1476 Register New = State.TempRegisters[TempRegID];
1477 if (Observer)
1478 Observer->changingAllUsesOfReg(MRI, Reg: Old);
1479 MRI.replaceRegWith(FromReg: Old, ToReg: New);
1480 if (Observer)
1481 Observer->finishedChangingAllUsesOfReg();
1482 break;
1483 }
1484 case GIR_Coverage: {
1485 uint32_t RuleID = readU32();
1486 assert(CoverageInfo);
1487 CoverageInfo->setCovered(RuleID);
1488
1489 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << CurrentIdx
1490 << ": GIR_Coverage("
1491 << RuleID << ")");
1492 break;
1493 }
1494
1495 case GIR_Done:
1496 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1497 dbgs() << CurrentIdx << ": GIR_Done\n");
1498 propagateFlags();
1499 return true;
1500 default:
1501 llvm_unreachable("Unexpected command");
1502 }
1503 }
1504}
1505
1506} // end namespace llvm
1507
1508#endif // LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
1509

source code of llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h