1 | //===- llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h -------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | /// \file This file implements GIMatchTableExecutor's `executeMatchTable` |
10 | /// function. This is implemented in a separate file because the function is |
11 | /// quite large. |
12 | // |
13 | //===----------------------------------------------------------------------===// |
14 | |
15 | #ifndef LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H |
16 | #define LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H |
17 | |
18 | #include "llvm/ADT/SmallVector.h" |
19 | #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h" |
20 | #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" |
21 | #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" |
22 | #include "llvm/CodeGen/GlobalISel/Utils.h" |
23 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
24 | #include "llvm/CodeGen/MachineOperand.h" |
25 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
26 | #include "llvm/CodeGen/RegisterBankInfo.h" |
27 | #include "llvm/CodeGen/TargetInstrInfo.h" |
28 | #include "llvm/CodeGen/TargetOpcodes.h" |
29 | #include "llvm/CodeGen/TargetRegisterInfo.h" |
30 | #include "llvm/IR/Constants.h" |
31 | #include "llvm/IR/DataLayout.h" |
32 | #include "llvm/IR/Type.h" |
33 | #include "llvm/Support/CodeGenCoverage.h" |
34 | #include "llvm/Support/Debug.h" |
35 | #include "llvm/Support/ErrorHandling.h" |
36 | #include "llvm/Support/LEB128.h" |
37 | #include "llvm/Support/raw_ostream.h" |
38 | #include <cassert> |
39 | #include <cstddef> |
40 | #include <cstdint> |
41 | |
42 | namespace llvm { |
43 | |
44 | template <class TgtExecutor, class PredicateBitset, class ComplexMatcherMemFn, |
45 | class CustomRendererFn> |
46 | bool GIMatchTableExecutor::executeMatchTable( |
47 | TgtExecutor &Exec, MatcherState &State, |
48 | const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn> |
49 | &ExecInfo, |
50 | MachineIRBuilder &Builder, const uint8_t *MatchTable, |
51 | const TargetInstrInfo &TII, MachineRegisterInfo &MRI, |
52 | const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI, |
53 | const PredicateBitset &AvailableFeatures, |
54 | CodeGenCoverage *CoverageInfo) const { |
55 | |
56 | uint64_t CurrentIdx = 0; |
57 | SmallVector<uint64_t, 4> OnFailResumeAt; |
58 | NewMIVector OutMIs; |
59 | |
60 | GISelChangeObserver *Observer = Builder.getObserver(); |
61 | // Bypass the flag check on the instruction, and only look at the MCInstrDesc. |
62 | bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException(); |
63 | |
64 | const uint16_t Flags = State.MIs[0]->getFlags(); |
65 | |
66 | enum RejectAction { RejectAndGiveUp, RejectAndResume }; |
67 | auto handleReject = [&]() -> RejectAction { |
68 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
69 | dbgs() << CurrentIdx << ": Rejected\n" ); |
70 | if (OnFailResumeAt.empty()) |
71 | return RejectAndGiveUp; |
72 | CurrentIdx = OnFailResumeAt.pop_back_val(); |
73 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
74 | dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" |
75 | << OnFailResumeAt.size() << " try-blocks remain)\n" ); |
76 | return RejectAndResume; |
77 | }; |
78 | |
79 | const auto propagateFlags = [&]() { |
80 | for (auto MIB : OutMIs) { |
81 | // Set the NoFPExcept flag when no original matched instruction could |
82 | // raise an FP exception, but the new instruction potentially might. |
83 | uint16_t MIBFlags = Flags; |
84 | if (NoFPException && MIB->mayRaiseFPException()) |
85 | MIBFlags |= MachineInstr::NoFPExcept; |
86 | if (Observer) |
87 | Observer->changingInstr(MI&: *MIB); |
88 | MIB.setMIFlags(MIBFlags); |
89 | if (Observer) |
90 | Observer->changedInstr(MI&: *MIB); |
91 | } |
92 | }; |
93 | |
94 | // If the index is >= 0, it's an index in the type objects generated by |
95 | // TableGen. If the index is <0, it's an index in the recorded types object. |
96 | const auto getTypeFromIdx = [&](int64_t Idx) -> LLT { |
97 | if (Idx >= 0) |
98 | return ExecInfo.TypeObjects[Idx]; |
99 | return State.RecordedTypes[1 - Idx]; |
100 | }; |
101 | |
102 | const auto readULEB = [&]() { |
103 | unsigned N = 0; |
104 | uint64_t Val = decodeULEB128(p: MatchTable + CurrentIdx, n: &N); |
105 | CurrentIdx += N; |
106 | return Val; |
107 | }; |
108 | |
109 | // Convenience function to return a signed value. This avoids |
110 | // us forgetting to first cast to int8_t before casting to a |
111 | // wider signed int type. |
112 | // if we casted uint8 directly to a wider type we'd lose |
113 | // negative values. |
114 | const auto readS8 = [&]() { return (int8_t)MatchTable[CurrentIdx++]; }; |
115 | |
116 | const auto readU16 = [&]() { |
117 | auto V = readBytesAs<uint16_t>(MatchTable: MatchTable + CurrentIdx); |
118 | CurrentIdx += 2; |
119 | return V; |
120 | }; |
121 | |
122 | const auto readU32 = [&]() { |
123 | auto V = readBytesAs<uint32_t>(MatchTable: MatchTable + CurrentIdx); |
124 | CurrentIdx += 4; |
125 | return V; |
126 | }; |
127 | |
128 | const auto readU64 = [&]() { |
129 | auto V = readBytesAs<uint64_t>(MatchTable: MatchTable + CurrentIdx); |
130 | CurrentIdx += 8; |
131 | return V; |
132 | }; |
133 | |
134 | const auto eraseImpl = [&](MachineInstr *MI) { |
135 | // If we're erasing the insertion point, ensure we don't leave a dangling |
136 | // pointer in the builder. |
137 | if (Builder.getInsertPt() == MI) |
138 | Builder.setInsertPt(MBB&: *MI->getParent(), II: ++MI->getIterator()); |
139 | if (Observer) |
140 | Observer->erasingInstr(MI&: *MI); |
141 | MI->eraseFromParent(); |
142 | }; |
143 | |
144 | while (true) { |
145 | assert(CurrentIdx != ~0u && "Invalid MatchTable index" ); |
146 | uint8_t MatcherOpcode = MatchTable[CurrentIdx++]; |
147 | switch (MatcherOpcode) { |
148 | case GIM_Try: { |
149 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
150 | dbgs() << CurrentIdx << ": Begin try-block\n" ); |
151 | OnFailResumeAt.push_back(Elt: readU32()); |
152 | break; |
153 | } |
154 | |
155 | case GIM_RecordInsn: |
156 | case GIM_RecordInsnIgnoreCopies: { |
157 | uint64_t NewInsnID = readULEB(); |
158 | uint64_t InsnID = readULEB(); |
159 | uint64_t OpIdx = readULEB(); |
160 | |
161 | // As an optimisation we require that MIs[0] is always the root. Refuse |
162 | // any attempt to modify it. |
163 | assert(NewInsnID != 0 && "Refusing to modify MIs[0]" ); |
164 | |
165 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
166 | if (!MO.isReg()) { |
167 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
168 | dbgs() << CurrentIdx << ": Not a register\n" ); |
169 | if (handleReject() == RejectAndGiveUp) |
170 | return false; |
171 | break; |
172 | } |
173 | if (MO.getReg().isPhysical()) { |
174 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
175 | dbgs() << CurrentIdx << ": Is a physical register\n" ); |
176 | if (handleReject() == RejectAndGiveUp) |
177 | return false; |
178 | break; |
179 | } |
180 | |
181 | MachineInstr *NewMI; |
182 | if (MatcherOpcode == GIM_RecordInsnIgnoreCopies) |
183 | NewMI = getDefIgnoringCopies(Reg: MO.getReg(), MRI); |
184 | else |
185 | NewMI = MRI.getVRegDef(Reg: MO.getReg()); |
186 | |
187 | if ((size_t)NewInsnID < State.MIs.size()) |
188 | State.MIs[NewInsnID] = NewMI; |
189 | else { |
190 | assert((size_t)NewInsnID == State.MIs.size() && |
191 | "Expected to store MIs in order" ); |
192 | State.MIs.push_back(Elt: NewMI); |
193 | } |
194 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
195 | dbgs() << CurrentIdx << ": MIs[" << NewInsnID |
196 | << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx |
197 | << ")\n" ); |
198 | break; |
199 | } |
200 | |
201 | case GIM_CheckFeatures: { |
202 | uint16_t ExpectedBitsetID = readU16(); |
203 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
204 | dbgs() << CurrentIdx |
205 | << ": GIM_CheckFeatures(ExpectedBitsetID=" |
206 | << ExpectedBitsetID << ")\n" ); |
207 | if ((AvailableFeatures & ExecInfo.FeatureBitsets[ExpectedBitsetID]) != |
208 | ExecInfo.FeatureBitsets[ExpectedBitsetID]) { |
209 | if (handleReject() == RejectAndGiveUp) |
210 | return false; |
211 | } |
212 | break; |
213 | } |
214 | case GIM_CheckOpcode: |
215 | case GIM_CheckOpcodeIsEither: { |
216 | uint64_t InsnID = readULEB(); |
217 | uint16_t Expected0 = readU16(); |
218 | uint16_t Expected1 = -1; |
219 | if (MatcherOpcode == GIM_CheckOpcodeIsEither) |
220 | Expected1 = readU16(); |
221 | |
222 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
223 | unsigned Opcode = State.MIs[InsnID]->getOpcode(); |
224 | |
225 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
226 | dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID |
227 | << "], ExpectedOpcode=" << Expected0; |
228 | if (MatcherOpcode == GIM_CheckOpcodeIsEither) dbgs() |
229 | << " || " << Expected1; |
230 | dbgs() << ") // Got=" << Opcode << "\n" ;); |
231 | |
232 | if (Opcode != Expected0 && Opcode != Expected1) { |
233 | if (handleReject() == RejectAndGiveUp) |
234 | return false; |
235 | } |
236 | break; |
237 | } |
238 | case GIM_SwitchOpcode: { |
239 | uint64_t InsnID = readULEB(); |
240 | uint16_t LowerBound = readU16(); |
241 | uint16_t UpperBound = readU16(); |
242 | uint32_t Default = readU32(); |
243 | |
244 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
245 | const int64_t Opcode = State.MIs[InsnID]->getOpcode(); |
246 | |
247 | DEBUG_WITH_TYPE(TgtExecutor::getName(), { |
248 | dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], [" |
249 | << LowerBound << ", " << UpperBound << "), Default=" << Default |
250 | << ", JumpTable...) // Got=" << Opcode << "\n" ; |
251 | }); |
252 | if (Opcode < LowerBound || UpperBound <= Opcode) { |
253 | CurrentIdx = Default; |
254 | break; |
255 | } |
256 | const auto EntryIdx = (Opcode - LowerBound); |
257 | // Each entry is 4 bytes |
258 | CurrentIdx = |
259 | readBytesAs<uint32_t>(MatchTable: MatchTable + CurrentIdx + (EntryIdx * 4)); |
260 | if (!CurrentIdx) { |
261 | CurrentIdx = Default; |
262 | break; |
263 | } |
264 | OnFailResumeAt.push_back(Elt: Default); |
265 | break; |
266 | } |
267 | |
268 | case GIM_SwitchType: { |
269 | uint64_t InsnID = readULEB(); |
270 | uint64_t OpIdx = readULEB(); |
271 | uint16_t LowerBound = readU16(); |
272 | uint16_t UpperBound = readU16(); |
273 | int64_t Default = readU32(); |
274 | |
275 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
276 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
277 | |
278 | DEBUG_WITH_TYPE(TgtExecutor::getName(), { |
279 | dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID |
280 | << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", " |
281 | << UpperBound << "), Default=" << Default |
282 | << ", JumpTable...) // Got=" ; |
283 | if (!MO.isReg()) |
284 | dbgs() << "Not a VReg\n" ; |
285 | else |
286 | dbgs() << MRI.getType(MO.getReg()) << "\n" ; |
287 | }); |
288 | if (!MO.isReg()) { |
289 | CurrentIdx = Default; |
290 | break; |
291 | } |
292 | const LLT Ty = MRI.getType(Reg: MO.getReg()); |
293 | const auto TyI = ExecInfo.TypeIDMap.find(Ty); |
294 | if (TyI == ExecInfo.TypeIDMap.end()) { |
295 | CurrentIdx = Default; |
296 | break; |
297 | } |
298 | const int64_t TypeID = TyI->second; |
299 | if (TypeID < LowerBound || UpperBound <= TypeID) { |
300 | CurrentIdx = Default; |
301 | break; |
302 | } |
303 | const auto NumEntry = (TypeID - LowerBound); |
304 | // Each entry is 4 bytes |
305 | CurrentIdx = |
306 | readBytesAs<uint32_t>(MatchTable: MatchTable + CurrentIdx + (NumEntry * 4)); |
307 | if (!CurrentIdx) { |
308 | CurrentIdx = Default; |
309 | break; |
310 | } |
311 | OnFailResumeAt.push_back(Elt: Default); |
312 | break; |
313 | } |
314 | |
315 | case GIM_CheckNumOperands: { |
316 | uint64_t InsnID = readULEB(); |
317 | uint64_t Expected = readULEB(); |
318 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
319 | dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs[" |
320 | << InsnID << "], Expected=" << Expected << ")\n" ); |
321 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
322 | if (State.MIs[InsnID]->getNumOperands() != Expected) { |
323 | if (handleReject() == RejectAndGiveUp) |
324 | return false; |
325 | } |
326 | break; |
327 | } |
328 | case GIM_CheckI64ImmPredicate: |
329 | case GIM_CheckImmOperandPredicate: { |
330 | uint64_t InsnID = readULEB(); |
331 | unsigned OpIdx = |
332 | MatcherOpcode == GIM_CheckImmOperandPredicate ? readULEB() : 1; |
333 | uint16_t Predicate = readU16(); |
334 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
335 | dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs[" |
336 | << InsnID << "]->getOperand(" << OpIdx |
337 | << "), Predicate=" << Predicate << ")\n" ); |
338 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
339 | assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() || |
340 | State.MIs[InsnID]->getOperand(OpIdx).isCImm()) && |
341 | "Expected immediate operand" ); |
342 | assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate" ); |
343 | int64_t Value = 0; |
344 | if (State.MIs[InsnID]->getOperand(i: OpIdx).isCImm()) |
345 | Value = State.MIs[InsnID]->getOperand(i: OpIdx).getCImm()->getSExtValue(); |
346 | else if (State.MIs[InsnID]->getOperand(i: OpIdx).isImm()) |
347 | Value = State.MIs[InsnID]->getOperand(i: OpIdx).getImm(); |
348 | else |
349 | llvm_unreachable("Expected Imm or CImm operand" ); |
350 | |
351 | if (!testImmPredicate_I64(Predicate, Value)) |
352 | if (handleReject() == RejectAndGiveUp) |
353 | return false; |
354 | break; |
355 | } |
356 | case GIM_CheckAPIntImmPredicate: { |
357 | uint64_t InsnID = readULEB(); |
358 | uint16_t Predicate = readU16(); |
359 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
360 | dbgs() |
361 | << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs[" |
362 | << InsnID << "], Predicate=" << Predicate << ")\n" ); |
363 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
364 | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && |
365 | "Expected G_CONSTANT" ); |
366 | assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate" ); |
367 | if (!State.MIs[InsnID]->getOperand(i: 1).isCImm()) |
368 | llvm_unreachable("Expected Imm or CImm operand" ); |
369 | |
370 | const APInt &Value = |
371 | State.MIs[InsnID]->getOperand(i: 1).getCImm()->getValue(); |
372 | if (!testImmPredicate_APInt(Predicate, Value)) |
373 | if (handleReject() == RejectAndGiveUp) |
374 | return false; |
375 | break; |
376 | } |
377 | case GIM_CheckAPFloatImmPredicate: { |
378 | uint64_t InsnID = readULEB(); |
379 | uint16_t Predicate = readU16(); |
380 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
381 | dbgs() |
382 | << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs[" |
383 | << InsnID << "], Predicate=" << Predicate << ")\n" ); |
384 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
385 | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && |
386 | "Expected G_FCONSTANT" ); |
387 | assert(State.MIs[InsnID]->getOperand(1).isFPImm() && |
388 | "Expected FPImm operand" ); |
389 | assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate" ); |
390 | const APFloat &Value = |
391 | State.MIs[InsnID]->getOperand(i: 1).getFPImm()->getValueAPF(); |
392 | |
393 | if (!testImmPredicate_APFloat(Predicate, Value)) |
394 | if (handleReject() == RejectAndGiveUp) |
395 | return false; |
396 | break; |
397 | } |
398 | case GIM_CheckIsBuildVectorAllOnes: |
399 | case GIM_CheckIsBuildVectorAllZeros: { |
400 | uint64_t InsnID = readULEB(); |
401 | |
402 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
403 | dbgs() << CurrentIdx |
404 | << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs[" |
405 | << InsnID << "])\n" ); |
406 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
407 | |
408 | const MachineInstr *MI = State.MIs[InsnID]; |
409 | assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR || |
410 | MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) && |
411 | "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC" ); |
412 | |
413 | if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) { |
414 | if (!isBuildVectorAllOnes(MI: *MI, MRI)) { |
415 | if (handleReject() == RejectAndGiveUp) |
416 | return false; |
417 | } |
418 | } else { |
419 | if (!isBuildVectorAllZeros(MI: *MI, MRI)) { |
420 | if (handleReject() == RejectAndGiveUp) |
421 | return false; |
422 | } |
423 | } |
424 | |
425 | break; |
426 | } |
427 | case GIM_CheckSimplePredicate: { |
428 | // Note: we don't check for invalid here because this is purely a hook to |
429 | // allow some executors (such as the combiner) to check arbitrary, |
430 | // contextless predicates, such as whether a rule is enabled or not. |
431 | uint16_t Predicate = readU16(); |
432 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
433 | dbgs() << CurrentIdx |
434 | << ": GIM_CheckSimplePredicate(Predicate=" |
435 | << Predicate << ")\n" ); |
436 | assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate" ); |
437 | if (!testSimplePredicate(Predicate)) { |
438 | if (handleReject() == RejectAndGiveUp) |
439 | return false; |
440 | } |
441 | break; |
442 | } |
443 | case GIM_CheckCxxInsnPredicate: { |
444 | uint64_t InsnID = readULEB(); |
445 | uint16_t Predicate = readU16(); |
446 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
447 | dbgs() |
448 | << CurrentIdx << ": GIM_CheckCxxPredicate(MIs[" |
449 | << InsnID << "], Predicate=" << Predicate << ")\n" ); |
450 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
451 | assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate" ); |
452 | |
453 | if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID], State)) |
454 | if (handleReject() == RejectAndGiveUp) |
455 | return false; |
456 | break; |
457 | } |
458 | case GIM_CheckHasNoUse: { |
459 | uint64_t InsnID = readULEB(); |
460 | |
461 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
462 | dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs[" |
463 | << InsnID << "]\n" ); |
464 | |
465 | const MachineInstr *MI = State.MIs[InsnID]; |
466 | assert(MI && "Used insn before defined" ); |
467 | assert(MI->getNumDefs() > 0 && "No defs" ); |
468 | const Register Res = MI->getOperand(i: 0).getReg(); |
469 | |
470 | if (!MRI.use_nodbg_empty(RegNo: Res)) { |
471 | if (handleReject() == RejectAndGiveUp) |
472 | return false; |
473 | } |
474 | |
475 | break; |
476 | } |
477 | case GIM_CheckAtomicOrdering: { |
478 | uint64_t InsnID = readULEB(); |
479 | auto Ordering = (AtomicOrdering)readULEB(); |
480 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
481 | dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs[" |
482 | << InsnID << "], " << (uint64_t)Ordering << ")\n" ); |
483 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
484 | if (!State.MIs[InsnID]->hasOneMemOperand()) |
485 | if (handleReject() == RejectAndGiveUp) |
486 | return false; |
487 | |
488 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) |
489 | if (MMO->getMergedOrdering() != Ordering) |
490 | if (handleReject() == RejectAndGiveUp) |
491 | return false; |
492 | break; |
493 | } |
494 | case GIM_CheckAtomicOrderingOrStrongerThan: { |
495 | uint64_t InsnID = readULEB(); |
496 | auto Ordering = (AtomicOrdering)readULEB(); |
497 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
498 | dbgs() << CurrentIdx |
499 | << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs[" |
500 | << InsnID << "], " << (uint64_t)Ordering << ")\n" ); |
501 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
502 | if (!State.MIs[InsnID]->hasOneMemOperand()) |
503 | if (handleReject() == RejectAndGiveUp) |
504 | return false; |
505 | |
506 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) |
507 | if (!isAtLeastOrStrongerThan(AO: MMO->getMergedOrdering(), Other: Ordering)) |
508 | if (handleReject() == RejectAndGiveUp) |
509 | return false; |
510 | break; |
511 | } |
512 | case GIM_CheckAtomicOrderingWeakerThan: { |
513 | uint64_t InsnID = readULEB(); |
514 | auto Ordering = (AtomicOrdering)readULEB(); |
515 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
516 | dbgs() << CurrentIdx |
517 | << ": GIM_CheckAtomicOrderingWeakerThan(MIs[" |
518 | << InsnID << "], " << (uint64_t)Ordering << ")\n" ); |
519 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
520 | if (!State.MIs[InsnID]->hasOneMemOperand()) |
521 | if (handleReject() == RejectAndGiveUp) |
522 | return false; |
523 | |
524 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) |
525 | if (!isStrongerThan(AO: Ordering, Other: MMO->getMergedOrdering())) |
526 | if (handleReject() == RejectAndGiveUp) |
527 | return false; |
528 | break; |
529 | } |
530 | case GIM_CheckMemoryAddressSpace: { |
531 | uint64_t InsnID = readULEB(); |
532 | uint64_t MMOIdx = readULEB(); |
533 | // This accepts a list of possible address spaces. |
534 | const uint64_t NumAddrSpace = readULEB(); |
535 | |
536 | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { |
537 | if (handleReject() == RejectAndGiveUp) |
538 | return false; |
539 | break; |
540 | } |
541 | |
542 | // Need to still jump to the end of the list of address spaces if we find |
543 | // a match earlier. |
544 | const uint64_t LastIdx = CurrentIdx + NumAddrSpace; |
545 | |
546 | const MachineMemOperand *MMO = |
547 | *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); |
548 | const unsigned MMOAddrSpace = MMO->getAddrSpace(); |
549 | |
550 | bool Success = false; |
551 | for (unsigned I = 0; I != NumAddrSpace; ++I) { |
552 | uint64_t AddrSpace = readULEB(); |
553 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
554 | dbgs() << "addrspace(" << MMOAddrSpace << ") vs " |
555 | << AddrSpace << '\n'); |
556 | |
557 | if (AddrSpace == MMOAddrSpace) { |
558 | Success = true; |
559 | break; |
560 | } |
561 | } |
562 | |
563 | CurrentIdx = LastIdx; |
564 | if (!Success && handleReject() == RejectAndGiveUp) |
565 | return false; |
566 | break; |
567 | } |
568 | case GIM_CheckMemoryAlignment: { |
569 | uint64_t InsnID = readULEB(); |
570 | uint64_t MMOIdx = readULEB(); |
571 | uint64_t MinAlign = readULEB(); |
572 | |
573 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
574 | |
575 | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { |
576 | if (handleReject() == RejectAndGiveUp) |
577 | return false; |
578 | break; |
579 | } |
580 | |
581 | MachineMemOperand *MMO = |
582 | *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); |
583 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
584 | dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment" |
585 | << "(MIs[" << InsnID << "]->memoperands() + " |
586 | << MMOIdx << ")->getAlignment() >= " << MinAlign |
587 | << ")\n" ); |
588 | if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp) |
589 | return false; |
590 | |
591 | break; |
592 | } |
593 | case GIM_CheckMemorySizeEqualTo: { |
594 | uint64_t InsnID = readULEB(); |
595 | uint64_t MMOIdx = readULEB(); |
596 | uint32_t Size = readU32(); |
597 | |
598 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
599 | dbgs() << CurrentIdx << ": GIM_CheckMemorySizeEqual(MIs[" |
600 | << InsnID << "]->memoperands() + " << MMOIdx |
601 | << ", Size=" << Size << ")\n" ); |
602 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
603 | |
604 | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { |
605 | if (handleReject() == RejectAndGiveUp) |
606 | return false; |
607 | break; |
608 | } |
609 | |
610 | MachineMemOperand *MMO = |
611 | *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); |
612 | |
613 | DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << MMO->getSize() |
614 | << " bytes vs " << Size |
615 | << " bytes\n" ); |
616 | if (MMO->getSize() != Size) |
617 | if (handleReject() == RejectAndGiveUp) |
618 | return false; |
619 | |
620 | break; |
621 | } |
622 | case GIM_CheckMemorySizeEqualToLLT: |
623 | case GIM_CheckMemorySizeLessThanLLT: |
624 | case GIM_CheckMemorySizeGreaterThanLLT: { |
625 | uint64_t InsnID = readULEB(); |
626 | uint64_t MMOIdx = readULEB(); |
627 | uint64_t OpIdx = readULEB(); |
628 | |
629 | DEBUG_WITH_TYPE( |
630 | TgtExecutor::getName(), |
631 | dbgs() << CurrentIdx << ": GIM_CheckMemorySize" |
632 | << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT ? "EqualTo" |
633 | : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT |
634 | ? "GreaterThan" |
635 | : "LessThan" ) |
636 | << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx |
637 | << ", OpIdx=" << OpIdx << ")\n" ); |
638 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
639 | |
640 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
641 | if (!MO.isReg()) { |
642 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
643 | dbgs() << CurrentIdx << ": Not a register\n" ); |
644 | if (handleReject() == RejectAndGiveUp) |
645 | return false; |
646 | break; |
647 | } |
648 | |
649 | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { |
650 | if (handleReject() == RejectAndGiveUp) |
651 | return false; |
652 | break; |
653 | } |
654 | |
655 | MachineMemOperand *MMO = |
656 | *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); |
657 | |
658 | unsigned Size = MRI.getType(Reg: MO.getReg()).getSizeInBits(); |
659 | if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT && |
660 | MMO->getSizeInBits().getValue() != Size) { |
661 | if (handleReject() == RejectAndGiveUp) |
662 | return false; |
663 | } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT && |
664 | MMO->getSizeInBits().getValue() >= Size) { |
665 | if (handleReject() == RejectAndGiveUp) |
666 | return false; |
667 | } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT && |
668 | MMO->getSizeInBits().getValue() <= Size) |
669 | if (handleReject() == RejectAndGiveUp) |
670 | return false; |
671 | |
672 | break; |
673 | } |
674 | case GIM_RootCheckType: |
675 | case GIM_CheckType: { |
676 | uint64_t InsnID = (MatcherOpcode == GIM_RootCheckType) ? 0 : readULEB(); |
677 | uint64_t OpIdx = readULEB(); |
678 | int TypeID = readS8(); |
679 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
680 | dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID |
681 | << "]->getOperand(" << OpIdx |
682 | << "), TypeID=" << TypeID << ")\n" ); |
683 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
684 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
685 | if (!MO.isReg() || MRI.getType(Reg: MO.getReg()) != getTypeFromIdx(TypeID)) { |
686 | if (handleReject() == RejectAndGiveUp) |
687 | return false; |
688 | } |
689 | break; |
690 | } |
691 | case GIM_CheckPointerToAny: { |
692 | uint64_t InsnID = readULEB(); |
693 | uint64_t OpIdx = readULEB(); |
694 | uint64_t SizeInBits = readULEB(); |
695 | |
696 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
697 | dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs[" |
698 | << InsnID << "]->getOperand(" << OpIdx |
699 | << "), SizeInBits=" << SizeInBits << ")\n" ); |
700 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
701 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
702 | const LLT Ty = MRI.getType(Reg: MO.getReg()); |
703 | |
704 | // iPTR must be looked up in the target. |
705 | if (SizeInBits == 0) { |
706 | MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent(); |
707 | const unsigned AddrSpace = Ty.getAddressSpace(); |
708 | SizeInBits = MF->getDataLayout().getPointerSizeInBits(AS: AddrSpace); |
709 | } |
710 | |
711 | assert(SizeInBits != 0 && "Pointer size must be known" ); |
712 | |
713 | if (MO.isReg()) { |
714 | if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) |
715 | if (handleReject() == RejectAndGiveUp) |
716 | return false; |
717 | } else if (handleReject() == RejectAndGiveUp) |
718 | return false; |
719 | |
720 | break; |
721 | } |
722 | case GIM_RecordNamedOperand: { |
723 | uint64_t InsnID = readULEB(); |
724 | uint64_t OpIdx = readULEB(); |
725 | uint64_t StoreIdx = readULEB(); |
726 | |
727 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
728 | dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs[" |
729 | << InsnID << "]->getOperand(" << OpIdx |
730 | << "), StoreIdx=" << StoreIdx << ")\n" ); |
731 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
732 | assert(StoreIdx < State.RecordedOperands.size() && "Index out of range" ); |
733 | State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(i: OpIdx); |
734 | break; |
735 | } |
736 | case GIM_RecordRegType: { |
737 | uint64_t InsnID = readULEB(); |
738 | uint64_t OpIdx = readULEB(); |
739 | int TypeIdx = readS8(); |
740 | |
741 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
742 | dbgs() << CurrentIdx << ": GIM_RecordRegType(MIs[" |
743 | << InsnID << "]->getOperand(" << OpIdx |
744 | << "), TypeIdx=" << TypeIdx << ")\n" ); |
745 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
746 | assert(TypeIdx < 0 && "Temp types always have negative indexes!" ); |
747 | // Indexes start at -1. |
748 | TypeIdx = 1 - TypeIdx; |
749 | const auto &Op = State.MIs[InsnID]->getOperand(i: OpIdx); |
750 | if (State.RecordedTypes.size() <= (uint64_t)TypeIdx) |
751 | State.RecordedTypes.resize(N: TypeIdx + 1, NV: LLT()); |
752 | State.RecordedTypes[TypeIdx] = MRI.getType(Reg: Op.getReg()); |
753 | break; |
754 | } |
755 | |
756 | case GIM_RootCheckRegBankForClass: |
757 | case GIM_CheckRegBankForClass: { |
758 | uint64_t InsnID = |
759 | (MatcherOpcode == GIM_RootCheckRegBankForClass) ? 0 : readULEB(); |
760 | uint64_t OpIdx = readULEB(); |
761 | uint16_t RCEnum = readU16(); |
762 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
763 | dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs[" |
764 | << InsnID << "]->getOperand(" << OpIdx |
765 | << "), RCEnum=" << RCEnum << ")\n" ); |
766 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
767 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
768 | if (!MO.isReg() || |
769 | &RBI.getRegBankFromRegClass(RC: *TRI.getRegClass(i: RCEnum), |
770 | Ty: MRI.getType(Reg: MO.getReg())) != |
771 | RBI.getRegBank(Reg: MO.getReg(), MRI, TRI)) { |
772 | if (handleReject() == RejectAndGiveUp) |
773 | return false; |
774 | } |
775 | break; |
776 | } |
777 | |
778 | case GIM_CheckComplexPattern: { |
779 | uint64_t InsnID = readULEB(); |
780 | uint64_t OpIdx = readULEB(); |
781 | uint16_t RendererID = readU16(); |
782 | uint16_t ComplexPredicateID = readU16(); |
783 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
784 | dbgs() << CurrentIdx << ": State.Renderers[" << RendererID |
785 | << "] = GIM_CheckComplexPattern(MIs[" << InsnID |
786 | << "]->getOperand(" << OpIdx |
787 | << "), ComplexPredicateID=" << ComplexPredicateID |
788 | << ")\n" ); |
789 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
790 | // FIXME: Use std::invoke() when it's available. |
791 | ComplexRendererFns Renderer = |
792 | (Exec.*ExecInfo.ComplexPredicates[ComplexPredicateID])( |
793 | State.MIs[InsnID]->getOperand(i: OpIdx)); |
794 | if (Renderer) |
795 | State.Renderers[RendererID] = *Renderer; |
796 | else if (handleReject() == RejectAndGiveUp) |
797 | return false; |
798 | break; |
799 | } |
800 | |
801 | case GIM_CheckConstantInt: |
802 | case GIM_CheckConstantInt8: { |
803 | const bool IsInt8 = (MatcherOpcode == GIM_CheckConstantInt8); |
804 | |
805 | uint64_t InsnID = readULEB(); |
806 | uint64_t OpIdx = readULEB(); |
807 | uint64_t Value = IsInt8 ? (int64_t)readS8() : readU64(); |
808 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
809 | dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs[" |
810 | << InsnID << "]->getOperand(" << OpIdx |
811 | << "), Value=" << Value << ")\n" ); |
812 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
813 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
814 | if (MO.isReg()) { |
815 | // isOperandImmEqual() will sign-extend to 64-bits, so should we. |
816 | LLT Ty = MRI.getType(Reg: MO.getReg()); |
817 | // If the type is > 64 bits, it can't be a constant int, so we bail |
818 | // early because SignExtend64 will assert otherwise. |
819 | if (Ty.getScalarSizeInBits() > 64) { |
820 | if (handleReject() == RejectAndGiveUp) |
821 | return false; |
822 | break; |
823 | } |
824 | |
825 | Value = SignExtend64(X: Value, B: Ty.getScalarSizeInBits()); |
826 | if (!isOperandImmEqual(MO, Value, MRI, /*Splat=*/Splat: true)) { |
827 | if (handleReject() == RejectAndGiveUp) |
828 | return false; |
829 | } |
830 | } else if (handleReject() == RejectAndGiveUp) |
831 | return false; |
832 | |
833 | break; |
834 | } |
835 | |
836 | case GIM_CheckLiteralInt: { |
837 | uint64_t InsnID = readULEB(); |
838 | uint64_t OpIdx = readULEB(); |
839 | int64_t Value = readU64(); |
840 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
841 | dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs[" |
842 | << InsnID << "]->getOperand(" << OpIdx |
843 | << "), Value=" << Value << ")\n" ); |
844 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
845 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
846 | if (MO.isImm() && MO.getImm() == Value) |
847 | break; |
848 | |
849 | if (MO.isCImm() && MO.getCImm()->equalsInt(V: Value)) |
850 | break; |
851 | |
852 | if (handleReject() == RejectAndGiveUp) |
853 | return false; |
854 | |
855 | break; |
856 | } |
857 | |
858 | case GIM_CheckIntrinsicID: { |
859 | uint64_t InsnID = readULEB(); |
860 | uint64_t OpIdx = readULEB(); |
861 | uint16_t Value = readU16(); |
862 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
863 | dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs[" |
864 | << InsnID << "]->getOperand(" << OpIdx |
865 | << "), Value=" << Value << ")\n" ); |
866 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
867 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
868 | if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value) |
869 | if (handleReject() == RejectAndGiveUp) |
870 | return false; |
871 | break; |
872 | } |
873 | case GIM_CheckCmpPredicate: { |
874 | uint64_t InsnID = readULEB(); |
875 | uint64_t OpIdx = readULEB(); |
876 | uint16_t Value = readU16(); |
877 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
878 | dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs[" |
879 | << InsnID << "]->getOperand(" << OpIdx |
880 | << "), Value=" << Value << ")\n" ); |
881 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
882 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
883 | if (!MO.isPredicate() || MO.getPredicate() != Value) |
884 | if (handleReject() == RejectAndGiveUp) |
885 | return false; |
886 | break; |
887 | } |
888 | case GIM_CheckIsMBB: { |
889 | uint64_t InsnID = readULEB(); |
890 | uint64_t OpIdx = readULEB(); |
891 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
892 | dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID |
893 | << "]->getOperand(" << OpIdx << "))\n" ); |
894 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
895 | if (!State.MIs[InsnID]->getOperand(i: OpIdx).isMBB()) { |
896 | if (handleReject() == RejectAndGiveUp) |
897 | return false; |
898 | } |
899 | break; |
900 | } |
901 | case GIM_CheckIsImm: { |
902 | uint64_t InsnID = readULEB(); |
903 | uint64_t OpIdx = readULEB(); |
904 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
905 | dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID |
906 | << "]->getOperand(" << OpIdx << "))\n" ); |
907 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
908 | if (!State.MIs[InsnID]->getOperand(i: OpIdx).isImm()) { |
909 | if (handleReject() == RejectAndGiveUp) |
910 | return false; |
911 | } |
912 | break; |
913 | } |
914 | case GIM_CheckIsSafeToFold: { |
915 | uint64_t NumInsn = MatchTable[CurrentIdx++]; |
916 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
917 | dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(N = " |
918 | << NumInsn << ")\n" ); |
919 | MachineInstr &Root = *State.MIs[0]; |
920 | for (unsigned K = 1, E = NumInsn + 1; K < E; ++K) { |
921 | if (!isObviouslySafeToFold(MI&: *State.MIs[K], IntoMI&: Root)) { |
922 | if (handleReject() == RejectAndGiveUp) |
923 | return false; |
924 | } |
925 | } |
926 | break; |
927 | } |
928 | case GIM_CheckIsSameOperand: |
929 | case GIM_CheckIsSameOperandIgnoreCopies: { |
930 | uint64_t InsnID = readULEB(); |
931 | uint64_t OpIdx = readULEB(); |
932 | uint64_t OtherInsnID = readULEB(); |
933 | uint64_t OtherOpIdx = readULEB(); |
934 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
935 | dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs[" |
936 | << InsnID << "][" << OpIdx << "], MIs[" |
937 | << OtherInsnID << "][" << OtherOpIdx << "])\n" ); |
938 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
939 | assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined" ); |
940 | |
941 | MachineOperand &Op = State.MIs[InsnID]->getOperand(i: OpIdx); |
942 | MachineOperand &OtherOp = State.MIs[OtherInsnID]->getOperand(i: OtherOpIdx); |
943 | |
944 | if (MatcherOpcode == GIM_CheckIsSameOperandIgnoreCopies) { |
945 | if (Op.isReg() && OtherOp.isReg()) { |
946 | if (getSrcRegIgnoringCopies(Reg: Op.getReg(), MRI) == |
947 | getSrcRegIgnoringCopies(Reg: OtherOp.getReg(), MRI)) |
948 | break; |
949 | } |
950 | } |
951 | |
952 | if (!Op.isIdenticalTo(Other: OtherOp)) { |
953 | if (handleReject() == RejectAndGiveUp) |
954 | return false; |
955 | } |
956 | break; |
957 | } |
958 | case GIM_CheckCanReplaceReg: { |
959 | uint64_t OldInsnID = readULEB(); |
960 | uint64_t OldOpIdx = readULEB(); |
961 | uint64_t NewInsnID = readULEB(); |
962 | uint64_t NewOpIdx = readULEB(); |
963 | |
964 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
965 | dbgs() << CurrentIdx << ": GIM_CheckCanReplaceReg(MIs[" |
966 | << OldInsnID << "][" << OldOpIdx << "] = MIs[" |
967 | << NewInsnID << "][" << NewOpIdx << "])\n" ); |
968 | |
969 | Register Old = State.MIs[OldInsnID]->getOperand(i: OldOpIdx).getReg(); |
970 | Register New = State.MIs[NewInsnID]->getOperand(i: NewOpIdx).getReg(); |
971 | if (!canReplaceReg(DstReg: Old, SrcReg: New, MRI)) { |
972 | if (handleReject() == RejectAndGiveUp) |
973 | return false; |
974 | } |
975 | break; |
976 | } |
977 | case GIM_MIFlags: { |
978 | uint64_t InsnID = readULEB(); |
979 | uint32_t Flags = readU32(); |
980 | |
981 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
982 | dbgs() << CurrentIdx << ": GIM_MIFlags(MIs[" << InsnID |
983 | << "], " << Flags << ")\n" ); |
984 | if ((State.MIs[InsnID]->getFlags() & Flags) != Flags) { |
985 | if (handleReject() == RejectAndGiveUp) |
986 | return false; |
987 | } |
988 | break; |
989 | } |
990 | case GIM_MIFlagsNot: { |
991 | uint64_t InsnID = readULEB(); |
992 | uint32_t Flags = readU32(); |
993 | |
994 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
995 | dbgs() << CurrentIdx << ": GIM_MIFlagsNot(MIs[" << InsnID |
996 | << "], " << Flags << ")\n" ); |
997 | if ((State.MIs[InsnID]->getFlags() & Flags)) { |
998 | if (handleReject() == RejectAndGiveUp) |
999 | return false; |
1000 | } |
1001 | break; |
1002 | } |
1003 | case GIM_Reject: |
1004 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1005 | dbgs() << CurrentIdx << ": GIM_Reject\n" ); |
1006 | if (handleReject() == RejectAndGiveUp) |
1007 | return false; |
1008 | break; |
1009 | case GIR_MutateOpcode: { |
1010 | uint64_t OldInsnID = readULEB(); |
1011 | uint64_t NewInsnID = readULEB(); |
1012 | uint16_t NewOpcode = readU16(); |
1013 | if (NewInsnID >= OutMIs.size()) |
1014 | OutMIs.resize(N: NewInsnID + 1); |
1015 | |
1016 | MachineInstr *OldMI = State.MIs[OldInsnID]; |
1017 | if (Observer) |
1018 | Observer->changingInstr(MI&: *OldMI); |
1019 | OutMIs[NewInsnID] = MachineInstrBuilder(*OldMI->getMF(), OldMI); |
1020 | OutMIs[NewInsnID]->setDesc(TII.get(Opcode: NewOpcode)); |
1021 | if (Observer) |
1022 | Observer->changedInstr(MI&: *OldMI); |
1023 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1024 | dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs[" |
1025 | << NewInsnID << "], MIs[" << OldInsnID << "], " |
1026 | << NewOpcode << ")\n" ); |
1027 | break; |
1028 | } |
1029 | |
1030 | case GIR_BuildRootMI: |
1031 | case GIR_BuildMI: { |
1032 | uint64_t NewInsnID = (MatcherOpcode == GIR_BuildRootMI) ? 0 : readULEB(); |
1033 | uint16_t Opcode = readU16(); |
1034 | if (NewInsnID >= OutMIs.size()) |
1035 | OutMIs.resize(N: NewInsnID + 1); |
1036 | |
1037 | OutMIs[NewInsnID] = Builder.buildInstr(Opcode); |
1038 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1039 | dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs[" |
1040 | << NewInsnID << "], " << Opcode << ")\n" ); |
1041 | break; |
1042 | } |
1043 | |
1044 | case GIR_BuildConstant: { |
1045 | uint64_t TempRegID = readULEB(); |
1046 | uint64_t Imm = readU64(); |
1047 | Builder.buildConstant(Res: State.TempRegisters[TempRegID], Val: Imm); |
1048 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1049 | dbgs() << CurrentIdx << ": GIR_BuildConstant(TempReg[" |
1050 | << TempRegID << "], Imm=" << Imm << ")\n" ); |
1051 | break; |
1052 | } |
1053 | |
1054 | case GIR_RootToRootCopy: |
1055 | case GIR_Copy: { |
1056 | uint64_t NewInsnID = |
1057 | (MatcherOpcode == GIR_RootToRootCopy) ? 0 : readULEB(); |
1058 | uint64_t OldInsnID = |
1059 | (MatcherOpcode == GIR_RootToRootCopy) ? 0 : readULEB(); |
1060 | uint64_t OpIdx = readULEB(); |
1061 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction" ); |
1062 | OutMIs[NewInsnID].add(MO: State.MIs[OldInsnID]->getOperand(i: OpIdx)); |
1063 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1064 | dbgs() |
1065 | << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID |
1066 | << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n" ); |
1067 | break; |
1068 | } |
1069 | |
1070 | case GIR_CopyOrAddZeroReg: { |
1071 | uint64_t NewInsnID = readULEB(); |
1072 | uint64_t OldInsnID = readULEB(); |
1073 | uint64_t OpIdx = readULEB(); |
1074 | uint16_t ZeroReg = readU16(); |
1075 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction" ); |
1076 | MachineOperand &MO = State.MIs[OldInsnID]->getOperand(i: OpIdx); |
1077 | if (isOperandImmEqual(MO, Value: 0, MRI)) |
1078 | OutMIs[NewInsnID].addReg(RegNo: ZeroReg); |
1079 | else |
1080 | OutMIs[NewInsnID].add(MO); |
1081 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1082 | dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs[" |
1083 | << NewInsnID << "], MIs[" << OldInsnID << "], " |
1084 | << OpIdx << ", " << ZeroReg << ")\n" ); |
1085 | break; |
1086 | } |
1087 | |
1088 | case GIR_CopySubReg: { |
1089 | uint64_t NewInsnID = readULEB(); |
1090 | uint64_t OldInsnID = readULEB(); |
1091 | uint64_t OpIdx = readULEB(); |
1092 | uint16_t SubRegIdx = readU16(); |
1093 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction" ); |
1094 | OutMIs[NewInsnID].addReg(RegNo: State.MIs[OldInsnID]->getOperand(i: OpIdx).getReg(), |
1095 | flags: 0, SubReg: SubRegIdx); |
1096 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1097 | dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs[" |
1098 | << NewInsnID << "], MIs[" << OldInsnID << "], " |
1099 | << OpIdx << ", " << SubRegIdx << ")\n" ); |
1100 | break; |
1101 | } |
1102 | |
1103 | case GIR_AddImplicitDef: { |
1104 | uint64_t InsnID = readULEB(); |
1105 | uint16_t RegNum = readU16(); |
1106 | uint16_t Flags = readU16(); |
1107 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1108 | Flags |= RegState::Implicit; |
1109 | OutMIs[InsnID].addDef(RegNo: RegNum, Flags); |
1110 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1111 | dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs[" |
1112 | << InsnID << "], " << RegNum << ")\n" ); |
1113 | break; |
1114 | } |
1115 | |
1116 | case GIR_AddImplicitUse: { |
1117 | uint64_t InsnID = readULEB(); |
1118 | uint16_t RegNum = readU16(); |
1119 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1120 | OutMIs[InsnID].addUse(RegNo: RegNum, Flags: RegState::Implicit); |
1121 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1122 | dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs[" |
1123 | << InsnID << "], " << RegNum << ")\n" ); |
1124 | break; |
1125 | } |
1126 | |
1127 | case GIR_AddRegister: { |
1128 | uint64_t InsnID = readULEB(); |
1129 | uint16_t RegNum = readU16(); |
1130 | uint16_t RegFlags = readU16(); |
1131 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1132 | OutMIs[InsnID].addReg(RegNo: RegNum, flags: RegFlags); |
1133 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1134 | dbgs() |
1135 | << CurrentIdx << ": GIR_AddRegister(OutMIs[" << InsnID |
1136 | << "], " << RegNum << ", " << RegFlags << ")\n" ); |
1137 | break; |
1138 | } |
1139 | case GIR_AddIntrinsicID: { |
1140 | uint64_t InsnID = readULEB(); |
1141 | uint16_t Value = readU16(); |
1142 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1143 | OutMIs[InsnID].addIntrinsicID(ID: (Intrinsic::ID)Value); |
1144 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1145 | dbgs() << CurrentIdx << ": GIR_AddIntrinsicID(OutMIs[" |
1146 | << InsnID << "], " << Value << ")\n" ); |
1147 | break; |
1148 | } |
1149 | case GIR_SetImplicitDefDead: { |
1150 | uint64_t InsnID = readULEB(); |
1151 | uint64_t OpIdx = readULEB(); |
1152 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1153 | dbgs() << CurrentIdx << ": GIR_SetImplicitDefDead(OutMIs[" |
1154 | << InsnID << "], OpIdx=" << OpIdx << ")\n" ); |
1155 | MachineInstr *MI = OutMIs[InsnID]; |
1156 | assert(MI && "Modifying undefined instruction" ); |
1157 | MI->getOperand(i: MI->getNumExplicitOperands() + OpIdx).setIsDead(); |
1158 | break; |
1159 | } |
1160 | case GIR_SetMIFlags: { |
1161 | uint64_t InsnID = readULEB(); |
1162 | uint32_t Flags = readU32(); |
1163 | |
1164 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1165 | dbgs() << CurrentIdx << ": GIR_SetMIFlags(OutMIs[" |
1166 | << InsnID << "], " << Flags << ")\n" ); |
1167 | MachineInstr *MI = OutMIs[InsnID]; |
1168 | MI->setFlags(MI->getFlags() | Flags); |
1169 | break; |
1170 | } |
1171 | case GIR_UnsetMIFlags: { |
1172 | uint64_t InsnID = readULEB(); |
1173 | uint32_t Flags = readU32(); |
1174 | |
1175 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1176 | dbgs() << CurrentIdx << ": GIR_UnsetMIFlags(OutMIs[" |
1177 | << InsnID << "], " << Flags << ")\n" ); |
1178 | MachineInstr *MI = OutMIs[InsnID]; |
1179 | MI->setFlags(MI->getFlags() & ~Flags); |
1180 | break; |
1181 | } |
1182 | case GIR_CopyMIFlags: { |
1183 | uint64_t InsnID = readULEB(); |
1184 | uint64_t OldInsnID = readULEB(); |
1185 | |
1186 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1187 | dbgs() << CurrentIdx << ": GIR_CopyMIFlags(OutMIs[" |
1188 | << InsnID << "], MIs[" << OldInsnID << "])\n" ); |
1189 | MachineInstr *MI = OutMIs[InsnID]; |
1190 | MI->setFlags(MI->getFlags() | State.MIs[OldInsnID]->getFlags()); |
1191 | break; |
1192 | } |
1193 | case GIR_AddSimpleTempRegister: |
1194 | case GIR_AddTempRegister: |
1195 | case GIR_AddTempSubRegister: { |
1196 | uint64_t InsnID = readULEB(); |
1197 | uint64_t TempRegID = readULEB(); |
1198 | uint16_t TempRegFlags = 0; |
1199 | if (MatcherOpcode != GIR_AddSimpleTempRegister) |
1200 | TempRegFlags = readU16(); |
1201 | uint16_t SubReg = 0; |
1202 | if (MatcherOpcode == GIR_AddTempSubRegister) |
1203 | SubReg = readU16(); |
1204 | |
1205 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1206 | |
1207 | OutMIs[InsnID].addReg(RegNo: State.TempRegisters[TempRegID], flags: TempRegFlags, |
1208 | SubReg); |
1209 | DEBUG_WITH_TYPE( |
1210 | TgtExecutor::getName(), |
1211 | dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" << InsnID |
1212 | << "], TempRegisters[" << TempRegID << "]" ; |
1213 | if (SubReg) dbgs() << '.' << TRI.getSubRegIndexName(SubReg); |
1214 | dbgs() << ", " << TempRegFlags << ")\n" ); |
1215 | break; |
1216 | } |
1217 | |
1218 | case GIR_AddImm8: |
1219 | case GIR_AddImm: { |
1220 | const bool IsAdd8 = (MatcherOpcode == GIR_AddImm8); |
1221 | uint64_t InsnID = readULEB(); |
1222 | uint64_t Imm = IsAdd8 ? (int64_t)readS8() : readU64(); |
1223 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1224 | OutMIs[InsnID].addImm(Val: Imm); |
1225 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1226 | dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID |
1227 | << "], " << Imm << ")\n" ); |
1228 | break; |
1229 | } |
1230 | |
1231 | case GIR_AddCImm: { |
1232 | uint64_t InsnID = readULEB(); |
1233 | int TypeID = readS8(); |
1234 | uint64_t Imm = readU64(); |
1235 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1236 | |
1237 | unsigned Width = ExecInfo.TypeObjects[TypeID].getScalarSizeInBits(); |
1238 | LLVMContext &Ctx = MF->getFunction().getContext(); |
1239 | OutMIs[InsnID].addCImm( |
1240 | Val: ConstantInt::get(Ty: IntegerType::get(C&: Ctx, NumBits: Width), V: Imm, /*signed*/ IsSigned: true)); |
1241 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1242 | dbgs() << CurrentIdx << ": GIR_AddCImm(OutMIs[" << InsnID |
1243 | << "], TypeID=" << TypeID << ", Imm=" << Imm |
1244 | << ")\n" ); |
1245 | break; |
1246 | } |
1247 | |
1248 | case GIR_ComplexRenderer: { |
1249 | uint64_t InsnID = readULEB(); |
1250 | uint16_t RendererID = readU16(); |
1251 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1252 | for (const auto &RenderOpFn : State.Renderers[RendererID]) |
1253 | RenderOpFn(OutMIs[InsnID]); |
1254 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1255 | dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs[" |
1256 | << InsnID << "], " << RendererID << ")\n" ); |
1257 | break; |
1258 | } |
1259 | case GIR_ComplexSubOperandRenderer: { |
1260 | uint64_t InsnID = readULEB(); |
1261 | uint16_t RendererID = readU16(); |
1262 | uint64_t RenderOpID = readULEB(); |
1263 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1264 | State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]); |
1265 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1266 | dbgs() << CurrentIdx |
1267 | << ": GIR_ComplexSubOperandRenderer(OutMIs[" |
1268 | << InsnID << "], " << RendererID << ", " |
1269 | << RenderOpID << ")\n" ); |
1270 | break; |
1271 | } |
1272 | case GIR_ComplexSubOperandSubRegRenderer: { |
1273 | uint64_t InsnID = readULEB(); |
1274 | uint16_t RendererID = readU16(); |
1275 | uint64_t RenderOpID = readULEB(); |
1276 | uint16_t SubRegIdx = readU16(); |
1277 | MachineInstrBuilder &MI = OutMIs[InsnID]; |
1278 | assert(MI && "Attempted to add to undefined instruction" ); |
1279 | State.Renderers[RendererID][RenderOpID](MI); |
1280 | MI->getOperand(i: MI->getNumOperands() - 1).setSubReg(SubRegIdx); |
1281 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1282 | dbgs() << CurrentIdx |
1283 | << ": GIR_ComplexSubOperandSubRegRenderer(OutMIs[" |
1284 | << InsnID << "], " << RendererID << ", " |
1285 | << RenderOpID << ", " << SubRegIdx << ")\n" ); |
1286 | break; |
1287 | } |
1288 | |
1289 | case GIR_CopyConstantAsSImm: { |
1290 | uint64_t NewInsnID = readULEB(); |
1291 | uint64_t OldInsnID = readULEB(); |
1292 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction" ); |
1293 | assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && |
1294 | "Expected G_CONSTANT" ); |
1295 | if (State.MIs[OldInsnID]->getOperand(i: 1).isCImm()) { |
1296 | OutMIs[NewInsnID].addImm( |
1297 | Val: State.MIs[OldInsnID]->getOperand(i: 1).getCImm()->getSExtValue()); |
1298 | } else if (State.MIs[OldInsnID]->getOperand(i: 1).isImm()) |
1299 | OutMIs[NewInsnID].add(MO: State.MIs[OldInsnID]->getOperand(i: 1)); |
1300 | else |
1301 | llvm_unreachable("Expected Imm or CImm operand" ); |
1302 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1303 | dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs[" |
1304 | << NewInsnID << "], MIs[" << OldInsnID << "])\n" ); |
1305 | break; |
1306 | } |
1307 | |
1308 | // TODO: Needs a test case once we have a pattern that uses this. |
1309 | case GIR_CopyFConstantAsFPImm: { |
1310 | uint64_t NewInsnID = readULEB(); |
1311 | uint64_t OldInsnID = readULEB(); |
1312 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction" ); |
1313 | assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && |
1314 | "Expected G_FCONSTANT" ); |
1315 | if (State.MIs[OldInsnID]->getOperand(i: 1).isFPImm()) |
1316 | OutMIs[NewInsnID].addFPImm( |
1317 | Val: State.MIs[OldInsnID]->getOperand(i: 1).getFPImm()); |
1318 | else |
1319 | llvm_unreachable("Expected FPImm operand" ); |
1320 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1321 | dbgs() |
1322 | << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs[" |
1323 | << NewInsnID << "], MIs[" << OldInsnID << "])\n" ); |
1324 | break; |
1325 | } |
1326 | |
1327 | case GIR_CustomRenderer: { |
1328 | uint64_t InsnID = readULEB(); |
1329 | uint64_t OldInsnID = readULEB(); |
1330 | uint16_t RendererFnID = readU16(); |
1331 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1332 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1333 | dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs[" |
1334 | << InsnID << "], MIs[" << OldInsnID << "], " |
1335 | << RendererFnID << ")\n" ); |
1336 | (Exec.*ExecInfo.CustomRenderers[RendererFnID])( |
1337 | OutMIs[InsnID], *State.MIs[OldInsnID], |
1338 | -1); // Not a source operand of the old instruction. |
1339 | break; |
1340 | } |
1341 | case GIR_CustomAction: { |
1342 | uint16_t FnID = readU16(); |
1343 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1344 | dbgs() << CurrentIdx << ": GIR_CustomAction(FnID=" << FnID |
1345 | << ")\n" ); |
1346 | assert(FnID > GICXXCustomAction_Invalid && "Expected a valid FnID" ); |
1347 | runCustomAction(FnID, State, OutMIs); |
1348 | break; |
1349 | } |
1350 | case GIR_CustomOperandRenderer: { |
1351 | uint64_t InsnID = readULEB(); |
1352 | uint64_t OldInsnID = readULEB(); |
1353 | uint64_t OpIdx = readULEB(); |
1354 | uint16_t RendererFnID = readU16(); |
1355 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1356 | |
1357 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1358 | dbgs() << CurrentIdx |
1359 | << ": GIR_CustomOperandRenderer(OutMIs[" << InsnID |
1360 | << "], MIs[" << OldInsnID << "]->getOperand(" |
1361 | << OpIdx << "), " << RendererFnID << ")\n" ); |
1362 | (Exec.*ExecInfo.CustomRenderers[RendererFnID])( |
1363 | OutMIs[InsnID], *State.MIs[OldInsnID], OpIdx); |
1364 | break; |
1365 | } |
1366 | case GIR_ConstrainOperandRC: { |
1367 | uint64_t InsnID = readULEB(); |
1368 | uint64_t OpIdx = readULEB(); |
1369 | uint16_t RCEnum = readU16(); |
1370 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1371 | MachineInstr &I = *OutMIs[InsnID].getInstr(); |
1372 | MachineFunction &MF = *I.getParent()->getParent(); |
1373 | MachineRegisterInfo &MRI = MF.getRegInfo(); |
1374 | const TargetRegisterClass &RC = *TRI.getRegClass(i: RCEnum); |
1375 | MachineOperand &MO = I.getOperand(i: OpIdx); |
1376 | constrainOperandRegClass(MF, TRI, MRI, TII, RBI, InsertPt&: I, RegClass: RC, RegMO&: MO); |
1377 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1378 | dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs[" |
1379 | << InsnID << "], " << OpIdx << ", " << RCEnum |
1380 | << ")\n" ); |
1381 | break; |
1382 | } |
1383 | |
1384 | case GIR_RootConstrainSelectedInstOperands: |
1385 | case GIR_ConstrainSelectedInstOperands: { |
1386 | uint64_t InsnID = (MatcherOpcode == GIR_RootConstrainSelectedInstOperands) |
1387 | ? 0 |
1388 | : readULEB(); |
1389 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1390 | constrainSelectedInstRegOperands(I&: *OutMIs[InsnID].getInstr(), TII, TRI, |
1391 | RBI); |
1392 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1393 | dbgs() << CurrentIdx |
1394 | << ": GIR_ConstrainSelectedInstOperands(OutMIs[" |
1395 | << InsnID << "])\n" ); |
1396 | break; |
1397 | } |
1398 | case GIR_MergeMemOperands: { |
1399 | uint64_t InsnID = readULEB(); |
1400 | uint64_t NumInsn = MatchTable[CurrentIdx++]; |
1401 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1402 | |
1403 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1404 | dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs[" |
1405 | << InsnID << "]" ); |
1406 | for (unsigned K = 0; K < NumInsn; ++K) { |
1407 | uint64_t NextID = readULEB(); |
1408 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1409 | dbgs() << ", MIs[" << NextID << "]" ); |
1410 | for (const auto &MMO : State.MIs[NextID]->memoperands()) |
1411 | OutMIs[InsnID].addMemOperand(MMO); |
1412 | } |
1413 | DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << ")\n" ); |
1414 | break; |
1415 | } |
1416 | case GIR_EraseFromParent: { |
1417 | uint64_t InsnID = readULEB(); |
1418 | MachineInstr *MI = State.MIs[InsnID]; |
1419 | assert(MI && "Attempted to erase an undefined instruction" ); |
1420 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1421 | dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs[" |
1422 | << InsnID << "])\n" ); |
1423 | eraseImpl(MI); |
1424 | break; |
1425 | } |
1426 | case GIR_EraseRootFromParent_Done: { |
1427 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1428 | dbgs() |
1429 | << CurrentIdx << ": GIR_EraseRootFromParent_Done\n" ); |
1430 | eraseImpl(State.MIs[0]); |
1431 | propagateFlags(); |
1432 | return true; |
1433 | } |
1434 | case GIR_MakeTempReg: { |
1435 | uint64_t TempRegID = readULEB(); |
1436 | int TypeID = readS8(); |
1437 | |
1438 | State.TempRegisters[TempRegID] = |
1439 | MRI.createGenericVirtualRegister(Ty: getTypeFromIdx(TypeID)); |
1440 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1441 | dbgs() << CurrentIdx << ": TempRegs[" << TempRegID |
1442 | << "] = GIR_MakeTempReg(" << TypeID << ")\n" ); |
1443 | break; |
1444 | } |
1445 | case GIR_ReplaceReg: { |
1446 | uint64_t OldInsnID = readULEB(); |
1447 | uint64_t OldOpIdx = readULEB(); |
1448 | uint64_t NewInsnID = readULEB(); |
1449 | uint64_t NewOpIdx = readULEB(); |
1450 | |
1451 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1452 | dbgs() << CurrentIdx << ": GIR_ReplaceReg(MIs[" |
1453 | << OldInsnID << "][" << OldOpIdx << "] = MIs[" |
1454 | << NewInsnID << "][" << NewOpIdx << "])\n" ); |
1455 | |
1456 | Register Old = State.MIs[OldInsnID]->getOperand(i: OldOpIdx).getReg(); |
1457 | Register New = State.MIs[NewInsnID]->getOperand(i: NewOpIdx).getReg(); |
1458 | if (Observer) |
1459 | Observer->changingAllUsesOfReg(MRI, Reg: Old); |
1460 | MRI.replaceRegWith(FromReg: Old, ToReg: New); |
1461 | if (Observer) |
1462 | Observer->finishedChangingAllUsesOfReg(); |
1463 | break; |
1464 | } |
1465 | case GIR_ReplaceRegWithTempReg: { |
1466 | uint64_t OldInsnID = readULEB(); |
1467 | uint64_t OldOpIdx = readULEB(); |
1468 | uint64_t TempRegID = readULEB(); |
1469 | |
1470 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1471 | dbgs() << CurrentIdx << ": GIR_ReplaceRegWithTempReg(MIs[" |
1472 | << OldInsnID << "][" << OldOpIdx << "] = TempRegs[" |
1473 | << TempRegID << "])\n" ); |
1474 | |
1475 | Register Old = State.MIs[OldInsnID]->getOperand(i: OldOpIdx).getReg(); |
1476 | Register New = State.TempRegisters[TempRegID]; |
1477 | if (Observer) |
1478 | Observer->changingAllUsesOfReg(MRI, Reg: Old); |
1479 | MRI.replaceRegWith(FromReg: Old, ToReg: New); |
1480 | if (Observer) |
1481 | Observer->finishedChangingAllUsesOfReg(); |
1482 | break; |
1483 | } |
1484 | case GIR_Coverage: { |
1485 | uint32_t RuleID = readU32(); |
1486 | assert(CoverageInfo); |
1487 | CoverageInfo->setCovered(RuleID); |
1488 | |
1489 | DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << CurrentIdx |
1490 | << ": GIR_Coverage(" |
1491 | << RuleID << ")" ); |
1492 | break; |
1493 | } |
1494 | |
1495 | case GIR_Done: |
1496 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1497 | dbgs() << CurrentIdx << ": GIR_Done\n" ); |
1498 | propagateFlags(); |
1499 | return true; |
1500 | default: |
1501 | llvm_unreachable("Unexpected command" ); |
1502 | } |
1503 | } |
1504 | } |
1505 | |
1506 | } // end namespace llvm |
1507 | |
1508 | #endif // LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H |
1509 | |