1 | //===----- R600Packetizer.cpp - VLIW packetizer ---------------------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | /// \file |
10 | /// This pass implements instructions packetization for R600. It unsets isLast |
11 | /// bit of instructions inside a bundle and substitutes src register with |
12 | /// PreviousVector when applicable. |
13 | // |
14 | //===----------------------------------------------------------------------===// |
15 | |
16 | #include "MCTargetDesc/R600MCTargetDesc.h" |
17 | #include "R600.h" |
18 | #include "R600Subtarget.h" |
19 | #include "llvm/CodeGen/DFAPacketizer.h" |
20 | #include "llvm/CodeGen/MachineDominators.h" |
21 | #include "llvm/CodeGen/MachineLoopInfo.h" |
22 | #include "llvm/CodeGen/ScheduleDAG.h" |
23 | |
24 | using namespace llvm; |
25 | |
26 | #define DEBUG_TYPE "packets" |
27 | |
28 | namespace { |
29 | |
30 | class R600Packetizer : public MachineFunctionPass { |
31 | |
32 | public: |
33 | static char ID; |
34 | R600Packetizer() : MachineFunctionPass(ID) {} |
35 | |
36 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
37 | AU.setPreservesCFG(); |
38 | AU.addRequired<MachineDominatorTree>(); |
39 | AU.addPreserved<MachineDominatorTree>(); |
40 | AU.addRequired<MachineLoopInfo>(); |
41 | AU.addPreserved<MachineLoopInfo>(); |
42 | MachineFunctionPass::getAnalysisUsage(AU); |
43 | } |
44 | |
45 | StringRef getPassName() const override { return "R600 Packetizer" ; } |
46 | |
47 | bool runOnMachineFunction(MachineFunction &Fn) override; |
48 | }; |
49 | |
50 | class R600PacketizerList : public VLIWPacketizerList { |
51 | private: |
52 | const R600InstrInfo *TII; |
53 | const R600RegisterInfo &TRI; |
54 | bool VLIW5; |
55 | bool ConsideredInstUsesAlreadyWrittenVectorElement; |
56 | |
57 | unsigned getSlot(const MachineInstr &MI) const { |
58 | return TRI.getHWRegChan(reg: MI.getOperand(i: 0).getReg()); |
59 | } |
60 | |
61 | /// \returns register to PV chan mapping for bundle/single instructions that |
62 | /// immediately precedes I. |
63 | DenseMap<unsigned, unsigned> getPreviousVector(MachineBasicBlock::iterator I) |
64 | const { |
65 | DenseMap<unsigned, unsigned> Result; |
66 | I--; |
67 | if (!TII->isALUInstr(Opcode: I->getOpcode()) && !I->isBundle()) |
68 | return Result; |
69 | MachineBasicBlock::instr_iterator BI = I.getInstrIterator(); |
70 | if (I->isBundle()) |
71 | BI++; |
72 | int LastDstChan = -1; |
73 | do { |
74 | bool isTrans = false; |
75 | int BISlot = getSlot(MI: *BI); |
76 | if (LastDstChan >= BISlot) |
77 | isTrans = true; |
78 | LastDstChan = BISlot; |
79 | if (TII->isPredicated(MI: *BI)) |
80 | continue; |
81 | int OperandIdx = TII->getOperandIdx(BI->getOpcode(), R600::OpName::write); |
82 | if (OperandIdx > -1 && BI->getOperand(i: OperandIdx).getImm() == 0) |
83 | continue; |
84 | int DstIdx = TII->getOperandIdx(BI->getOpcode(), R600::OpName::dst); |
85 | if (DstIdx == -1) { |
86 | continue; |
87 | } |
88 | Register Dst = BI->getOperand(i: DstIdx).getReg(); |
89 | if (isTrans || TII->isTransOnly(MI: *BI)) { |
90 | Result[Dst] = R600::PS; |
91 | continue; |
92 | } |
93 | if (BI->getOpcode() == R600::DOT4_r600 || |
94 | BI->getOpcode() == R600::DOT4_eg) { |
95 | Result[Dst] = R600::PV_X; |
96 | continue; |
97 | } |
98 | if (Dst == R600::OQAP) { |
99 | continue; |
100 | } |
101 | unsigned PVReg = 0; |
102 | switch (TRI.getHWRegChan(reg: Dst)) { |
103 | case 0: |
104 | PVReg = R600::PV_X; |
105 | break; |
106 | case 1: |
107 | PVReg = R600::PV_Y; |
108 | break; |
109 | case 2: |
110 | PVReg = R600::PV_Z; |
111 | break; |
112 | case 3: |
113 | PVReg = R600::PV_W; |
114 | break; |
115 | default: |
116 | llvm_unreachable("Invalid Chan" ); |
117 | } |
118 | Result[Dst] = PVReg; |
119 | } while ((++BI)->isBundledWithPred()); |
120 | return Result; |
121 | } |
122 | |
123 | void substitutePV(MachineInstr &MI, const DenseMap<unsigned, unsigned> &PVs) |
124 | const { |
125 | unsigned Ops[] = { |
126 | R600::OpName::src0, |
127 | R600::OpName::src1, |
128 | R600::OpName::src2 |
129 | }; |
130 | for (unsigned Op : Ops) { |
131 | int OperandIdx = TII->getOperandIdx(MI.getOpcode(), Op); |
132 | if (OperandIdx < 0) |
133 | continue; |
134 | Register Src = MI.getOperand(OperandIdx).getReg(); |
135 | const DenseMap<unsigned, unsigned>::const_iterator It = PVs.find(Src); |
136 | if (It != PVs.end()) |
137 | MI.getOperand(OperandIdx).setReg(It->second); |
138 | } |
139 | } |
140 | public: |
141 | // Ctor. |
142 | R600PacketizerList(MachineFunction &MF, const R600Subtarget &ST, |
143 | MachineLoopInfo &MLI) |
144 | : VLIWPacketizerList(MF, MLI, nullptr), |
145 | TII(ST.getInstrInfo()), |
146 | TRI(TII->getRegisterInfo()) { |
147 | VLIW5 = !ST.hasCaymanISA(); |
148 | } |
149 | |
150 | // initPacketizerState - initialize some internal flags. |
151 | void initPacketizerState() override { |
152 | ConsideredInstUsesAlreadyWrittenVectorElement = false; |
153 | } |
154 | |
155 | // ignorePseudoInstruction - Ignore bundling of pseudo instructions. |
156 | bool ignorePseudoInstruction(const MachineInstr &MI, |
157 | const MachineBasicBlock *MBB) override { |
158 | return false; |
159 | } |
160 | |
161 | // isSoloInstruction - return true if instruction MI can not be packetized |
162 | // with any other instruction, which means that MI itself is a packet. |
163 | bool isSoloInstruction(const MachineInstr &MI) override { |
164 | if (TII->isVector(MI)) |
165 | return true; |
166 | if (!TII->isALUInstr(Opcode: MI.getOpcode())) |
167 | return true; |
168 | if (MI.getOpcode() == R600::GROUP_BARRIER) |
169 | return true; |
170 | // XXX: This can be removed once the packetizer properly handles all the |
171 | // LDS instruction group restrictions. |
172 | return TII->isLDSInstr(Opcode: MI.getOpcode()); |
173 | } |
174 | |
175 | // isLegalToPacketizeTogether - Is it legal to packetize SUI and SUJ |
176 | // together. |
177 | bool isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) override { |
178 | MachineInstr *MII = SUI->getInstr(), *MIJ = SUJ->getInstr(); |
179 | if (getSlot(MI: *MII) == getSlot(MI: *MIJ)) |
180 | ConsideredInstUsesAlreadyWrittenVectorElement = true; |
181 | // Does MII and MIJ share the same pred_sel ? |
182 | int OpI = TII->getOperandIdx(MII->getOpcode(), R600::OpName::pred_sel), |
183 | OpJ = TII->getOperandIdx(MIJ->getOpcode(), R600::OpName::pred_sel); |
184 | Register PredI = (OpI > -1)?MII->getOperand(i: OpI).getReg() : Register(), |
185 | PredJ = (OpJ > -1)?MIJ->getOperand(i: OpJ).getReg() : Register(); |
186 | if (PredI != PredJ) |
187 | return false; |
188 | if (SUJ->isSucc(N: SUI)) { |
189 | for (unsigned i = 0, e = SUJ->Succs.size(); i < e; ++i) { |
190 | const SDep &Dep = SUJ->Succs[i]; |
191 | if (Dep.getSUnit() != SUI) |
192 | continue; |
193 | if (Dep.getKind() == SDep::Anti) |
194 | continue; |
195 | if (Dep.getKind() == SDep::Output) |
196 | if (MII->getOperand(i: 0).getReg() != MIJ->getOperand(i: 0).getReg()) |
197 | continue; |
198 | return false; |
199 | } |
200 | } |
201 | |
202 | bool ARDef = |
203 | TII->definesAddressRegister(MI&: *MII) || TII->definesAddressRegister(MI&: *MIJ); |
204 | bool ARUse = |
205 | TII->usesAddressRegister(MI&: *MII) || TII->usesAddressRegister(MI&: *MIJ); |
206 | |
207 | return !ARDef || !ARUse; |
208 | } |
209 | |
210 | // isLegalToPruneDependencies - Is it legal to prune dependency between SUI |
211 | // and SUJ. |
212 | bool isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) override { |
213 | return false; |
214 | } |
215 | |
216 | void setIsLastBit(MachineInstr *MI, unsigned Bit) const { |
217 | unsigned LastOp = TII->getOperandIdx(MI->getOpcode(), R600::OpName::last); |
218 | MI->getOperand(i: LastOp).setImm(Bit); |
219 | } |
220 | |
221 | bool isBundlableWithCurrentPMI(MachineInstr &MI, |
222 | const DenseMap<unsigned, unsigned> &PV, |
223 | std::vector<R600InstrInfo::BankSwizzle> &BS, |
224 | bool &isTransSlot) { |
225 | isTransSlot = TII->isTransOnly(MI); |
226 | assert (!isTransSlot || VLIW5); |
227 | |
228 | // Is the dst reg sequence legal ? |
229 | if (!isTransSlot && !CurrentPacketMIs.empty()) { |
230 | if (getSlot(MI) <= getSlot(MI: *CurrentPacketMIs.back())) { |
231 | if (ConsideredInstUsesAlreadyWrittenVectorElement && |
232 | !TII->isVectorOnly(MI) && VLIW5) { |
233 | isTransSlot = true; |
234 | LLVM_DEBUG({ |
235 | dbgs() << "Considering as Trans Inst :" ; |
236 | MI.dump(); |
237 | }); |
238 | } |
239 | else |
240 | return false; |
241 | } |
242 | } |
243 | |
244 | // Are the Constants limitations met ? |
245 | CurrentPacketMIs.push_back(x: &MI); |
246 | if (!TII->fitsConstReadLimitations(CurrentPacketMIs)) { |
247 | LLVM_DEBUG({ |
248 | dbgs() << "Couldn't pack :\n" ; |
249 | MI.dump(); |
250 | dbgs() << "with the following packets :\n" ; |
251 | for (unsigned i = 0, e = CurrentPacketMIs.size() - 1; i < e; i++) { |
252 | CurrentPacketMIs[i]->dump(); |
253 | dbgs() << "\n" ; |
254 | } |
255 | dbgs() << "because of Consts read limitations\n" ; |
256 | }); |
257 | CurrentPacketMIs.pop_back(); |
258 | return false; |
259 | } |
260 | |
261 | // Is there a BankSwizzle set that meet Read Port limitations ? |
262 | if (!TII->fitsReadPortLimitations(CurrentPacketMIs, |
263 | PV, BS, isTransSlot)) { |
264 | LLVM_DEBUG({ |
265 | dbgs() << "Couldn't pack :\n" ; |
266 | MI.dump(); |
267 | dbgs() << "with the following packets :\n" ; |
268 | for (unsigned i = 0, e = CurrentPacketMIs.size() - 1; i < e; i++) { |
269 | CurrentPacketMIs[i]->dump(); |
270 | dbgs() << "\n" ; |
271 | } |
272 | dbgs() << "because of Read port limitations\n" ; |
273 | }); |
274 | CurrentPacketMIs.pop_back(); |
275 | return false; |
276 | } |
277 | |
278 | // We cannot read LDS source registers from the Trans slot. |
279 | if (isTransSlot && TII->readsLDSSrcReg(MI)) |
280 | return false; |
281 | |
282 | CurrentPacketMIs.pop_back(); |
283 | return true; |
284 | } |
285 | |
286 | MachineBasicBlock::iterator addToPacket(MachineInstr &MI) override { |
287 | MachineBasicBlock::iterator FirstInBundle = |
288 | CurrentPacketMIs.empty() ? &MI : CurrentPacketMIs.front(); |
289 | const DenseMap<unsigned, unsigned> &PV = |
290 | getPreviousVector(I: FirstInBundle); |
291 | std::vector<R600InstrInfo::BankSwizzle> BS; |
292 | bool isTransSlot; |
293 | |
294 | if (isBundlableWithCurrentPMI(MI, PV, BS, isTransSlot)) { |
295 | for (unsigned i = 0, e = CurrentPacketMIs.size(); i < e; i++) { |
296 | MachineInstr *MI = CurrentPacketMIs[i]; |
297 | unsigned Op = TII->getOperandIdx(MI->getOpcode(), |
298 | R600::OpName::bank_swizzle); |
299 | MI->getOperand(i: Op).setImm(BS[i]); |
300 | } |
301 | unsigned Op = |
302 | TII->getOperandIdx(MI.getOpcode(), R600::OpName::bank_swizzle); |
303 | MI.getOperand(i: Op).setImm(BS.back()); |
304 | if (!CurrentPacketMIs.empty()) |
305 | setIsLastBit(MI: CurrentPacketMIs.back(), Bit: 0); |
306 | substitutePV(MI, PVs: PV); |
307 | MachineBasicBlock::iterator It = VLIWPacketizerList::addToPacket(MI); |
308 | if (isTransSlot) { |
309 | endPacket(MBB: std::next(x: It)->getParent(), MI: std::next(x: It)); |
310 | } |
311 | return It; |
312 | } |
313 | endPacket(MBB: MI.getParent(), MI); |
314 | if (TII->isTransOnly(MI)) |
315 | return MI; |
316 | return VLIWPacketizerList::addToPacket(MI); |
317 | } |
318 | }; |
319 | |
320 | bool R600Packetizer::runOnMachineFunction(MachineFunction &Fn) { |
321 | const R600Subtarget &ST = Fn.getSubtarget<R600Subtarget>(); |
322 | const R600InstrInfo *TII = ST.getInstrInfo(); |
323 | |
324 | MachineLoopInfo &MLI = getAnalysis<MachineLoopInfo>(); |
325 | |
326 | // Instantiate the packetizer. |
327 | R600PacketizerList Packetizer(Fn, ST, MLI); |
328 | |
329 | // DFA state table should not be empty. |
330 | assert(Packetizer.getResourceTracker() && "Empty DFA table!" ); |
331 | assert(Packetizer.getResourceTracker()->getInstrItins()); |
332 | |
333 | if (Packetizer.getResourceTracker()->getInstrItins()->isEmpty()) |
334 | return false; |
335 | |
336 | // |
337 | // Loop over all basic blocks and remove KILL pseudo-instructions |
338 | // These instructions confuse the dependence analysis. Consider: |
339 | // D0 = ... (Insn 0) |
340 | // R0 = KILL R0, D0 (Insn 1) |
341 | // R0 = ... (Insn 2) |
342 | // Here, Insn 1 will result in the dependence graph not emitting an output |
343 | // dependence between Insn 0 and Insn 2. This can lead to incorrect |
344 | // packetization |
345 | // |
346 | for (MachineBasicBlock &MBB : Fn) { |
347 | for (MachineInstr &MI : llvm::make_early_inc_range(Range&: MBB)) { |
348 | if (MI.isKill() || MI.getOpcode() == R600::IMPLICIT_DEF || |
349 | (MI.getOpcode() == R600::CF_ALU && !MI.getOperand(i: 8).getImm())) |
350 | MBB.erase(I: MI); |
351 | } |
352 | } |
353 | |
354 | // Loop over all of the basic blocks. |
355 | for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end(); |
356 | MBB != MBBe; ++MBB) { |
357 | // Find scheduling regions and schedule / packetize each region. |
358 | unsigned RemainingCount = MBB->size(); |
359 | for(MachineBasicBlock::iterator RegionEnd = MBB->end(); |
360 | RegionEnd != MBB->begin();) { |
361 | // The next region starts above the previous region. Look backward in the |
362 | // instruction stream until we find the nearest boundary. |
363 | MachineBasicBlock::iterator I = RegionEnd; |
364 | for(;I != MBB->begin(); --I, --RemainingCount) { |
365 | if (TII->isSchedulingBoundary(*std::prev(x: I), &*MBB, Fn)) |
366 | break; |
367 | } |
368 | I = MBB->begin(); |
369 | |
370 | // Skip empty scheduling regions. |
371 | if (I == RegionEnd) { |
372 | RegionEnd = std::prev(x: RegionEnd); |
373 | --RemainingCount; |
374 | continue; |
375 | } |
376 | // Skip regions with one instruction. |
377 | if (I == std::prev(x: RegionEnd)) { |
378 | RegionEnd = std::prev(x: RegionEnd); |
379 | continue; |
380 | } |
381 | |
382 | Packetizer.PacketizeMIs(MBB: &*MBB, BeginItr: &*I, EndItr: RegionEnd); |
383 | RegionEnd = I; |
384 | } |
385 | } |
386 | |
387 | return true; |
388 | |
389 | } |
390 | |
391 | } // end anonymous namespace |
392 | |
393 | INITIALIZE_PASS_BEGIN(R600Packetizer, DEBUG_TYPE, |
394 | "R600 Packetizer" , false, false) |
395 | INITIALIZE_PASS_END(R600Packetizer, DEBUG_TYPE, |
396 | "R600 Packetizer" , false, false) |
397 | |
398 | char R600Packetizer::ID = 0; |
399 | |
400 | char &llvm::R600PacketizerID = R600Packetizer::ID; |
401 | |
402 | llvm::FunctionPass *llvm::createR600Packetizer() { |
403 | return new R600Packetizer(); |
404 | } |
405 | |