1 | //===-- R600EmitClauseMarkers.cpp - Emit CF_ALU ---------------------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | /// \file |
10 | /// Add CF_ALU. R600 Alu instructions are grouped in clause which can hold |
11 | /// 128 Alu instructions ; these instructions can access up to 4 prefetched |
12 | /// 4 lines of 16 registers from constant buffers. Such ALU clauses are |
13 | /// initiated by CF_ALU instructions. |
14 | //===----------------------------------------------------------------------===// |
15 | |
16 | #include "MCTargetDesc/R600MCTargetDesc.h" |
17 | #include "R600.h" |
18 | #include "R600Defines.h" |
19 | #include "R600Subtarget.h" |
20 | #include "llvm/CodeGen/MachineFunctionPass.h" |
21 | |
22 | using namespace llvm; |
23 | |
24 | namespace llvm { |
25 | |
26 | void initializeR600EmitClauseMarkersPass(PassRegistry&); |
27 | |
28 | } // end namespace llvm |
29 | |
30 | namespace { |
31 | |
32 | class R600EmitClauseMarkers : public MachineFunctionPass { |
33 | private: |
34 | const R600InstrInfo *TII = nullptr; |
35 | int Address = 0; |
36 | |
37 | unsigned OccupiedDwords(MachineInstr &MI) const { |
38 | switch (MI.getOpcode()) { |
39 | case R600::INTERP_PAIR_XY: |
40 | case R600::INTERP_PAIR_ZW: |
41 | case R600::INTERP_VEC_LOAD: |
42 | case R600::DOT_4: |
43 | return 4; |
44 | case R600::KILL: |
45 | return 0; |
46 | default: |
47 | break; |
48 | } |
49 | |
50 | // These will be expanded to two ALU instructions in the |
51 | // ExpandSpecialInstructions pass. |
52 | if (TII->isLDSRetInstr(Opcode: MI.getOpcode())) |
53 | return 2; |
54 | |
55 | if (TII->isVector(MI) || TII->isCubeOp(opcode: MI.getOpcode()) || |
56 | TII->isReductionOp(opcode: MI.getOpcode())) |
57 | return 4; |
58 | |
59 | unsigned NumLiteral = 0; |
60 | for (MachineInstr::mop_iterator It = MI.operands_begin(), |
61 | E = MI.operands_end(); |
62 | It != E; ++It) { |
63 | MachineOperand &MO = *It; |
64 | if (MO.isReg() && MO.getReg() == R600::ALU_LITERAL_X) |
65 | ++NumLiteral; |
66 | } |
67 | return 1 + NumLiteral; |
68 | } |
69 | |
70 | bool isALU(const MachineInstr &MI) const { |
71 | if (TII->isALUInstr(Opcode: MI.getOpcode())) |
72 | return true; |
73 | if (TII->isVector(MI) || TII->isCubeOp(opcode: MI.getOpcode())) |
74 | return true; |
75 | switch (MI.getOpcode()) { |
76 | case R600::PRED_X: |
77 | case R600::INTERP_PAIR_XY: |
78 | case R600::INTERP_PAIR_ZW: |
79 | case R600::INTERP_VEC_LOAD: |
80 | case R600::COPY: |
81 | case R600::DOT_4: |
82 | return true; |
83 | default: |
84 | return false; |
85 | } |
86 | } |
87 | |
88 | bool IsTrivialInst(MachineInstr &MI) const { |
89 | switch (MI.getOpcode()) { |
90 | case R600::KILL: |
91 | case R600::RETURN: |
92 | case R600::IMPLICIT_DEF: |
93 | return true; |
94 | default: |
95 | return false; |
96 | } |
97 | } |
98 | |
99 | std::pair<unsigned, unsigned> getAccessedBankLine(unsigned Sel) const { |
100 | // Sel is (512 + (kc_bank << 12) + ConstIndex) << 2 |
101 | // (See also R600ISelLowering.cpp) |
102 | // ConstIndex value is in [0, 4095]; |
103 | return std::pair<unsigned, unsigned>( |
104 | ((Sel >> 2) - 512) >> 12, // KC_BANK |
105 | // Line Number of ConstIndex |
106 | // A line contains 16 constant registers however KCX bank can lock |
107 | // two line at the same time ; thus we want to get an even line number. |
108 | // Line number can be retrieved with (>>4), using (>>5) <<1 generates |
109 | // an even number. |
110 | ((((Sel >> 2) - 512) & 4095) >> 5) << 1); |
111 | } |
112 | |
113 | bool |
114 | SubstituteKCacheBank(MachineInstr &MI, |
115 | std::vector<std::pair<unsigned, unsigned>> &CachedConsts, |
116 | bool UpdateInstr = true) const { |
117 | std::vector<std::pair<unsigned, unsigned>> UsedKCache; |
118 | |
119 | if (!TII->isALUInstr(Opcode: MI.getOpcode()) && MI.getOpcode() != R600::DOT_4) |
120 | return true; |
121 | |
122 | const SmallVectorImpl<std::pair<MachineOperand *, int64_t>> &Consts = |
123 | TII->getSrcs(MI); |
124 | assert( |
125 | (TII->isALUInstr(MI.getOpcode()) || MI.getOpcode() == R600::DOT_4) && |
126 | "Can't assign Const" ); |
127 | for (unsigned i = 0, n = Consts.size(); i < n; ++i) { |
128 | if (Consts[i].first->getReg() != R600::ALU_CONST) |
129 | continue; |
130 | unsigned Sel = Consts[i].second; |
131 | unsigned Chan = Sel & 3, Index = ((Sel >> 2) - 512) & 31; |
132 | unsigned KCacheIndex = Index * 4 + Chan; |
133 | const std::pair<unsigned, unsigned> &BankLine = getAccessedBankLine(Sel); |
134 | if (CachedConsts.empty()) { |
135 | CachedConsts.push_back(x: BankLine); |
136 | UsedKCache.push_back(x: std::pair<unsigned, unsigned>(0, KCacheIndex)); |
137 | continue; |
138 | } |
139 | if (CachedConsts[0] == BankLine) { |
140 | UsedKCache.push_back(x: std::pair<unsigned, unsigned>(0, KCacheIndex)); |
141 | continue; |
142 | } |
143 | if (CachedConsts.size() == 1) { |
144 | CachedConsts.push_back(x: BankLine); |
145 | UsedKCache.push_back(x: std::pair<unsigned, unsigned>(1, KCacheIndex)); |
146 | continue; |
147 | } |
148 | if (CachedConsts[1] == BankLine) { |
149 | UsedKCache.push_back(x: std::pair<unsigned, unsigned>(1, KCacheIndex)); |
150 | continue; |
151 | } |
152 | return false; |
153 | } |
154 | |
155 | if (!UpdateInstr) |
156 | return true; |
157 | |
158 | for (unsigned i = 0, j = 0, n = Consts.size(); i < n; ++i) { |
159 | if (Consts[i].first->getReg() != R600::ALU_CONST) |
160 | continue; |
161 | switch(UsedKCache[j].first) { |
162 | case 0: |
163 | Consts[i].first->setReg( |
164 | R600::R600_KC0RegClass.getRegister(UsedKCache[j].second)); |
165 | break; |
166 | case 1: |
167 | Consts[i].first->setReg( |
168 | R600::R600_KC1RegClass.getRegister(UsedKCache[j].second)); |
169 | break; |
170 | default: |
171 | llvm_unreachable("Wrong Cache Line" ); |
172 | } |
173 | j++; |
174 | } |
175 | return true; |
176 | } |
177 | |
178 | bool canClauseLocalKillFitInClause( |
179 | unsigned AluInstCount, |
180 | std::vector<std::pair<unsigned, unsigned>> KCacheBanks, |
181 | MachineBasicBlock::iterator Def, |
182 | MachineBasicBlock::iterator BBEnd) { |
183 | const R600RegisterInfo &TRI = TII->getRegisterInfo(); |
184 | //TODO: change this to defs? |
185 | for (MachineInstr::const_mop_iterator |
186 | MOI = Def->operands_begin(), |
187 | MOE = Def->operands_end(); MOI != MOE; ++MOI) { |
188 | if (!MOI->isReg() || !MOI->isDef() || |
189 | TRI.isPhysRegLiveAcrossClauses(Reg: MOI->getReg())) |
190 | continue; |
191 | |
192 | // Def defines a clause local register, so check that its use will fit |
193 | // in the clause. |
194 | unsigned LastUseCount = 0; |
195 | for (MachineBasicBlock::iterator UseI = Def; UseI != BBEnd; ++UseI) { |
196 | AluInstCount += OccupiedDwords(MI&: *UseI); |
197 | // Make sure we won't need to end the clause due to KCache limitations. |
198 | if (!SubstituteKCacheBank(MI&: *UseI, CachedConsts&: KCacheBanks, UpdateInstr: false)) |
199 | return false; |
200 | |
201 | // We have reached the maximum instruction limit before finding the |
202 | // use that kills this register, so we cannot use this def in the |
203 | // current clause. |
204 | if (AluInstCount >= TII->getMaxAlusPerClause()) |
205 | return false; |
206 | |
207 | // TODO: Is this true? kill flag appears to work OK below |
208 | // Register kill flags have been cleared by the time we get to this |
209 | // pass, but it is safe to assume that all uses of this register |
210 | // occur in the same basic block as its definition, because |
211 | // it is illegal for the scheduler to schedule them in |
212 | // different blocks. |
213 | if (UseI->readsRegister(MOI->getReg(), &TRI)) |
214 | LastUseCount = AluInstCount; |
215 | |
216 | // Exit early if the current use kills the register |
217 | if (UseI != Def && UseI->killsRegister(MOI->getReg(), &TRI)) |
218 | break; |
219 | } |
220 | if (LastUseCount) |
221 | return LastUseCount <= TII->getMaxAlusPerClause(); |
222 | llvm_unreachable("Clause local register live at end of clause." ); |
223 | } |
224 | return true; |
225 | } |
226 | |
227 | MachineBasicBlock::iterator |
228 | MakeALUClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator I) { |
229 | MachineBasicBlock::iterator ClauseHead = I; |
230 | std::vector<std::pair<unsigned, unsigned>> KCacheBanks; |
231 | bool PushBeforeModifier = false; |
232 | unsigned AluInstCount = 0; |
233 | for (MachineBasicBlock::iterator E = MBB.end(); I != E; ++I) { |
234 | if (IsTrivialInst(MI&: *I)) |
235 | continue; |
236 | if (!isALU(MI: *I)) |
237 | break; |
238 | if (AluInstCount > TII->getMaxAlusPerClause()) |
239 | break; |
240 | if (I->getOpcode() == R600::PRED_X) { |
241 | // We put PRED_X in its own clause to ensure that ifcvt won't create |
242 | // clauses with more than 128 insts. |
243 | // IfCvt is indeed checking that "then" and "else" branches of an if |
244 | // statement have less than ~60 insts thus converted clauses can't be |
245 | // bigger than ~121 insts (predicate setter needs to be in the same |
246 | // clause as predicated alus). |
247 | if (AluInstCount > 0) |
248 | break; |
249 | if (TII->getFlagOp(MI&: *I).getImm() & MO_FLAG_PUSH) |
250 | PushBeforeModifier = true; |
251 | AluInstCount ++; |
252 | continue; |
253 | } |
254 | // XXX: GROUP_BARRIER instructions cannot be in the same ALU clause as: |
255 | // |
256 | // * KILL or INTERP instructions |
257 | // * Any instruction that sets UPDATE_EXEC_MASK or UPDATE_PRED bits |
258 | // * Uses waterfalling (i.e. INDEX_MODE = AR.X) |
259 | // |
260 | // XXX: These checks have not been implemented yet. |
261 | if (TII->mustBeLastInClause(Opcode: I->getOpcode())) { |
262 | I++; |
263 | break; |
264 | } |
265 | |
266 | // If this instruction defines a clause local register, make sure |
267 | // its use can fit in this clause. |
268 | if (!canClauseLocalKillFitInClause(AluInstCount, KCacheBanks, Def: I, BBEnd: E)) |
269 | break; |
270 | |
271 | if (!SubstituteKCacheBank(MI&: *I, CachedConsts&: KCacheBanks)) |
272 | break; |
273 | AluInstCount += OccupiedDwords(MI&: *I); |
274 | } |
275 | unsigned Opcode = PushBeforeModifier ? |
276 | R600::CF_ALU_PUSH_BEFORE : R600::CF_ALU; |
277 | BuildMI(MBB, ClauseHead, MBB.findDebugLoc(MBBI: ClauseHead), TII->get(Opcode)) |
278 | // We don't use the ADDR field until R600ControlFlowFinalizer pass, where |
279 | // it is safe to assume it is 0. However if we always put 0 here, the ifcvt |
280 | // pass may assume that identical ALU clause starter at the beginning of a |
281 | // true and false branch can be factorized which is not the case. |
282 | .addImm(Address++) // ADDR |
283 | .addImm(KCacheBanks.empty()?0:KCacheBanks[0].first) // KB0 |
284 | .addImm((KCacheBanks.size() < 2)?0:KCacheBanks[1].first) // KB1 |
285 | .addImm(KCacheBanks.empty()?0:2) // KM0 |
286 | .addImm((KCacheBanks.size() < 2)?0:2) // KM1 |
287 | .addImm(KCacheBanks.empty()?0:KCacheBanks[0].second) // KLINE0 |
288 | .addImm((KCacheBanks.size() < 2)?0:KCacheBanks[1].second) // KLINE1 |
289 | .addImm(AluInstCount) // COUNT |
290 | .addImm(1); // Enabled |
291 | return I; |
292 | } |
293 | |
294 | public: |
295 | static char ID; |
296 | |
297 | R600EmitClauseMarkers() : MachineFunctionPass(ID) { |
298 | initializeR600EmitClauseMarkersPass(*PassRegistry::getPassRegistry()); |
299 | } |
300 | |
301 | bool runOnMachineFunction(MachineFunction &MF) override { |
302 | const R600Subtarget &ST = MF.getSubtarget<R600Subtarget>(); |
303 | TII = ST.getInstrInfo(); |
304 | |
305 | for (MachineBasicBlock &MBB : MF) { |
306 | MachineBasicBlock::iterator I = MBB.begin(); |
307 | if (I != MBB.end() && I->getOpcode() == R600::CF_ALU) |
308 | continue; // BB was already parsed |
309 | for (MachineBasicBlock::iterator E = MBB.end(); I != E;) { |
310 | if (isALU(MI: *I)) { |
311 | auto next = MakeALUClause(MBB, I); |
312 | assert(next != I); |
313 | I = next; |
314 | } else |
315 | ++I; |
316 | } |
317 | } |
318 | return false; |
319 | } |
320 | |
321 | StringRef getPassName() const override { |
322 | return "R600 Emit Clause Markers Pass" ; |
323 | } |
324 | }; |
325 | |
326 | char R600EmitClauseMarkers::ID = 0; |
327 | |
328 | } // end anonymous namespace |
329 | |
330 | INITIALIZE_PASS_BEGIN(R600EmitClauseMarkers, "emitclausemarkers" , |
331 | "R600 Emit Clause Markers" , false, false) |
332 | INITIALIZE_PASS_END(R600EmitClauseMarkers, "emitclausemarkers" , |
333 | "R600 Emit Clause Markers" , false, false) |
334 | |
335 | FunctionPass *llvm::createR600EmitClauseMarkers() { |
336 | return new R600EmitClauseMarkers(); |
337 | } |
338 | |