1//===-- GCNHazardRecognizers.h - GCN Hazard Recognizers ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines hazard recognizers for scheduling on GCN processors.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
14#define LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
15
16#include "llvm/ADT/BitVector.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/CodeGen/ScheduleHazardRecognizer.h"
19#include "llvm/CodeGen/TargetSchedule.h"
20#include <list>
21
22namespace llvm {
23
24class MachineFunction;
25class MachineInstr;
26class MachineOperand;
27class MachineRegisterInfo;
28class SIInstrInfo;
29class SIRegisterInfo;
30class GCNSubtarget;
31
32class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
33public:
34 typedef function_ref<bool(const MachineInstr &)> IsHazardFn;
35
36private:
37 // Distinguish if we are called from scheduler or hazard recognizer
38 bool IsHazardRecognizerMode;
39
40 // This variable stores the instruction that has been emitted this cycle. It
41 // will be added to EmittedInstrs, when AdvanceCycle() or RecedeCycle() is
42 // called.
43 MachineInstr *CurrCycleInstr;
44 std::list<MachineInstr*> EmittedInstrs;
45 const MachineFunction &MF;
46 const GCNSubtarget &ST;
47 const SIInstrInfo &TII;
48 const SIRegisterInfo &TRI;
49 TargetSchedModel TSchedModel;
50 bool RunLdsBranchVmemWARHazardFixup;
51
52 /// RegUnits of uses in the current soft memory clause.
53 BitVector ClauseUses;
54
55 /// RegUnits of defs in the current soft memory clause.
56 BitVector ClauseDefs;
57
58 void resetClause() {
59 ClauseUses.reset();
60 ClauseDefs.reset();
61 }
62
63 void addClauseInst(const MachineInstr &MI);
64
65 /// \returns the number of wait states before another MFMA instruction can be
66 /// issued after \p MI.
67 unsigned getMFMAPipelineWaitStates(const MachineInstr &MI) const;
68
69 // Advance over a MachineInstr bundle. Look for hazards in the bundled
70 // instructions.
71 void processBundle();
72
73 // Run on an individual instruction in hazard recognizer mode. This can be
74 // used on a newly inserted instruction before returning from PreEmitNoops.
75 void runOnInstruction(MachineInstr *MI);
76
77 int getWaitStatesSince(IsHazardFn IsHazard, int Limit);
78 int getWaitStatesSinceDef(unsigned Reg, IsHazardFn IsHazardDef, int Limit);
79 int getWaitStatesSinceSetReg(IsHazardFn IsHazard, int Limit);
80
81 int checkSoftClauseHazards(MachineInstr *SMEM);
82 int checkSMRDHazards(MachineInstr *SMRD);
83 int checkVMEMHazards(MachineInstr* VMEM);
84 int checkDPPHazards(MachineInstr *DPP);
85 int checkDivFMasHazards(MachineInstr *DivFMas);
86 int checkGetRegHazards(MachineInstr *GetRegInstr);
87 int checkSetRegHazards(MachineInstr *SetRegInstr);
88 int createsVALUHazard(const MachineInstr &MI);
89 int checkVALUHazards(MachineInstr *VALU);
90 int checkVALUHazardsHelper(const MachineOperand &Def, const MachineRegisterInfo &MRI);
91 int checkRWLaneHazards(MachineInstr *RWLane);
92 int checkRFEHazards(MachineInstr *RFE);
93 int checkInlineAsmHazards(MachineInstr *IA);
94 int checkReadM0Hazards(MachineInstr *SMovRel);
95 int checkNSAtoVMEMHazard(MachineInstr *MI);
96 int checkFPAtomicToDenormModeHazard(MachineInstr *MI);
97 void fixHazards(MachineInstr *MI);
98 bool fixVcmpxPermlaneHazards(MachineInstr *MI);
99 bool fixVMEMtoScalarWriteHazards(MachineInstr *MI);
100 bool fixSMEMtoVectorWriteHazards(MachineInstr *MI);
101 bool fixVcmpxExecWARHazard(MachineInstr *MI);
102 bool fixLdsBranchVmemWARHazard(MachineInstr *MI);
103 bool fixLdsDirectVALUHazard(MachineInstr *MI);
104 bool fixLdsDirectVMEMHazard(MachineInstr *MI);
105 bool fixVALUPartialForwardingHazard(MachineInstr *MI);
106 bool fixVALUTransUseHazard(MachineInstr *MI);
107 bool fixWMMAHazards(MachineInstr *MI);
108 bool fixShift64HighRegBug(MachineInstr *MI);
109 bool fixVALUMaskWriteHazard(MachineInstr *MI);
110
111 int checkMAIHazards(MachineInstr *MI);
112 int checkMAIHazards908(MachineInstr *MI);
113 int checkMAIHazards90A(MachineInstr *MI);
114 /// Pad the latency between neighboring MFMA instructions with s_nops. The
115 /// percentage of wait states to fill with s_nops is specified by the command
116 /// line option '-amdgpu-mfma-padding-ratio'.
117 ///
118 /// For example, with '-amdgpu-mfma-padding-ratio=100':
119 ///
120 /// 2 pass MFMA instructions have a latency of 2 wait states. Therefore, a
121 /// 'S_NOP 1' will be added between sequential MFMA instructions.
122 ///
123 /// V_MFMA_F32_4X4X1F32
124 /// V_MFMA_F32_4X4X1F32
125 ///-->
126 /// V_MFMA_F32_4X4X1F32
127 /// S_NOP 1
128 /// V_MFMA_F32_4X4X1F32
129 int checkMFMAPadding(MachineInstr *MI);
130 int checkMAIVALUHazards(MachineInstr *MI);
131 int checkMAILdStHazards(MachineInstr *MI);
132
133public:
134 GCNHazardRecognizer(const MachineFunction &MF);
135 // We can only issue one instruction per cycle.
136 bool atIssueLimit() const override { return true; }
137 void EmitInstruction(SUnit *SU) override;
138 void EmitInstruction(MachineInstr *MI) override;
139 HazardType getHazardType(SUnit *SU, int Stalls) override;
140 void EmitNoop() override;
141 unsigned PreEmitNoops(MachineInstr *) override;
142 unsigned PreEmitNoopsCommon(MachineInstr *);
143 void AdvanceCycle() override;
144 void RecedeCycle() override;
145 bool ShouldPreferAnother(SUnit *SU) override;
146 void Reset() override;
147};
148
149} // end namespace llvm
150
151#endif //LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
152

source code of llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h