1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
|
//===- AMDGPUReleaseVGPRs.cpp - Automatically release vgprs on GFX11+ -----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
/// \file
/// Insert S_SENDMSG instructions to release vgprs on GFX11+.
//
//===----------------------------------------------------------------------===//
#include "AMDGPU.h"
#include "AMDGPUSubtarget.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIDefines.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineOperand.h"
using namespace llvm;
#define DEBUG_TYPE "release-vgprs"
namespace {
class AMDGPUReleaseVGPRs : public MachineFunctionPass {
public:
static char ID;
const SIInstrInfo *SII;
const SIRegisterInfo *TRI;
AMDGPUReleaseVGPRs() : MachineFunctionPass(ID) {}
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesAll();
MachineFunctionPass::getAnalysisUsage(AU);
}
// Used to cache the result of isLastInstructionVMEMStore for each block
using BlockVMEMStoreType = DenseMap<MachineBasicBlock *, bool>;
BlockVMEMStoreType BlockVMEMStore;
// Return true if the last instruction referencing a vgpr in this MBB
// is a VMEM store, otherwise return false.
// Visit previous basic blocks to find this last instruction if needed.
// Because this pass is late in the pipeline, it is expected that the
// last vgpr use will likely be one of vmem store, ds, exp.
// Loads and others vgpr operations would have been
// deleted by this point, except for complex control flow involving loops.
// This is why we are just testing the type of instructions rather
// than the operands.
bool isLastVGPRUseVMEMStore(MachineBasicBlock &MBB) {
// Use the cache to break infinite loop and save some time. Initialize to
// false in case we have a cycle.
BlockVMEMStoreType::iterator It;
bool Inserted;
std::tie(It, Inserted) = BlockVMEMStore.insert({&MBB, false});
bool &CacheEntry = It->second;
if (!Inserted)
return CacheEntry;
for (auto &MI : reverse(MBB.instrs())) {
// If it's a VMEM store, a vgpr will be used, return true.
if ((SIInstrInfo::isVMEM(MI) || SIInstrInfo::isFLAT(MI)) && MI.mayStore())
return CacheEntry = true;
// If it's referencing a VGPR but is not a VMEM store, return false.
if (SIInstrInfo::isDS(MI) || SIInstrInfo::isEXP(MI) ||
SIInstrInfo::isVMEM(MI) || SIInstrInfo::isFLAT(MI) ||
SIInstrInfo::isVALU(MI))
return CacheEntry = false;
}
// Recursive call into parent blocks. Look into predecessors if there is no
// vgpr used in this block.
return CacheEntry = llvm::any_of(MBB.predecessors(),
[this](MachineBasicBlock *Parent) {
return isLastVGPRUseVMEMStore(*Parent);
});
}
bool runOnMachineBasicBlock(MachineBasicBlock &MBB) {
bool Changed = false;
for (auto &MI : MBB.terminators()) {
// Look for S_ENDPGM instructions
if (MI.getOpcode() == AMDGPU::S_ENDPGM ||
MI.getOpcode() == AMDGPU::S_ENDPGM_SAVED) {
// If the last instruction using a VGPR in the block is a VMEM store,
// release VGPRs. The VGPRs release will be placed just before ending
// the program
if (isLastVGPRUseVMEMStore(MBB)) {
BuildMI(MBB, MI, DebugLoc(), SII->get(AMDGPU::S_SENDMSG))
.addImm(AMDGPU::SendMsg::ID_DEALLOC_VGPRS_GFX11Plus);
Changed = true;
}
}
}
return Changed;
}
bool runOnMachineFunction(MachineFunction &MF) override {
Function &F = MF.getFunction();
if (skipFunction(F) || !AMDGPU::isEntryFunctionCC(F.getCallingConv()))
return false;
// This pass only runs on GFX11+
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
if (ST.getGeneration() < AMDGPUSubtarget::GFX11)
return false;
LLVM_DEBUG(dbgs() << "AMDGPUReleaseVGPRs running on " << MF.getName()
<< "\n");
SII = ST.getInstrInfo();
TRI = ST.getRegisterInfo();
bool Changed = false;
for (auto &MBB : MF) {
Changed |= runOnMachineBasicBlock(MBB);
}
BlockVMEMStore.clear();
return Changed;
}
};
} // namespace
char AMDGPUReleaseVGPRs::ID = 0;
char &llvm::AMDGPUReleaseVGPRsID = AMDGPUReleaseVGPRs::ID;
INITIALIZE_PASS(AMDGPUReleaseVGPRs, DEBUG_TYPE, "Release VGPRs", false, false)
|
