Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions llvm/lib/Target/AMDGPU/AMDGPU.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,13 @@ FunctionPass *createAMDGPUPostLegalizeCombiner(bool IsOptNone);
FunctionPass *createAMDGPURegBankCombiner(bool IsOptNone);
void initializeAMDGPURegBankCombinerPass(PassRegistry &);
FunctionPass *createAMDGPUGlobalISelDivergenceLoweringPass();

class AMDGPUGlobalISelDivergenceLoweringPass
: public RequiredPassInfoMixin<AMDGPUGlobalISelDivergenceLoweringPass> {
public:
PreservedAnalyses run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM);
};
FunctionPass *createAMDGPURegBankSelectPass();
FunctionPass *createAMDGPURegBankLegalizePass();

Expand Down Expand Up @@ -211,8 +218,8 @@ extern char &SILowerWWMCopiesLegacyID;
void initializeSILowerI1CopiesLegacyPass(PassRegistry &);
extern char &SILowerI1CopiesLegacyID;

void initializeAMDGPUGlobalISelDivergenceLoweringPass(PassRegistry &);
extern char &AMDGPUGlobalISelDivergenceLoweringID;
void initializeAMDGPUGlobalISelDivergenceLoweringLegacyPass(PassRegistry &);
extern char &AMDGPUGlobalISelDivergenceLoweringLegacyID;

void initializeAMDGPURegBankSelectPass(PassRegistry &);
extern char &AMDGPURegBankSelectID;
Expand Down
73 changes: 47 additions & 26 deletions llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@
#include "AMDGPUGlobalISelUtils.h"
#include "SILowerI1Copies.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachinePassManager.h"
#include "llvm/CodeGen/MachineUniformityAnalysis.h"
#include "llvm/InitializePasses.h"

Expand All @@ -29,12 +31,12 @@ using namespace llvm;

namespace {

class AMDGPUGlobalISelDivergenceLowering : public MachineFunctionPass {
class AMDGPUGlobalISelDivergenceLoweringLegacy : public MachineFunctionPass {
public:
static char ID;

public:
AMDGPUGlobalISelDivergenceLowering() : MachineFunctionPass(ID) {}
AMDGPUGlobalISelDivergenceLoweringLegacy() : MachineFunctionPass(ID) {}

bool runOnMachineFunction(MachineFunction &MF) override;

Expand Down Expand Up @@ -290,26 +292,50 @@ bool DivergenceLoweringHelper::lowerTemporalDivergenceI1() {
return false;
}

static bool runDivergenceLowering(MachineFunction &MF, MachineDominatorTree &DT,
MachinePostDominatorTree &PDT,
MachineUniformityInfo &MUI) {
DivergenceLoweringHelper Helper(&MF, &DT, &PDT, &MUI);

bool Changed = false;
// Temporal divergence lowering needs to inspect list of instructions used
// outside cycle with divergent exit provided by uniformity analysis. Uniform
// instructions from the list require lowering, no instruction is deleted.
// Thus it needs to be run before lowerPhis that deletes phis that require
// lowering and replaces them with new instructions.

// Non-i1 temporal divergence lowering.
Changed |= Helper.lowerTemporalDivergence();
// This covers both uniform and divergent i1s. Lane masks are in sgpr and need
// to be updated in each iteration.
Changed |= Helper.lowerTemporalDivergenceI1();
// Temporal divergence lowering of divergent i1 phi used outside of the cycle
// could also be handled by lowerPhis but we do it in lowerTempDivergenceI1
// since in some case lowerPhis does unnecessary lane mask merging.
Changed |= Helper.lowerPhis();
return Changed;
}

} // End anonymous namespace.

INITIALIZE_PASS_BEGIN(AMDGPUGlobalISelDivergenceLowering, DEBUG_TYPE,
INITIALIZE_PASS_BEGIN(AMDGPUGlobalISelDivergenceLoweringLegacy, DEBUG_TYPE,
"AMDGPU GlobalISel divergence lowering", false, false)
INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTreeWrapperPass)
INITIALIZE_PASS_DEPENDENCY(MachineUniformityAnalysisPass)
INITIALIZE_PASS_END(AMDGPUGlobalISelDivergenceLowering, DEBUG_TYPE,
INITIALIZE_PASS_END(AMDGPUGlobalISelDivergenceLoweringLegacy, DEBUG_TYPE,
"AMDGPU GlobalISel divergence lowering", false, false)

char AMDGPUGlobalISelDivergenceLowering::ID = 0;
char AMDGPUGlobalISelDivergenceLoweringLegacy::ID = 0;

char &llvm::AMDGPUGlobalISelDivergenceLoweringID =
AMDGPUGlobalISelDivergenceLowering::ID;
char &llvm::AMDGPUGlobalISelDivergenceLoweringLegacyID =
AMDGPUGlobalISelDivergenceLoweringLegacy::ID;

FunctionPass *llvm::createAMDGPUGlobalISelDivergenceLoweringPass() {
return new AMDGPUGlobalISelDivergenceLowering();
return new AMDGPUGlobalISelDivergenceLoweringLegacy();
}

bool AMDGPUGlobalISelDivergenceLowering::runOnMachineFunction(
bool AMDGPUGlobalISelDivergenceLoweringLegacy::runOnMachineFunction(
MachineFunction &MF) {
MachineDominatorTree &DT =
getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
Expand All @@ -318,23 +344,18 @@ bool AMDGPUGlobalISelDivergenceLowering::runOnMachineFunction(
MachineUniformityInfo &MUI =
getAnalysis<MachineUniformityAnalysisPass>().getUniformityInfo();

DivergenceLoweringHelper Helper(&MF, &DT, &PDT, &MUI);
return runDivergenceLowering(MF, DT, PDT, MUI);
}

bool Changed = false;
// Temporal divergence lowering needs to inspect list of instructions used
// outside cycle with divergent exit provided by uniformity analysis. Uniform
// instructions from the list require lowering, no instruction is deleted.
// Thus it needs to be run before lowerPhis that deletes phis that require
// lowering and replaces them with new instructions.
PreservedAnalyses AMDGPUGlobalISelDivergenceLoweringPass::run(
MachineFunction &MF, MachineFunctionAnalysisManager &MFAM) {
MachineDominatorTree &DT = MFAM.getResult<MachineDominatorTreeAnalysis>(MF);
MachinePostDominatorTree &PDT =
MFAM.getResult<MachinePostDominatorTreeAnalysis>(MF);
MachineUniformityInfo &MUI = MFAM.getResult<MachineUniformityAnalysis>(MF);

// Non-i1 temporal divergence lowering.
Changed |= Helper.lowerTemporalDivergence();
// This covers both uniform and divergent i1s. Lane masks are in sgpr and need
// to be updated in each iteration.
Changed |= Helper.lowerTemporalDivergenceI1();
// Temporal divergence lowering of divergent i1 phi used outside of the cycle
// could also be handled by lowerPhis but we do it in lowerTempDivergenceI1
// since in some case lowerPhis does unnecessary lane mask merging.
Changed |= Helper.lowerPhis();
return Changed;
if (!runDivergenceLowering(MF, DT, PDT, MUI))
return PreservedAnalyses::all();

return getMachineFunctionPassPreservedAnalyses().preserveSet<CFGAnalyses>();
}
3 changes: 2 additions & 1 deletion llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,8 @@ MACHINE_FUNCTION_ANALYSIS("amdgpu-next-use-analysis", AMDGPUNextUseAnalysisPass(
#define MACHINE_FUNCTION_PASS(NAME, CREATE_PASS)
#endif
MACHINE_FUNCTION_PASS("amdgpu-asm-printer", AMDGPUAsmPrinterPass())
MACHINE_FUNCTION_PASS("amdgpu-global-isel-divergence-lowering",
AMDGPUGlobalISelDivergenceLoweringPass())
MACHINE_FUNCTION_PASS("amdgpu-insert-delay-alu", AMDGPUInsertDelayAluPass())
MACHINE_FUNCTION_PASS("amdgpu-isel", AMDGPUISelDAGToDAGPass(*this))
MACHINE_FUNCTION_PASS("amdgpu-lower-vgpr-encoding", AMDGPULowerVGPREncodingPass())
Expand Down Expand Up @@ -162,7 +164,6 @@ MACHINE_FUNCTION_PASS("si-wqm", SIWholeQuadModePass())
// Global ISel passes
DUMMY_MACHINE_FUNCTION_PASS("amdgpu-prelegalizer-combiner", AMDGPUPreLegalizerCombinerPass())
DUMMY_MACHINE_FUNCTION_PASS("amdgpu-postlegalizer-combiner", AMDGPUPostLegalizerCombinerPass())
DUMMY_MACHINE_FUNCTION_PASS("amdgpu-global-isel-divergence-lowering", AMDGPUGlobalISelDivergenceLoweringPass())
DUMMY_MACHINE_FUNCTION_PASS("amdgpu-regbankselect", AMDGPURegBankSelectPass())
DUMMY_MACHINE_FUNCTION_PASS("amdgpu-regbanklegalize", AMDGPURegBankLegalizePass())
DUMMY_MACHINE_FUNCTION_PASS("amdgpu-regbank-combiner", AMDGPURegBankCombinerPass())
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -673,7 +673,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeAMDGPUPrepareAGPRAllocLegacyPass(*PR);
initializeGCNDPPCombineLegacyPass(*PR);
initializeSILowerI1CopiesLegacyPass(*PR);
initializeAMDGPUGlobalISelDivergenceLoweringPass(*PR);
initializeAMDGPUGlobalISelDivergenceLoweringLegacyPass(*PR);
initializeAMDGPURegBankSelectPass(*PR);
initializeAMDGPURegBankLegalizePass(*PR);
initializeSILowerWWMCopiesLegacyPass(*PR);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
# RUN: llc -mtriple=amdgpu10.10-mesa-amdpal -run-pass=amdgpu-global-isel-divergence-lowering -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX10 %s
# RUN: llc -mtriple=amdgpu10.10-mesa-amdpal -passes=amdgpu-global-isel-divergence-lowering -verify-each %s -o - | FileCheck -check-prefix=GFX10 %s

--- |
define void @divergent_i1_phi_uniform_branch() {ret void}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
# RUN: llc -mtriple=amdgpu10.10-mesa-amdpal -run-pass=amdgpu-global-isel-divergence-lowering -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX10 %s
# RUN: llc -mtriple=amdgpu10.10-mesa-amdpal -passes=amdgpu-global-isel-divergence-lowering -verify-each %s -o - | FileCheck -check-prefix=GFX10 %s

---
name: divergent_i1_phi_used_outside_loop
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
# RUN: llc -mtriple=amdgpu10.10-mesa-amdpal -run-pass=amdgpu-global-isel-divergence-lowering -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX10 %s
# RUN: llc -mtriple=amdgpu10.10-mesa-amdpal -passes=amdgpu-global-isel-divergence-lowering -verify-each %s -o - | FileCheck -check-prefix=GFX10 %s

---
name: divergent_i1_phi_if_then
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
# RUN: llc -mtriple=amdgpu10.10-mesa-amdpal -run-pass=amdgpu-global-isel-divergence-lowering -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX10 %s
# RUN: llc -mtriple=amdgpu10.10-mesa-amdpal -passes=amdgpu-global-isel-divergence-lowering -verify-each %s -o - | FileCheck -check-prefix=GFX10 %s

---
name: temporal_divergent_i1_phi
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
# RUN: llc -mtriple=amdgpu10.10-mesa-amdpal -run-pass=amdgpu-global-isel-divergence-lowering -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX10 %s
# RUN: llc -mtriple=amdgpu10.10-mesa-amdpal -passes=amdgpu-global-isel-divergence-lowering -verify-each %s -o - | FileCheck -check-prefix=GFX10 %s

---
name: temporal_divergent_i32
Expand Down
Loading