diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 9e128651598f3..d21981886ad9f 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -5742,36 +5742,36 @@ static const MachineInstrBuilder &AddSubReg(const MachineInstrBuilder &MIB, } static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg, - unsigned NumRegs) { - // We really want the positive remainder mod 32 here, that happens to be + unsigned NumRegs, bool IsPred) { + // We really want the positive remainder mod 16/32 here, that happens to be // easily obtainable with a mask. - return ((DestReg - SrcReg) & 0x1f) < NumRegs; + unsigned MaxRegs = IsPred ? 0xf : 0x1f; + return ((DestReg - SrcReg) & MaxRegs) < NumRegs; } void AArch64InstrInfo::copyPhysRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, - unsigned Opcode, ArrayRef Indices) const { assert(Subtarget.hasNEON() && "Unexpected register copy without NEON"); const TargetRegisterInfo *TRI = &getRegisterInfo(); uint16_t DestEncoding = TRI->getEncodingValue(DestReg); uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg); unsigned NumRegs = Indices.size(); + bool IsPred = AArch64::PPR2RegClass.contains(DestReg); int SubReg = 0, End = NumRegs, Incr = 1; - if (forwardCopyWillClobberTuple(DestEncoding, SrcEncoding, NumRegs)) { + if (forwardCopyWillClobberTuple(DestEncoding, SrcEncoding, NumRegs, IsPred)) { SubReg = NumRegs - 1; End = -1; Incr = -1; } for (; SubReg != End; SubReg += Incr) { - const MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opcode)); - AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI); - AddSubReg(MIB, SrcReg, Indices[SubReg], {}, TRI); - AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI); + MCRegister DestSubReg = TRI->getSubReg(DestReg, Indices[SubReg]); + MCRegister SrcSubReg = TRI->getSubReg(SrcReg, Indices[SubReg]); + copyPhysReg(MBB, I, DL, DestSubReg, SrcSubReg, KillSrc); } } @@ -5991,6 +5991,16 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, return; } + // Copy a predicate register pair by copying the individual sub-registers. + if (AArch64::PPR2RegClass.contains(DestReg) && + AArch64::PPR2RegClass.contains(SrcReg)) { + assert(Subtarget.isSVEorStreamingSVEAvailable() && + "Unexpected SVE predicate register."); + static const unsigned Indices[] = {AArch64::psub0, AArch64::psub1}; + copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices); + return; + } + // Copy a Z register by ORRing with itself. if (AArch64::ZPRRegClass.contains(DestReg) && AArch64::ZPRRegClass.contains(SrcReg)) { @@ -6010,8 +6020,7 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, assert(Subtarget.isSVEorStreamingSVEAvailable() && "Unexpected SVE register."); static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1}; - copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORR_ZZZ, - Indices); + copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices); return; } @@ -6022,8 +6031,7 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, "Unexpected SVE register."); static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1, AArch64::zsub2}; - copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORR_ZZZ, - Indices); + copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices); return; } @@ -6036,8 +6044,7 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, "Unexpected SVE register."); static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1, AArch64::zsub2, AArch64::zsub3}; - copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORR_ZZZ, - Indices); + copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices); return; } @@ -6046,8 +6053,7 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, AArch64::DDDDRegClass.contains(SrcReg)) { static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1, AArch64::dsub2, AArch64::dsub3}; - copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8, - Indices); + copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices); return; } @@ -6056,8 +6062,7 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, AArch64::DDDRegClass.contains(SrcReg)) { static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1, AArch64::dsub2}; - copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8, - Indices); + copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices); return; } @@ -6065,8 +6070,7 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, if (AArch64::DDRegClass.contains(DestReg) && AArch64::DDRegClass.contains(SrcReg)) { static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1}; - copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8, - Indices); + copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices); return; } @@ -6075,8 +6079,7 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, AArch64::QQQQRegClass.contains(SrcReg)) { static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1, AArch64::qsub2, AArch64::qsub3}; - copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8, - Indices); + copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices); return; } @@ -6085,8 +6088,7 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, AArch64::QQQRegClass.contains(SrcReg)) { static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1, AArch64::qsub2}; - copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8, - Indices); + copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices); return; } @@ -6094,8 +6096,7 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, if (AArch64::QQRegClass.contains(DestReg) && AArch64::QQRegClass.contains(SrcReg)) { static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1}; - copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8, - Indices); + copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices); return; } diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h index 15bd832de8d25..821c2f3a7af42 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h @@ -355,7 +355,7 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo { void copyPhysRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, - MCRegister SrcReg, bool KillSrc, unsigned Opcode, + MCRegister SrcReg, bool KillSrc, llvm::ArrayRef Indices) const; void copyGPRRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, diff --git a/llvm/test/CodeGen/AArch64/arm64-copy-tuple.ll b/llvm/test/CodeGen/AArch64/arm64-copy-tuple.ll index 9d71338c9c97a..8d517d47c75a1 100644 --- a/llvm/test/CodeGen/AArch64/arm64-copy-tuple.ll +++ b/llvm/test/CodeGen/AArch64/arm64-copy-tuple.ll @@ -9,8 +9,8 @@ define void @test_D1D2_from_D0D1(ptr %addr) #0 { ; CHECK-LABEL: test_D1D2_from_D0D1: -; CHECK: mov.8b v2, v1 -; CHECK: mov.8b v1, v0 +; CHECK: fmov d2, d1 +; CHECK: fmov d1, d0 entry: %vec = tail call { <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld2.v8i8.p0(ptr %addr) %vec0 = extractvalue { <8 x i8>, <8 x i8> } %vec, 0 @@ -25,8 +25,8 @@ entry: define void @test_D0D1_from_D1D2(ptr %addr) #0 { ; CHECK-LABEL: test_D0D1_from_D1D2: -; CHECK: mov.8b v0, v1 -; CHECK: mov.8b v1, v2 +; CHECK: fmov d0, d1 +; CHECK: fmov d1, d2 entry: %vec = tail call { <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld2.v8i8.p0(ptr %addr) %vec0 = extractvalue { <8 x i8>, <8 x i8> } %vec, 0 @@ -41,8 +41,8 @@ entry: define void @test_D0D1_from_D31D0(ptr %addr) #0 { ; CHECK-LABEL: test_D0D1_from_D31D0: -; CHECK: mov.8b v1, v0 -; CHECK: mov.8b v0, v31 +; CHECK: fmov d1, d0 +; CHECK: fmov d0, d31 entry: %vec = tail call { <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld2.v8i8.p0(ptr %addr) %vec0 = extractvalue { <8 x i8>, <8 x i8> } %vec, 0 @@ -57,8 +57,8 @@ entry: define void @test_D31D0_from_D0D1(ptr %addr) #0 { ; CHECK-LABEL: test_D31D0_from_D0D1: -; CHECK: mov.8b v31, v0 -; CHECK: mov.8b v0, v1 +; CHECK: fmov d31, d0 +; CHECK: fmov d0, d1 entry: %vec = tail call { <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld2.v8i8.p0(ptr %addr) %vec0 = extractvalue { <8 x i8>, <8 x i8> } %vec, 0 @@ -73,9 +73,9 @@ entry: define void @test_D2D3D4_from_D0D1D2(ptr %addr) #0 { ; CHECK-LABEL: test_D2D3D4_from_D0D1D2: -; CHECK: mov.8b v4, v2 -; CHECK: mov.8b v3, v1 -; CHECK: mov.8b v2, v0 +; CHECK: fmov d4, d2 +; CHECK: fmov d3, d1 +; CHECK: fmov d2, d0 entry: %vec = tail call { <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld3.v8i8.p0(ptr %addr) %vec0 = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } %vec, 0 diff --git a/llvm/test/CodeGen/AArch64/sve-copy-pprpair.mir b/llvm/test/CodeGen/AArch64/sve-copy-pprpair.mir new file mode 100644 index 0000000000000..799528d6e3774 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-copy-pprpair.mir @@ -0,0 +1,66 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6 +# RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -run-pass=postrapseudos -simplify-mir -verify-machineinstrs %s -o - | FileCheck %s + +--- +name: copy_ppr2 +alignment: 4 +tracksRegLiveness: true +liveins: + - { reg: '$p0_p1' } +frameInfo: + maxCallFrameSize: 0 +body: | + bb.0: + liveins: $p0_p1 + ; CHECK-LABEL: name: copy_ppr2 + ; CHECK: liveins: $p0_p1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: $p2 = ORR_PPzPP $p0, $p0, killed $p0 + ; CHECK-NEXT: $p3 = ORR_PPzPP $p1, $p1, killed $p1 + ; CHECK-NEXT: RET_ReallyLR + $p2_p3 = COPY killed renamable $p0_p1 + RET_ReallyLR + +... +--- +name: copy_ppr2_overlap +alignment: 4 +tracksRegLiveness: true +liveins: + - { reg: '$p0_p1' } +frameInfo: + maxCallFrameSize: 0 +body: | + bb.0: + liveins: $p0_p1 + ; CHECK-LABEL: name: copy_ppr2_overlap + ; CHECK: liveins: $p0_p1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: $p2 = ORR_PPzPP $p1, $p1, killed $p1 + ; CHECK-NEXT: $p1 = ORR_PPzPP $p0, $p0, killed $p0 + ; CHECK-NEXT: RET_ReallyLR + $p1_p2 = COPY killed renamable $p0_p1 + RET_ReallyLR + +... +--- +name: copy_ppr2_max +alignment: 4 +tracksRegLiveness: true +liveins: + - { reg: '$p15_p0' } +frameInfo: + maxCallFrameSize: 0 +body: | + bb.0: + liveins: $p15_p0 + ; CHECK-LABEL: name: copy_ppr2_max + ; CHECK: liveins: $p15_p0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: $p1 = ORR_PPzPP $p0, $p0, killed $p0 + ; CHECK-NEXT: $p0 = ORR_PPzPP $p15, $p15, killed $p15 + ; CHECK-NEXT: RET_ReallyLR + $p0_p1 = COPY killed renamable $p15_p0 + RET_ReallyLR + +...