diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 1da185f9cfdf4..e696430279d22 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1223,8 +1223,8 @@ class VPInstruction : public VPRecipeWithIRFlags, // operand). Only generates scalar values (either for the first lane only or // for all lanes, depending on its uses). PtrAdd, - // Returns a scalar boolean value, which is true if any lane of its single - // operand is true. + // Returns a scalar boolean value, which is true if any lane of its (only + // boolean) vector operand is true. AnyOf, }; diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp index 8fea2c6fd33b6..27357ff04b5f2 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp @@ -60,7 +60,10 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) { } case Instruction::ICmp: case VPInstruction::ActiveLaneMask: - return inferScalarType(R->getOperand(1)); + assert(inferScalarType(R->getOperand(0)) == + inferScalarType(R->getOperand(1)) && + "different types inferred for different operands"); + return IntegerType::get(Ctx, 1); case VPInstruction::ComputeReductionResult: { auto *PhiR = cast(R->getOperand(0)); auto *OrigPhi = cast(PhiR->getUnderlyingValue()); @@ -71,6 +74,9 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) { case VPInstruction::FirstOrderRecurrenceSplice: case VPInstruction::Not: case VPInstruction::ResumePhi: + case VPInstruction::CalculateTripCountMinusVF: + case VPInstruction::CanonicalIVIncrementForPart: + case VPInstruction::AnyOf: return SetResultTyFromOp(); case VPInstruction::ExtractFromEnd: { Type *BaseTy = inferScalarType(R->getOperand(0)); @@ -79,6 +85,9 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) { return BaseTy; } case VPInstruction::LogicalAnd: + assert(inferScalarType(R->getOperand(0))->isIntegerTy(1) && + inferScalarType(R->getOperand(1))->isIntegerTy(1) && + "LogicalAnd operands should be bool"); return IntegerType::get(Ctx, 1); case VPInstruction::PtrAdd: // Return the type based on the pointer argument (i.e. first operand). diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index be420a873bef5..a30bc82cbde85 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -26,6 +26,7 @@ using namespace llvm; namespace { class VPlanVerifier { const VPDominatorTree &VPDT; + VPTypeAnalysis &TypeInfo; SmallPtrSet WrappedIRBBs; @@ -58,7 +59,8 @@ class VPlanVerifier { bool verifyRegionRec(const VPRegionBlock *Region); public: - VPlanVerifier(VPDominatorTree &VPDT) : VPDT(VPDT) {} + VPlanVerifier(VPDominatorTree &VPDT, VPTypeAnalysis &TypeInfo) + : VPDT(VPDT), TypeInfo(TypeInfo) {} bool verify(const VPlan &Plan); }; @@ -195,6 +197,14 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) { return false; } for (const VPValue *V : R.definedValues()) { + // Verify that we can infer a scalar type for each defined value. With + // assertions enabled, inferScalarType will perform some consistency + // checks during type inference. + if (!TypeInfo.inferScalarType(V)) { + errs() << "Failed to infer scalar type!\n"; + return false; + } + for (const VPUser *U : V->users()) { auto *UI = dyn_cast(U); // TODO: check dominance of incoming values for phis properly. @@ -406,6 +416,8 @@ bool VPlanVerifier::verify(const VPlan &Plan) { bool llvm::verifyVPlanIsValid(const VPlan &Plan) { VPDominatorTree VPDT; VPDT.recalculate(const_cast(Plan)); - VPlanVerifier Verifier(VPDT); + VPTypeAnalysis TypeInfo( + const_cast(Plan).getCanonicalIV()->getScalarType()); + VPlanVerifier Verifier(VPDT, TypeInfo); return Verifier.verify(Plan); } diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp index f098ba0bce497..f818b49fdbe7f 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp @@ -20,14 +20,17 @@ using VPVerifierTest = VPlanTestBase; namespace { TEST_F(VPVerifierTest, VPInstructionUseBeforeDefSameBB) { VPlan &Plan = getPlan(); - VPInstruction *DefI = new VPInstruction(Instruction::Add, {}); + VPValue *Zero = Plan.getOrAddLiveIn(ConstantInt::get(Type::getInt32Ty(C), 0)); + VPInstruction *DefI = new VPInstruction(Instruction::Add, {Zero}); VPInstruction *UseI = new VPInstruction(Instruction::Sub, {DefI}); + auto *CanIV = new VPCanonicalIVPHIRecipe(Zero, {}); VPBasicBlock *VPBB1 = Plan.getEntry(); VPBB1->appendRecipe(UseI); VPBB1->appendRecipe(DefI); VPBasicBlock *VPBB2 = Plan.createVPBasicBlock(""); + VPBB2->appendRecipe(CanIV); VPRegionBlock *R1 = Plan.createVPRegionBlock(VPBB2, VPBB2, "R1"); VPBlockUtils::connectBlocks(VPBB1, R1); VPBlockUtils::connectBlocks(R1, Plan.getScalarHeader()); @@ -44,9 +47,10 @@ TEST_F(VPVerifierTest, VPInstructionUseBeforeDefSameBB) { TEST_F(VPVerifierTest, VPInstructionUseBeforeDefDifferentBB) { VPlan &Plan = getPlan(); - VPInstruction *DefI = new VPInstruction(Instruction::Add, {}); + VPValue *Zero = Plan.getOrAddLiveIn(ConstantInt::get(Type::getInt32Ty(C), 0)); + VPInstruction *DefI = new VPInstruction(Instruction::Add, {Zero}); VPInstruction *UseI = new VPInstruction(Instruction::Sub, {DefI}); - auto *CanIV = new VPCanonicalIVPHIRecipe(UseI, {}); + auto *CanIV = new VPCanonicalIVPHIRecipe(Zero, {}); VPInstruction *BranchOnCond = new VPInstruction(VPInstruction::BranchOnCond, {CanIV}); @@ -73,23 +77,22 @@ TEST_F(VPVerifierTest, VPInstructionUseBeforeDefDifferentBB) { } TEST_F(VPVerifierTest, VPBlendUseBeforeDefDifferentBB) { + VPlan &Plan = getPlan(); IntegerType *Int32 = IntegerType::get(C, 32); auto *Phi = PHINode::Create(Int32, 1); + VPValue *Zero = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 0)); - VPInstruction *I1 = new VPInstruction(Instruction::Add, {}); - VPInstruction *DefI = new VPInstruction(Instruction::Add, {}); - auto *CanIV = new VPCanonicalIVPHIRecipe(I1, {}); + VPInstruction *DefI = new VPInstruction(Instruction::Add, {Zero}); + auto *CanIV = new VPCanonicalIVPHIRecipe(Zero, {}); VPInstruction *BranchOnCond = new VPInstruction(VPInstruction::BranchOnCond, {CanIV}); auto *Blend = new VPBlendRecipe(Phi, {DefI}); - VPlan &Plan = getPlan(); VPBasicBlock *VPBB1 = Plan.getEntry(); VPBasicBlock *VPBB2 = Plan.createVPBasicBlock(""); VPBasicBlock *VPBB3 = Plan.createVPBasicBlock(""); VPBasicBlock *VPBB4 = Plan.createVPBasicBlock(""); - VPBB1->appendRecipe(I1); VPBB2->appendRecipe(CanIV); VPBB3->appendRecipe(Blend); VPBB4->appendRecipe(DefI); @@ -116,14 +119,15 @@ TEST_F(VPVerifierTest, VPBlendUseBeforeDefDifferentBB) { } TEST_F(VPVerifierTest, DuplicateSuccessorsOutsideRegion) { - VPInstruction *I1 = new VPInstruction(Instruction::Add, {}); - auto *CanIV = new VPCanonicalIVPHIRecipe(I1, {}); + VPlan &Plan = getPlan(); + VPValue *Zero = Plan.getOrAddLiveIn(ConstantInt::get(Type::getInt32Ty(C), 0)); + VPInstruction *I1 = new VPInstruction(Instruction::Add, {Zero}); + auto *CanIV = new VPCanonicalIVPHIRecipe(Zero, {}); VPInstruction *BranchOnCond = new VPInstruction(VPInstruction::BranchOnCond, {CanIV}); VPInstruction *BranchOnCond2 = new VPInstruction(VPInstruction::BranchOnCond, {I1}); - VPlan &Plan = getPlan(); VPBasicBlock *VPBB1 = Plan.getEntry(); VPBasicBlock *VPBB2 = Plan.createVPBasicBlock(""); @@ -149,14 +153,15 @@ TEST_F(VPVerifierTest, DuplicateSuccessorsOutsideRegion) { } TEST_F(VPVerifierTest, DuplicateSuccessorsInsideRegion) { - VPInstruction *I1 = new VPInstruction(Instruction::Add, {}); - auto *CanIV = new VPCanonicalIVPHIRecipe(I1, {}); + VPlan &Plan = getPlan(); + VPValue *Zero = Plan.getOrAddLiveIn(ConstantInt::get(Type::getInt32Ty(C), 0)); + VPInstruction *I1 = new VPInstruction(Instruction::Add, {Zero}); + auto *CanIV = new VPCanonicalIVPHIRecipe(Zero, {}); VPInstruction *BranchOnCond = new VPInstruction(VPInstruction::BranchOnCond, {CanIV}); VPInstruction *BranchOnCond2 = new VPInstruction(VPInstruction::BranchOnCond, {I1}); - VPlan &Plan = getPlan(); VPBasicBlock *VPBB1 = Plan.getEntry(); VPBasicBlock *VPBB2 = Plan.createVPBasicBlock(""); VPBasicBlock *VPBB3 = Plan.createVPBasicBlock(""); @@ -186,10 +191,15 @@ TEST_F(VPVerifierTest, DuplicateSuccessorsInsideRegion) { TEST_F(VPVerifierTest, BlockOutsideRegionWithParent) { VPlan &Plan = getPlan(); + VPBasicBlock *VPBB1 = Plan.getEntry(); VPBasicBlock *VPBB2 = Plan.createVPBasicBlock(""); - VPInstruction *DefI = new VPInstruction(Instruction::Add, {}); + VPValue *Zero = Plan.getOrAddLiveIn(ConstantInt::get(Type::getInt32Ty(C), 0)); + auto *CanIV = new VPCanonicalIVPHIRecipe(Zero, {}); + VPBB2->appendRecipe(CanIV); + + VPInstruction *DefI = new VPInstruction(Instruction::Add, {Zero}); VPInstruction *BranchOnCond = new VPInstruction(VPInstruction::BranchOnCond, {DefI});