diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index d32a463a996c4..e9af2dd32fdba 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -9480,6 +9480,8 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { VPlanTransforms::addActiveLaneMask(*Plan, ForControlFlow, WithoutRuntimeCheck); } + + assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid"); return Plan; } @@ -9553,6 +9555,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion(); VPBasicBlock *Header = VectorLoopRegion->getEntryBasicBlock(); VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock(); + SmallVector ToDelete; + for (VPRecipeBase &R : Header->phis()) { auto *PhiR = dyn_cast(&R); if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered())) @@ -9672,10 +9676,11 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( CM.useOrderedReductions(RdxDesc), CurrentLinkI->getDebugLoc()); // Append the recipe to the end of the VPBasicBlock because we need to // ensure that it comes after all of it's inputs, including CondOp. - // Note that this transformation may leave over dead recipes (including - // CurrentLink), which will be cleaned by a later VPlan transform. + // Delete CurrentLink as it will be invalid if its operand is replaced + // with a reduction defined at the bottom of the block in the next link. LinkVPBB->appendRecipe(RedRecipe); CurrentLink->replaceAllUsesWith(RedRecipe); + ToDelete.push_back(CurrentLink); PreviousLink = RedRecipe; } } @@ -9790,6 +9795,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( Cmp = Builder.createNot(Cmp); VPValue *Or = Builder.createOr(PhiR, Cmp); Select->getVPSingleValue()->replaceAllUsesWith(Or); + // Delete Select now that it has invalid types. + ToDelete.push_back(Select); // Convert the reduction phi to operate on bools. PhiR->setOperand(0, Plan->getOrAddLiveIn(ConstantInt::getFalse( @@ -9807,6 +9814,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( } VPlanTransforms::clearReductionWrapFlags(*Plan); + for (VPRecipeBase *R : ToDelete) + R->eraseFromParent(); } void VPDerivedIVRecipe::execute(VPTransformState &State) {