Skip to content

release/23.x: [SLP]Fix narrow-tree gate width and extract miscount #217569

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kartcq wants to merge 1 commit into
base: release/23.x
Choose a base branch
from
Open

release/23.x: [SLP]Fix narrow-tree gate width and extract miscount #217569

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13991,6 +13991,12 @@ unsigned BoUpSLP::getNumVectorInsts() const {
continue;
if (EU.User && EphValues.count(EU.User))
continue;
// Vector-typed scalars are consumed as whole vectors, no extract
// instruction is emitted for them.
if (isVectorizedTy(EU.Scalar->getType()) &&
(!SLPReVec ||
(EU.E.hasState() && EU.E.getOpcode() == Instruction::InsertElement)))
continue;
if (ExternalUsesAsOriginalScalar.contains(EU.Scalar))
continue;
if (!CountedExtracts.insert(EU.Scalar).second)
Expand Down Expand Up @@ -19457,8 +19463,17 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
return Cost;
// The narrow non-profitable tree in loop? Skip, may cause regressions.
constexpr unsigned PartLimit = 2;
const unsigned Sz =
getVectorElementSize(VectorizableTree.front()->Scalars.front());
// Measure the narrowness by the width the tree is vectorized at.
const TreeEntry *RootTE = VectorizableTree.front().get();
unsigned Sz;
if (auto It = MinBWs.find(RootTE); It != MinBWs.end()) {
Sz = It->second.first;
} else {
Sz = std::max<unsigned>(
getVectorElementSize(RootTE->Scalars.front()),
DL->getTypeSizeInBits(
getValueType(RootTE->Scalars.front())->getScalarType()));
}
const unsigned MinVF = getMinVF(Sz);
if (Cost >= -SLPCostThreshold &&
VectorizableTree.front()->Scalars.size() * PartLimit <= MinVF &&
Expand Down
111 changes: 111 additions & 0 deletions llvm/test/Transforms/SLPVectorizer/AArch64/abs-mul-buildvector-in-loop.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -S --passes=slp-vectorizer -mtriple=aarch64-linux-gnu -mcpu=cortex-a57 < %s | FileCheck %s

@lut = external constant [0 x i8]

define void @test(ptr noalias %src, ptr noalias %descriptor, i32 %N) {
; CHECK-LABEL: define void @test(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DESCRIPTOR:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[CMP32:%.*]] = icmp sgt i32 [[N]], 0
; CHECK-NEXT: br i1 [[CMP32]], label %[[FOR_BODY:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK: [[FOR_COND_CLEANUP]]:
; CHECK-NEXT: ret void
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[SRC_ADDR_035:%.*]] = phi ptr [ [[ARRAYIDX:%.*]], %[[FOR_BODY]] ], [ [[SRC]], %[[ENTRY]] ]
; CHECK-NEXT: [[P_034:%.*]] = phi ptr [ [[INCDEC_PTR10:%.*]], %[[FOR_BODY]] ], [ @lut, %[[ENTRY]] ]
; CHECK-NEXT: [[I_033:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[ARRAYIDX]] = getelementptr inbounds nuw i8, ptr [[SRC_ADDR_035]], i64 1
; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[SRC_ADDR_035]], i64 -1
; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_ADDR_035]], i64 17
; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[ARRAYIDX3]], align 1
; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i8, ptr [[SRC_ADDR_035]], i64 -17
; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[ARRAYIDX5]], align 1
; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[P_034]], align 1
; CHECK-NEXT: [[CONV9:%.*]] = zext i8 [[TMP4]] to i32
; CHECK-NEXT: [[INCDEC_PTR10]] = getelementptr inbounds nuw i8, ptr [[P_034]], i64 2
; CHECK-NEXT: [[BIN:%.*]] = getelementptr inbounds nuw i8, ptr [[P_034]], i64 1
; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[BIN]], align 1
; CHECK-NEXT: [[IDX_EXT:%.*]] = zext i8 [[TMP5]] to i64
; CHECK-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DESCRIPTOR]], i64 [[IDX_EXT]]
; CHECK-NEXT: [[CONV4:%.*]] = zext i8 [[TMP2]] to i32
; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP0]] to i32
; CHECK-NEXT: [[CONV6:%.*]] = zext i8 [[TMP3]] to i32
; CHECK-NEXT: [[CONV2:%.*]] = zext i8 [[TMP1]] to i32
; CHECK-NEXT: [[SUB7:%.*]] = sub nsw i32 [[CONV4]], [[CONV6]]
; CHECK-NEXT: [[SUB:%.*]] = sub nsw i32 [[CONV]], [[CONV2]]
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[SUB]], i64 0
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[SUB7]], i64 1
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x i32> poison, i32 [[CONV9]], i64 0
; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP15]], <2 x i32> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP10:%.*]] = mul nsw <2 x i32> [[TMP7]], [[TMP9]]
; CHECK-NEXT: [[TMP11:%.*]] = call <2 x i32> @llvm.abs.v2i32(<2 x i32> [[TMP10]], i1 true)
; CHECK-NEXT: [[TMP8:%.*]] = load <4 x i32>, ptr [[ADD_PTR]], align 4
; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <2 x i32> [[TMP10]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <2 x i32> [[TMP11]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP16]], <4 x i32> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
; CHECK-NEXT: [[TMP13:%.*]] = add nsw <4 x i32> [[TMP8]], [[TMP12]]
; CHECK-NEXT: store <4 x i32> [[TMP13]], ptr [[ADD_PTR]], align 4
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_033]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
;
entry:
%cmp32 = icmp sgt i32 %N, 0
br i1 %cmp32, label %for.body, label %for.cond.cleanup

for.cond.cleanup:
ret void

for.body:
%src.addr.035 = phi ptr [ %arrayidx, %for.body ], [ %src, %entry ]
%p.034 = phi ptr [ %incdec.ptr10, %for.body ], [ @lut, %entry ]
%i.033 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
%arrayidx = getelementptr inbounds nuw i8, ptr %src.addr.035, i64 1
%0 = load i8, ptr %arrayidx, align 1
%conv = zext i8 %0 to i32
%arrayidx1 = getelementptr inbounds i8, ptr %src.addr.035, i64 -1
%1 = load i8, ptr %arrayidx1, align 1
%conv2 = zext i8 %1 to i32
%sub = sub nsw i32 %conv, %conv2
%arrayidx3 = getelementptr inbounds nuw i8, ptr %src.addr.035, i64 17
%2 = load i8, ptr %arrayidx3, align 1
%conv4 = zext i8 %2 to i32
%arrayidx5 = getelementptr inbounds i8, ptr %src.addr.035, i64 -17
%3 = load i8, ptr %arrayidx5, align 1
%conv6 = zext i8 %3 to i32
%sub7 = sub nsw i32 %conv4, %conv6
%4 = load i8, ptr %p.034, align 1
%conv9 = zext i8 %4 to i32
%incdec.ptr10 = getelementptr inbounds nuw i8, ptr %p.034, i64 2
%bin = getelementptr inbounds nuw i8, ptr %p.034, i64 1
%5 = load i8, ptr %bin, align 1
%idx.ext = zext i8 %5 to i64
%add.ptr = getelementptr inbounds nuw [4 x i8], ptr %descriptor, i64 %idx.ext
%mul = mul nsw i32 %sub, %conv9
%mul12 = mul nsw i32 %sub7, %conv9
%6 = load i32, ptr %add.ptr, align 4
%add = add nsw i32 %6, %mul
store i32 %add, ptr %add.ptr, align 4
%arrayidx14 = getelementptr inbounds nuw i8, ptr %add.ptr, i64 4
%7 = load i32, ptr %arrayidx14, align 4
%add15 = add nsw i32 %7, %mul12
store i32 %add15, ptr %arrayidx14, align 4
%8 = tail call i32 @llvm.abs.i32(i32 %mul, i1 true)
%arrayidx16 = getelementptr inbounds nuw i8, ptr %add.ptr, i64 8
%9 = load i32, ptr %arrayidx16, align 4
%add17 = add nsw i32 %9, %8
store i32 %add17, ptr %arrayidx16, align 4
%10 = tail call i32 @llvm.abs.i32(i32 %mul12, i1 true)
%arrayidx18 = getelementptr inbounds nuw i8, ptr %add.ptr, i64 12
%11 = load i32, ptr %arrayidx18, align 4
%add19 = add nsw i32 %11, %10
store i32 %add19, ptr %arrayidx18, align 4
%inc = add nuw nsw i32 %i.033, 1
%exitcond.not = icmp eq i32 %inc, %N
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}

declare i32 @llvm.abs.i32(i32, i1 immarg)
6 changes: 4 additions & 2 deletions llvm/test/Transforms/SLPVectorizer/AArch64/long-non-power-of-2.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ define i1 @test(ptr %arg, ptr %arg1, i64 %arg2, ptr %arg3) {
; CHECK-NEXT: [[BB:.*:]]
; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr i8, ptr [[ARG1]], i64 [[ARG2]]
; CHECK-NEXT: [[GETELEMENTPTR4:%.*]] = getelementptr i8, ptr null, i64 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x ptr> <ptr null, ptr poison>, ptr [[ARG3]], i64 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, <2 x ptr> [[TMP2]], <2 x i64> <i64 -32, i64 -432>
; CHECK-NEXT: [[GETELEMENTPTR5:%.*]] = getelementptr i8, ptr null, i64 -32
; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[ARG3]], i64 -440
; CHECK-NEXT: [[GETELEMENTPTR7:%.*]] = getelementptr i8, ptr [[ARG1]], i64 0
Expand All @@ -28,8 +30,8 @@ define i1 @test(ptr %arg, ptr %arg1, i64 %arg2, ptr %arg3) {
; CHECK-NEXT: [[AND25:%.*]] = and i1 false, [[ICMP24]]
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x ptr> <ptr poison, ptr poison, ptr null, ptr null>, ptr [[ARG]], i64 1
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x ptr> [[TMP0]], ptr [[GETELEMENTPTR7]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x ptr> <ptr poison, ptr poison, ptr null, ptr null>, ptr [[GETELEMENTPTR5]], i64 0
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x ptr> [[TMP2]], ptr [[GETELEMENTPTR8]], i64 1
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x ptr> [[TMP1]], <2 x ptr> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x ptr> <ptr poison, ptr poison, ptr null, ptr null>, <4 x ptr> [[TMP4]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
; CHECK-NEXT: [[TMP20:%.*]] = icmp ult <4 x ptr> [[TMP19]], [[TMP7]]
; CHECK-NEXT: [[ICMP58:%.*]] = icmp ult ptr [[GETELEMENTPTR]], null
; CHECK-NEXT: [[ICMP62:%.*]] = icmp ult ptr [[GETELEMENTPTR]], null
Expand Down
10 changes: 5 additions & 5 deletions llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ define void @test(i32 %arg, i32 %arg1, i64 %arg2) {
; CHECK-NEXT: [[BB:.*]]:
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 0, i32 poison>, i32 [[ARG]], i64 3
; CHECK-NEXT: [[TMP27:%.*]] = insertelement <2 x i32> poison, i32 [[ARG1]], i64 0
; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <2 x i32> [[TMP27]], <2 x i32> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP29:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[ARG]], i64 1
; CHECK-NEXT: br label %[[BB3:.*]]
; CHECK: [[BB3]]:
; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ 0, %[[BB3]] ], [ 0, %[[BB]] ]
Expand All @@ -28,19 +28,19 @@ define void @test(i32 %arg, i32 %arg1, i64 %arg2) {
; CHECK-NEXT: [[OR11:%.*]] = or i32 [[TRUNC27]], 0
; CHECK-NEXT: [[TMP8:%.*]] = or <4 x i32> zeroinitializer, [[TMP2]]
; CHECK-NEXT: [[TMP9:%.*]] = mul <4 x i32> [[TMP5]], [[TMP8]]
; CHECK-NEXT: [[XOR38:%.*]] = xor i32 [[ARG]], [[TRUNC28]]
; CHECK-NEXT: [[TMP29:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[TRUNC19]], i64 0
; CHECK-NEXT: [[TMP28:%.*]] = insertelement <2 x i32> [[TMP27]], i32 [[TRUNC28]], i64 1
; CHECK-NEXT: [[TMP14:%.*]] = xor <2 x i32> [[TMP28]], [[TMP29]]
; CHECK-NEXT: [[XOR31:%.*]] = xor i32 [[ARG1]], [[TRUNC19]]
; CHECK-NEXT: [[SHL:%.*]] = shl i32 0, 1
; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> poison, i32 [[SHL]], i64 0
; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TRUNC10]], i64 1
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TRUNC27]], i64 3
; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <4 x i32> [[TMP13]], <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 3>
; CHECK-NEXT: [[TMP26:%.*]] = xor <4 x i32> [[TMP11]], [[TMP25]]
; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <4 x i32> [[TMP26]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP32:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[XOR31]], i64 5
; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <2 x i32> [[TMP14]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP24]], <8 x i32> [[TMP30]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 8, i32 9, i32 7>
; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x i32> [[TMP15]], i32 [[XOR38]], i64 7
; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <8 x i32> [[TMP32]], <8 x i32> [[TMP30]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <8 x i32> [[TMP16]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <8 x i32> [[TMP17]], <8 x i32> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <4 x i32> [[TMP9]], <4 x i32> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
Expand Down
14 changes: 8 additions & 6 deletions llvm/test/Transforms/SLPVectorizer/X86/multi-parent-instr-copyable-regular.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,21 +9,23 @@ define void @test(ptr %0, ptr %1, ptr %2, ptr %3, double %4, double %5) {
; CHECK-NEXT: [[TMP7:%.*]] = load double, ptr [[TMP1]], align 8
; CHECK-NEXT: [[TMP8:%.*]] = load double, ptr [[TMP0]], align 8
; CHECK-NEXT: [[TMP9:%.*]] = load double, ptr [[TMP2]], align 8
; CHECK-NEXT: [[TMP15:%.*]] = fmul double [[TMP9]], 0.000000e+00
; CHECK-NEXT: [[TMP16:%.*]] = fmul double [[TMP4]], 0.000000e+00
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x double> poison, double [[TMP4]], i64 0
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x double> [[TMP19]], double [[TMP5]], i64 1
; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x double> [[TMP11]], <4 x double> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x double> <double 0.000000e+00, double 0.000000e+00, double 1.000000e+00, double poison>, double [[TMP4]], i64 3
; CHECK-NEXT: [[TMP14:%.*]] = fmul <4 x double> [[TMP12]], [[TMP13]]
; CHECK-NEXT: [[TMP17:%.*]] = fmul double [[TMP15]], [[TMP8]]
; CHECK-NEXT: [[TMP18:%.*]] = fmul double [[TMP9]], [[TMP16]]
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x double> poison, double [[TMP4]], i64 0
; CHECK-NEXT: [[TMP16:%.*]] = insertelement <2 x double> [[TMP15]], double [[TMP9]], i64 1
; CHECK-NEXT: [[TMP17:%.*]] = fmul <2 x double> [[TMP16]], zeroinitializer
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x double> poison, double [[TMP9]], i64 0
; CHECK-NEXT: [[TMP35:%.*]] = insertelement <2 x double> [[TMP18]], double [[TMP8]], i64 1
; CHECK-NEXT: [[TMP20:%.*]] = fmul <2 x double> [[TMP35]], [[TMP17]]
; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[TMP12]], <4 x double> <double 1.000000e+00, double 1.000000e+00, double poison, double poison>, <4 x i32> <i32 4, i32 5, i32 0, i32 0>
; CHECK-NEXT: [[TMP22:%.*]] = fmul <4 x double> [[TMP12]], [[TMP21]]
; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x double> <double 0.000000e+00, double poison, double 1.000000e+00, double 0.000000e+00>, double [[TMP7]], i64 1
; CHECK-NEXT: [[TMP24:%.*]] = fmul <4 x double> [[TMP23]], [[TMP14]]
; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x double> [[TMP24]], double [[TMP18]], i64 0
; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x double> [[TMP25]], double [[TMP17]], i64 2
; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <2 x double> [[TMP20]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 poison>
; CHECK-NEXT: [[TMP26:%.*]] = shufflevector <4 x double> [[TMP24]], <4 x double> [[TMP25]], <4 x i32> <i32 4, i32 1, i32 6, i32 3>
; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <4 x double> [[TMP26]], <4 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
; CHECK-NEXT: [[TMP28:%.*]] = fmul <4 x double> [[TMP27]], zeroinitializer
; CHECK-NEXT: [[TMP29:%.*]] = fmul <4 x double> [[TMP22]], <double 1.000000e+00, double 1.000000e+00, double 0.000000e+00, double 0.000000e+00>
Expand Down
Loading