diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h index 6c7bb56563ab6..feee0989247ad 100644 --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -2605,10 +2605,13 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase { if (!PtrsTy) return InstructionCost::getInvalid(); + FixedVectorType *MaskTy = cast(ICA.getArgTypes()[2]); + Type *MaskEltTy = MaskTy->getScalarType(); + Align Alignment = thisT()->DL.getABITypeAlign(EltTy); InstructionCost Cost = 0; - Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, PtrsTy, - CostKind, 1, nullptr, nullptr); + + // Cost the main load->update->store sequence, for one element. Cost += thisT()->getMemoryOpCost(Instruction::Load, EltTy, Alignment, 0, CostKind); switch (IID) { @@ -2636,7 +2639,26 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase { } Cost += thisT()->getMemoryOpCost(Instruction::Store, EltTy, Alignment, 0, CostKind); + + // Add the cost of a compare + branch for the mask; for a type-only cost + // we cannot check whether the mask is all-true. + Cost += thisT()->getCmpSelInstrCost( + Instruction::ICmp, MaskEltTy, MaskEltTy, CmpInst::ICMP_EQ, CostKind); + Cost += thisT()->getCFInstrCost(Instruction::CondBr, CostKind); + + // Multiply to find the cost for all elements. Cost *= PtrsTy->getNumElements(); + + // Add in the cost of the extracts; the lanes may have different costs. + for (unsigned Lane = 0; Lane < PtrsTy->getNumElements(); ++Lane) { + // Pointer extract. + Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, PtrsTy, + CostKind, Lane, nullptr, nullptr); + // Mask extract. + Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, MaskTy, + CostKind, Lane, nullptr, nullptr); + } + return Cost; } case Intrinsic::get_active_lane_mask: { diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index b452666213738..824288cce51de 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -69,11 +69,6 @@ static cl::opt EnableOrLikeSelectOpt("enable-aarch64-or-like-select", static cl::opt EnableLSRCostOpt("enable-aarch64-lsr-cost-opt", cl::init(true), cl::Hidden); -// A complete guess as to a reasonable cost. -static cl::opt - BaseHistCntCost("aarch64-base-histcnt-cost", cl::init(8), cl::Hidden, - cl::desc("The cost of a histcnt instruction")); - static cl::opt DMBLookaheadThreshold( "dmb-lookahead-threshold", cl::init(10), cl::Hidden, cl::desc("The number of instructions to search for a redundant dmb")); @@ -564,8 +559,10 @@ static bool isUnpackedVectorVT(EVT VecVT) { VecVT.getSizeInBits().getKnownMinValue() < AArch64::SVEBitsPerBlock; } -static InstructionCost getHistogramCost(const AArch64Subtarget *ST, - const IntrinsicCostAttributes &ICA) { +InstructionCost +AArch64TTIImpl::getHistogramCost(const AArch64Subtarget *ST, + const IntrinsicCostAttributes &ICA, + TTI::TargetCostKind CostKind) const { // We need to know at least the number of elements in the vector of buckets // and the size of each element to update. if (ICA.getArgTypes().size() < 2) @@ -575,9 +572,18 @@ static InstructionCost getHistogramCost(const AArch64Subtarget *ST, if (!ST->hasSVE2()) return InstructionCost::getInvalid(); - Type *BucketPtrsTy = ICA.getArgTypes()[0]; // Type of vector of pointers - Type *EltTy = ICA.getArgTypes()[1]; // Type of bucket elements - unsigned TotalHistCnts = 1; + auto *BucketPtrsTy = cast(ICA.getArgTypes()[0]); + Type *EltTy = ICA.getArgTypes()[1]; + + InstructionCost Cost = 0; + /// Get gather/scatter costs. + /// TODO: Find a way to get more info about the pointers, so we can determine + /// whether we can use 32b indices or require full 64b pointers. + Type *DataTy = VectorType::get(EltTy, BucketPtrsTy->getElementCount()); + MemIntrinsicCostAttributes GMICA(Intrinsic::masked_gather, DataTy, Align(1)); + Cost += getGatherScatterOpCost(GMICA, CostKind); + MemIntrinsicCostAttributes SMICA(Intrinsic::masked_scatter, DataTy, Align(1)); + Cost += getGatherScatterOpCost(SMICA, CostKind); unsigned EltSize = EltTy->getScalarSizeInBits(); // Only allow (up to 64b) integers or pointers @@ -594,13 +600,14 @@ static InstructionCost getHistogramCost(const AArch64Subtarget *ST, // HistCnt only supports 32b and 64b element types unsigned LegalEltSize = EltSize <= 32 ? 32 : 64; - if (EC == 2 || (LegalEltSize == 32 && EC == 4)) - return InstructionCost(BaseHistCntCost); - + // HistCnt is the same cost as a vector integer add on 128b implementations, + // but will likely increase on wider vector types. Multiply by vscale as + // a quick estimate. + unsigned VScale = ST->getVScaleForTuning(); unsigned NaturalVectorWidth = AArch64::SVEBitsPerBlock / LegalEltSize; - TotalHistCnts = EC / NaturalVectorWidth; + unsigned TotalHistCnts = EC / NaturalVectorWidth; - return InstructionCost(BaseHistCntCost * TotalHistCnts); + return Cost + VScale * TotalHistCnts; } return InstructionCost::getInvalid(); @@ -620,7 +627,7 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, switch (ICA.getID()) { case Intrinsic::experimental_vector_histogram_add: { - InstructionCost HistCost = getHistogramCost(ST, ICA); + InstructionCost HistCost = getHistogramCost(ST, ICA, CostKind); // If the cost isn't valid, we may still be able to scalarize if (HistCost.isValid()) return HistCost; diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h index cd71b1179bd45..faa51a042fbbd 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -133,6 +133,12 @@ class AArch64TTIImpl final : public BasicTTIImplBase { return 31; } + /// Gets the cost for a histogram operation, consisting of at least one + /// set of gather + histcnt + scatter instructions. + InstructionCost getHistogramCost(const AArch64Subtarget *ST, + const IntrinsicCostAttributes &ICA, + TTI::TargetCostKind CostKind) const; + InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind) const override; diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 53ab77fbd28b1..4d41356dc427b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -2479,11 +2479,9 @@ void VPHistogramRecipe::execute(VPTransformState &State) { InstructionCost VPHistogramRecipe::computeCost(ElementCount VF, VPCostContext &Ctx) const { - // FIXME: Take the gather and scatter into account as well. For now we're - // generating the same cost as the fallback path, but we'll likely - // need to create a new TTI method for determining the cost, including + // FIXME: Improve the TTI method for determining the cost, including // whether we can use base + vec-of-smaller-indices or just - // vec-of-pointers. + // vec-of-pointers for the gather and scatter. assert(VF.isVector() && "Invalid VF for histogram cost"); Type *AddressTy = getOperand(0)->getScalarType(); VPValue *IncAmt = getOperand(1); diff --git a/llvm/test/Analysis/CostModel/AArch64/histograms.ll b/llvm/test/Analysis/CostModel/AArch64/histograms.ll index b3057dea58886..05938c13c7899 100644 --- a/llvm/test/Analysis/CostModel/AArch64/histograms.ll +++ b/llvm/test/Analysis/CostModel/AArch64/histograms.ll @@ -10,37 +10,37 @@ define void @histograms() { ; CHECK-NEON-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32( poison, i32 1, poison) ; CHECK-NEON-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16( poison, i16 1, poison) ; CHECK-NEON-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8( poison, i8 1, poison) -; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) -; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) -; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) -; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) ; CHECK-NEON-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv1p0.i64( poison, i64 1, poison) ; CHECK-NEON-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv2p0.i64( poison, i64 1, poison) ; CHECK-NEON-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv4p0.i32( poison, i32 1, poison) ; CHECK-NEON-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv8p0.i16( poison, i16 1, poison) ; CHECK-NEON-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv16p0.i8( poison, i8 1, poison) -; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) -; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) -; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) -; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 88 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 176 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) ; CHECK-NEON-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv1p0.i64( poison, i64 1, poison) ; CHECK-NEON-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv2p0.i64( poison, i64 1, poison) ; CHECK-NEON-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv4p0.i32( poison, i32 1, poison) ; CHECK-NEON-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv8p0.i16( poison, i16 1, poison) ; CHECK-NEON-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv16p0.i8( poison, i8 1, poison) -; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vector.histogram.umax.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) -; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) -; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.experimental.vector.histogram.umax.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) -; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.experimental.vector.histogram.umax.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.experimental.vector.histogram.umax.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 72 for instruction: call void @llvm.experimental.vector.histogram.umax.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 144 for instruction: call void @llvm.experimental.vector.histogram.umax.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) ; CHECK-NEON-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv1p0.i64( poison, i64 1, poison) ; CHECK-NEON-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv2p0.i64( poison, i64 1, poison) ; CHECK-NEON-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv4p0.i32( poison, i32 1, poison) ; CHECK-NEON-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv8p0.i16( poison, i16 1, poison) ; CHECK-NEON-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv16p0.i8( poison, i8 1, poison) -; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vector.histogram.umin.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) -; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) -; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.experimental.vector.histogram.umin.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) -; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.experimental.vector.histogram.umin.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.experimental.vector.histogram.umin.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 72 for instruction: call void @llvm.experimental.vector.histogram.umin.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 144 for instruction: call void @llvm.experimental.vector.histogram.umin.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) ; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; CHECK-SVE-LABEL: 'histograms' @@ -49,76 +49,76 @@ define void @histograms() { ; CHECK-SVE-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32( poison, i32 1, poison) ; CHECK-SVE-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16( poison, i16 1, poison) ; CHECK-SVE-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8( poison, i8 1, poison) -; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) -; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) -; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) -; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) ; CHECK-SVE-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv1p0.i64( poison, i64 1, poison) ; CHECK-SVE-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv2p0.i64( poison, i64 1, poison) ; CHECK-SVE-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv4p0.i32( poison, i32 1, poison) ; CHECK-SVE-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv8p0.i16( poison, i16 1, poison) ; CHECK-SVE-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv16p0.i8( poison, i8 1, poison) -; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) -; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) -; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) -; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 88 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 176 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) ; CHECK-SVE-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv1p0.i64( poison, i64 1, poison) ; CHECK-SVE-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv2p0.i64( poison, i64 1, poison) ; CHECK-SVE-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv4p0.i32( poison, i32 1, poison) ; CHECK-SVE-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv8p0.i16( poison, i16 1, poison) ; CHECK-SVE-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv16p0.i8( poison, i8 1, poison) -; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vector.histogram.umax.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) -; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) -; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.experimental.vector.histogram.umax.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) -; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.experimental.vector.histogram.umax.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.experimental.vector.histogram.umax.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 72 for instruction: call void @llvm.experimental.vector.histogram.umax.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 144 for instruction: call void @llvm.experimental.vector.histogram.umax.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) ; CHECK-SVE-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv1p0.i64( poison, i64 1, poison) ; CHECK-SVE-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv2p0.i64( poison, i64 1, poison) ; CHECK-SVE-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv4p0.i32( poison, i32 1, poison) ; CHECK-SVE-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv8p0.i16( poison, i16 1, poison) ; CHECK-SVE-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv16p0.i8( poison, i8 1, poison) -; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vector.histogram.umin.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) -; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) -; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.experimental.vector.histogram.umin.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) -; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.experimental.vector.histogram.umin.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.experimental.vector.histogram.umin.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 72 for instruction: call void @llvm.experimental.vector.histogram.umin.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 144 for instruction: call void @llvm.experimental.vector.histogram.umin.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) ; CHECK-SVE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; CHECK-SVE2-LABEL: 'histograms' ; CHECK-SVE2-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.nxv1p0.i64( poison, i64 1, poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64( poison, i64 1, poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32( poison, i32 1, poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16( poison, i16 1, poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8( poison, i8 1, poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 41 for instruction: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64( poison, i64 1, poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 81 for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32( poison, i32 1, poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 162 for instruction: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16( poison, i16 1, poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 324 for instruction: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8( poison, i8 1, poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) ; CHECK-SVE2-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv1p0.i64( poison, i64 1, poison) ; CHECK-SVE2-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv2p0.i64( poison, i64 1, poison) ; CHECK-SVE2-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv4p0.i32( poison, i32 1, poison) ; CHECK-SVE2-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv8p0.i16( poison, i16 1, poison) ; CHECK-SVE2-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv16p0.i8( poison, i8 1, poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 88 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 176 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) ; CHECK-SVE2-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv1p0.i64( poison, i64 1, poison) ; CHECK-SVE2-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv2p0.i64( poison, i64 1, poison) ; CHECK-SVE2-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv4p0.i32( poison, i32 1, poison) ; CHECK-SVE2-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv8p0.i16( poison, i16 1, poison) ; CHECK-SVE2-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv16p0.i8( poison, i8 1, poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vector.histogram.umax.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.experimental.vector.histogram.umax.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.experimental.vector.histogram.umax.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.experimental.vector.histogram.umax.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 72 for instruction: call void @llvm.experimental.vector.histogram.umax.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 144 for instruction: call void @llvm.experimental.vector.histogram.umax.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) ; CHECK-SVE2-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv1p0.i64( poison, i64 1, poison) ; CHECK-SVE2-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv2p0.i64( poison, i64 1, poison) ; CHECK-SVE2-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv4p0.i32( poison, i32 1, poison) ; CHECK-SVE2-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv8p0.i16( poison, i16 1, poison) ; CHECK-SVE2-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv16p0.i8( poison, i8 1, poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vector.histogram.umin.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.experimental.vector.histogram.umin.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) -; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.experimental.vector.histogram.umin.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.experimental.vector.histogram.umin.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 72 for instruction: call void @llvm.experimental.vector.histogram.umin.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 144 for instruction: call void @llvm.experimental.vector.histogram.umin.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) ; CHECK-SVE2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; call void @llvm.experimental.vector.histogram.add.nxv1p0.i64( poison, i64 1, poison) diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll index aaa19cb4d4904..5649a41a85ddc 100644 --- a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll +++ b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll @@ -1240,15 +1240,15 @@ define void @masked_scatter_v4bf16(<4 x bfloat> %data, <4 x ptr> %ptrs, <4 x i1> define void @histogram_nxv2i64( %buckets, %mask) #3 { ; CHECK-VSCALE-1-LABEL: 'histogram_nxv2i64' -; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 8 for: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64( %buckets, i64 1, %mask) +; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:41 CodeSize:41 Lat:101 SizeLat:41 for: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64( %buckets, i64 1, %mask) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; CHECK-VSCALE-2-LABEL: 'histogram_nxv2i64' -; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 8 for: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64( %buckets, i64 1, %mask) +; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:82 CodeSize:82 Lat:202 SizeLat:82 for: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64( %buckets, i64 1, %mask) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; TYPE_BASED_ONLY-LABEL: 'histogram_nxv2i64' -; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 8 for: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64( %buckets, i64 1, %mask) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:41 CodeSize:41 Lat:101 SizeLat:41 for: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64( %buckets, i64 1, %mask) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; call void @llvm.experimental.vector.histogram.add.nxv2p0.i64( %buckets, i64 1, %mask) @@ -1257,15 +1257,15 @@ define void @histogram_nxv2i64( %buckets, %m define void @histogram_nxv4i32( %buckets, %mask) #3 { ; CHECK-VSCALE-1-LABEL: 'histogram_nxv4i32' -; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 8 for: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32( %buckets, i32 1, %mask) +; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:81 CodeSize:81 Lat:201 SizeLat:81 for: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32( %buckets, i32 1, %mask) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; CHECK-VSCALE-2-LABEL: 'histogram_nxv4i32' -; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 8 for: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32( %buckets, i32 1, %mask) +; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:162 CodeSize:162 Lat:402 SizeLat:162 for: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32( %buckets, i32 1, %mask) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; TYPE_BASED_ONLY-LABEL: 'histogram_nxv4i32' -; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 8 for: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32( %buckets, i32 1, %mask) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:81 CodeSize:81 Lat:201 SizeLat:81 for: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32( %buckets, i32 1, %mask) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; call void @llvm.experimental.vector.histogram.add.nxv4p0.i32( %buckets, i32 1, %mask) @@ -1274,15 +1274,15 @@ define void @histogram_nxv4i32( %buckets, %m define void @histogram_nxv8i16( %buckets, %mask) #3 { ; CHECK-VSCALE-1-LABEL: 'histogram_nxv8i16' -; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 16 for: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16( %buckets, i16 1, %mask) +; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:162 CodeSize:162 Lat:402 SizeLat:162 for: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16( %buckets, i16 1, %mask) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; CHECK-VSCALE-2-LABEL: 'histogram_nxv8i16' -; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 16 for: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16( %buckets, i16 1, %mask) +; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:324 CodeSize:324 Lat:804 SizeLat:324 for: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16( %buckets, i16 1, %mask) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; TYPE_BASED_ONLY-LABEL: 'histogram_nxv8i16' -; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 16 for: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16( %buckets, i16 1, %mask) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:162 CodeSize:162 Lat:402 SizeLat:162 for: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16( %buckets, i16 1, %mask) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; call void @llvm.experimental.vector.histogram.add.nxv8p0.i16( %buckets, i16 1, %mask) @@ -1291,15 +1291,15 @@ define void @histogram_nxv8i16( %buckets, %m define void @histogram_nxv16i8( %buckets, %mask) #3 { ; CHECK-VSCALE-1-LABEL: 'histogram_nxv16i8' -; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 32 for: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8( %buckets, i8 1, %mask) +; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:324 CodeSize:324 Lat:804 SizeLat:324 for: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8( %buckets, i8 1, %mask) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; CHECK-VSCALE-2-LABEL: 'histogram_nxv16i8' -; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 32 for: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8( %buckets, i8 1, %mask) +; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:648 CodeSize:648 Lat:1608 SizeLat:648 for: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8( %buckets, i8 1, %mask) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; TYPE_BASED_ONLY-LABEL: 'histogram_nxv16i8' -; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 32 for: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8( %buckets, i8 1, %mask) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:324 CodeSize:324 Lat:804 SizeLat:324 for: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8( %buckets, i8 1, %mask) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; call void @llvm.experimental.vector.histogram.add.nxv16p0.i64( %buckets, i8 1, %mask) @@ -1308,15 +1308,15 @@ define void @histogram_nxv16i8( %buckets, define void @histogram_v2i64(<2 x ptr> %buckets, <2 x i1> %mask) { ; CHECK-VSCALE-1-LABEL: 'histogram_v2i64' -; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:10 CodeSize:8 Lat:16 SizeLat:10 for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask) +; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:16 CodeSize:15 Lat:24 SizeLat:18 for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; CHECK-VSCALE-2-LABEL: 'histogram_v2i64' -; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:10 CodeSize:8 Lat:16 SizeLat:10 for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask) +; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:16 CodeSize:15 Lat:24 SizeLat:18 for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; TYPE_BASED_ONLY-LABEL: 'histogram_v2i64' -; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:10 CodeSize:8 Lat:16 SizeLat:10 for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:16 CodeSize:15 Lat:24 SizeLat:18 for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask) @@ -1325,15 +1325,15 @@ define void @histogram_v2i64(<2 x ptr> %buckets, <2 x i1> %mask) { define void @histogram_v4i32(<4 x ptr> %buckets, <4 x i1> %mask) { ; CHECK-VSCALE-1-LABEL: 'histogram_v4i32' -; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:20 CodeSize:16 Lat:32 SizeLat:20 for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask) +; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:32 CodeSize:30 Lat:48 SizeLat:36 for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; CHECK-VSCALE-2-LABEL: 'histogram_v4i32' -; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:20 CodeSize:16 Lat:32 SizeLat:20 for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask) +; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:32 CodeSize:30 Lat:48 SizeLat:36 for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; TYPE_BASED_ONLY-LABEL: 'histogram_v4i32' -; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:20 CodeSize:16 Lat:32 SizeLat:20 for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:32 CodeSize:30 Lat:48 SizeLat:36 for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask) @@ -1342,15 +1342,15 @@ define void @histogram_v4i32(<4 x ptr> %buckets, <4 x i1> %mask) { define void @histogram_v8i16(<8 x ptr> %buckets, <8 x i1> %mask) { ; CHECK-VSCALE-1-LABEL: 'histogram_v8i16' -; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:40 CodeSize:32 Lat:64 SizeLat:40 for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask) +; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:64 CodeSize:60 Lat:96 SizeLat:72 for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; CHECK-VSCALE-2-LABEL: 'histogram_v8i16' -; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:32 Lat:64 SizeLat:40 for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask) +; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:64 CodeSize:60 Lat:96 SizeLat:72 for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; TYPE_BASED_ONLY-LABEL: 'histogram_v8i16' -; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:40 CodeSize:32 Lat:64 SizeLat:40 for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:64 CodeSize:60 Lat:96 SizeLat:72 for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask) @@ -1359,15 +1359,15 @@ define void @histogram_v8i16(<8 x ptr> %buckets, <8 x i1> %mask) { define void @histogram_v16i8(<16 x ptr> %buckets, <16 x i1> %mask) { ; CHECK-VSCALE-1-LABEL: 'histogram_v16i8' -; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:80 CodeSize:64 Lat:128 SizeLat:80 for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask) +; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:128 CodeSize:120 Lat:192 SizeLat:144 for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; CHECK-VSCALE-2-LABEL: 'histogram_v16i8' -; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:80 CodeSize:64 Lat:128 SizeLat:80 for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask) +; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:128 CodeSize:120 Lat:192 SizeLat:144 for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; TYPE_BASED_ONLY-LABEL: 'histogram_v16i8' -; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:80 CodeSize:64 Lat:128 SizeLat:80 for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:128 CodeSize:120 Lat:192 SizeLat:144 for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; call void @llvm.experimental.vector.histogram.add.v16p0.i64(<16 x ptr> %buckets, i8 1, <16 x i1> %mask) @@ -1376,15 +1376,15 @@ define void @histogram_v16i8(<16 x ptr> %buckets, <16 x i1> %mask) { define void @histogram_nxv4i64( %buckets, %mask) #3 { ; CHECK-VSCALE-1-LABEL: 'histogram_nxv4i64' -; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 16 for: call void @llvm.experimental.vector.histogram.add.nxv4p0.i64( %buckets, i64 1, %mask) +; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:82 CodeSize:82 Lat:202 SizeLat:82 for: call void @llvm.experimental.vector.histogram.add.nxv4p0.i64( %buckets, i64 1, %mask) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; CHECK-VSCALE-2-LABEL: 'histogram_nxv4i64' -; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 16 for: call void @llvm.experimental.vector.histogram.add.nxv4p0.i64( %buckets, i64 1, %mask) +; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:164 CodeSize:164 Lat:404 SizeLat:164 for: call void @llvm.experimental.vector.histogram.add.nxv4p0.i64( %buckets, i64 1, %mask) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; TYPE_BASED_ONLY-LABEL: 'histogram_nxv4i64' -; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 16 for: call void @llvm.experimental.vector.histogram.add.nxv4p0.i64( %buckets, i64 1, %mask) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:82 CodeSize:82 Lat:202 SizeLat:82 for: call void @llvm.experimental.vector.histogram.add.nxv4p0.i64( %buckets, i64 1, %mask) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; call void @llvm.experimental.vector.histogram.add.nxv4p0.i64( %buckets, i64 1, %mask) diff --git a/llvm/test/Analysis/CostModel/X86/histograms.ll b/llvm/test/Analysis/CostModel/X86/histograms.ll new file mode 100644 index 0000000000000..36fb0e9e91557 --- /dev/null +++ b/llvm/test/Analysis/CostModel/X86/histograms.ll @@ -0,0 +1,106 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=all -mcpu=x86-64 | FileCheck %s --check-prefixes=SSE,SSE2 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=all -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=SSE,SSE42 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=all -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=AVX,AVX2 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=all -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX,AVX512 + +target triple = "x86_64-unknown-linux-gnu" + +define void @histograms() { +; SSE2-LABEL: 'histograms' +; SSE2-NEXT: Cost Model: Found costs of RThru:14 CodeSize:16 Lat:22 SizeLat:16 for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; SSE2-NEXT: Cost Model: Found costs of RThru:28 CodeSize:32 Lat:44 SizeLat:32 for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; SSE2-NEXT: Cost Model: Found costs of RThru:56 CodeSize:64 Lat:88 SizeLat:64 for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; SSE2-NEXT: Cost Model: Found costs of RThru:112 CodeSize:128 Lat:176 SizeLat:128 for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; SSE2-NEXT: Cost Model: Found costs of RThru:16 CodeSize:22 Lat:26 SizeLat:28 for: call void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; SSE2-NEXT: Cost Model: Found costs of RThru:32 CodeSize:44 Lat:52 SizeLat:56 for: call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; SSE2-NEXT: Cost Model: Found costs of RThru:64 CodeSize:88 Lat:104 SizeLat:112 for: call void @llvm.experimental.vector.histogram.uadd.sat.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; SSE2-NEXT: Cost Model: Found costs of RThru:144 CodeSize:192 Lat:208 SizeLat:240 for: call void @llvm.experimental.vector.histogram.uadd.sat.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; SSE2-NEXT: Cost Model: Found costs of RThru:14 CodeSize:18 Lat:26 SizeLat:20 for: call void @llvm.experimental.vector.histogram.umax.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; SSE2-NEXT: Cost Model: Found costs of RThru:28 CodeSize:36 Lat:48 SizeLat:40 for: call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; SSE2-NEXT: Cost Model: Found costs of RThru:56 CodeSize:72 Lat:112 SizeLat:88 for: call void @llvm.experimental.vector.histogram.umax.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; SSE2-NEXT: Cost Model: Found costs of RThru:112 CodeSize:144 Lat:224 SizeLat:176 for: call void @llvm.experimental.vector.histogram.umax.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; SSE2-NEXT: Cost Model: Found costs of RThru:14 CodeSize:18 Lat:26 SizeLat:20 for: call void @llvm.experimental.vector.histogram.umin.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; SSE2-NEXT: Cost Model: Found costs of RThru:28 CodeSize:36 Lat:48 SizeLat:40 for: call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; SSE2-NEXT: Cost Model: Found costs of RThru:56 CodeSize:72 Lat:112 SizeLat:88 for: call void @llvm.experimental.vector.histogram.umin.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; SSE2-NEXT: Cost Model: Found costs of RThru:112 CodeSize:144 Lat:224 SizeLat:176 for: call void @llvm.experimental.vector.histogram.umin.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; SSE2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void +; +; SSE42-LABEL: 'histograms' +; SSE42-NEXT: Cost Model: Found costs of RThru:12 CodeSize:14 Lat:20 SizeLat:14 for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; SSE42-NEXT: Cost Model: Found costs of RThru:24 CodeSize:28 Lat:40 SizeLat:28 for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; SSE42-NEXT: Cost Model: Found costs of RThru:48 CodeSize:56 Lat:80 SizeLat:56 for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; SSE42-NEXT: Cost Model: Found costs of RThru:96 CodeSize:112 Lat:160 SizeLat:112 for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; SSE42-NEXT: Cost Model: Found costs of RThru:14 CodeSize:20 Lat:24 SizeLat:26 for: call void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; SSE42-NEXT: Cost Model: Found costs of RThru:28 CodeSize:40 Lat:48 SizeLat:52 for: call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; SSE42-NEXT: Cost Model: Found costs of RThru:56 CodeSize:80 Lat:96 SizeLat:104 for: call void @llvm.experimental.vector.histogram.uadd.sat.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; SSE42-NEXT: Cost Model: Found costs of RThru:128 CodeSize:176 Lat:192 SizeLat:224 for: call void @llvm.experimental.vector.histogram.uadd.sat.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; SSE42-NEXT: Cost Model: Found costs of RThru:12 CodeSize:16 Lat:24 SizeLat:18 for: call void @llvm.experimental.vector.histogram.umax.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; SSE42-NEXT: Cost Model: Found costs of RThru:24 CodeSize:32 Lat:44 SizeLat:36 for: call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; SSE42-NEXT: Cost Model: Found costs of RThru:48 CodeSize:64 Lat:104 SizeLat:80 for: call void @llvm.experimental.vector.histogram.umax.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; SSE42-NEXT: Cost Model: Found costs of RThru:96 CodeSize:128 Lat:208 SizeLat:160 for: call void @llvm.experimental.vector.histogram.umax.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; SSE42-NEXT: Cost Model: Found costs of RThru:12 CodeSize:16 Lat:24 SizeLat:18 for: call void @llvm.experimental.vector.histogram.umin.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; SSE42-NEXT: Cost Model: Found costs of RThru:24 CodeSize:32 Lat:44 SizeLat:36 for: call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; SSE42-NEXT: Cost Model: Found costs of RThru:48 CodeSize:64 Lat:104 SizeLat:80 for: call void @llvm.experimental.vector.histogram.umin.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; SSE42-NEXT: Cost Model: Found costs of RThru:96 CodeSize:128 Lat:208 SizeLat:160 for: call void @llvm.experimental.vector.histogram.umin.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; SSE42-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void +; +; AVX2-LABEL: 'histograms' +; AVX2-NEXT: Cost Model: Found costs of RThru:12 CodeSize:14 Lat:20 SizeLat:14 for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; AVX2-NEXT: Cost Model: Found costs of RThru:26 CodeSize:30 Lat:42 SizeLat:30 for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; AVX2-NEXT: Cost Model: Found costs of RThru:52 CodeSize:60 Lat:84 SizeLat:60 for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; AVX2-NEXT: Cost Model: Found costs of RThru:104 CodeSize:120 Lat:168 SizeLat:120 for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; AVX2-NEXT: Cost Model: Found costs of RThru:14 CodeSize:20 Lat:24 SizeLat:26 for: call void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; AVX2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:42 Lat:50 SizeLat:54 for: call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; AVX2-NEXT: Cost Model: Found costs of RThru:60 CodeSize:84 Lat:100 SizeLat:108 for: call void @llvm.experimental.vector.histogram.uadd.sat.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; AVX2-NEXT: Cost Model: Found costs of RThru:136 CodeSize:184 Lat:200 SizeLat:232 for: call void @llvm.experimental.vector.histogram.uadd.sat.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; AVX2-NEXT: Cost Model: Found costs of RThru:12 CodeSize:16 Lat:24 SizeLat:18 for: call void @llvm.experimental.vector.histogram.umax.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; AVX2-NEXT: Cost Model: Found costs of RThru:26 CodeSize:34 Lat:46 SizeLat:38 for: call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; AVX2-NEXT: Cost Model: Found costs of RThru:52 CodeSize:68 Lat:108 SizeLat:84 for: call void @llvm.experimental.vector.histogram.umax.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; AVX2-NEXT: Cost Model: Found costs of RThru:104 CodeSize:136 Lat:216 SizeLat:168 for: call void @llvm.experimental.vector.histogram.umax.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; AVX2-NEXT: Cost Model: Found costs of RThru:12 CodeSize:16 Lat:24 SizeLat:18 for: call void @llvm.experimental.vector.histogram.umin.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; AVX2-NEXT: Cost Model: Found costs of RThru:26 CodeSize:34 Lat:46 SizeLat:38 for: call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; AVX2-NEXT: Cost Model: Found costs of RThru:52 CodeSize:68 Lat:108 SizeLat:84 for: call void @llvm.experimental.vector.histogram.umin.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; AVX2-NEXT: Cost Model: Found costs of RThru:104 CodeSize:136 Lat:216 SizeLat:168 for: call void @llvm.experimental.vector.histogram.umin.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void +; +; AVX512-LABEL: 'histograms' +; AVX512-NEXT: Cost Model: Found costs of RThru:12 CodeSize:14 Lat:20 SizeLat:14 for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; AVX512-NEXT: Cost Model: Found costs of RThru:26 CodeSize:30 Lat:42 SizeLat:30 for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; AVX512-NEXT: Cost Model: Found costs of RThru:54 CodeSize:62 Lat:86 SizeLat:62 for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; AVX512-NEXT: Cost Model: Found costs of RThru:108 CodeSize:124 Lat:172 SizeLat:124 for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; AVX512-NEXT: Cost Model: Found costs of RThru:14 CodeSize:20 Lat:24 SizeLat:26 for: call void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; AVX512-NEXT: Cost Model: Found costs of RThru:30 CodeSize:42 Lat:50 SizeLat:54 for: call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; AVX512-NEXT: Cost Model: Found costs of RThru:62 CodeSize:86 Lat:102 SizeLat:110 for: call void @llvm.experimental.vector.histogram.uadd.sat.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; AVX512-NEXT: Cost Model: Found costs of RThru:140 CodeSize:188 Lat:204 SizeLat:236 for: call void @llvm.experimental.vector.histogram.uadd.sat.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; AVX512-NEXT: Cost Model: Found costs of RThru:12 CodeSize:16 Lat:24 SizeLat:18 for: call void @llvm.experimental.vector.histogram.umax.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; AVX512-NEXT: Cost Model: Found costs of RThru:26 CodeSize:34 Lat:46 SizeLat:38 for: call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; AVX512-NEXT: Cost Model: Found costs of RThru:54 CodeSize:70 Lat:110 SizeLat:86 for: call void @llvm.experimental.vector.histogram.umax.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; AVX512-NEXT: Cost Model: Found costs of RThru:108 CodeSize:140 Lat:220 SizeLat:172 for: call void @llvm.experimental.vector.histogram.umax.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; AVX512-NEXT: Cost Model: Found costs of RThru:12 CodeSize:16 Lat:24 SizeLat:18 for: call void @llvm.experimental.vector.histogram.umin.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) +; AVX512-NEXT: Cost Model: Found costs of RThru:26 CodeSize:34 Lat:46 SizeLat:38 for: call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) +; AVX512-NEXT: Cost Model: Found costs of RThru:54 CodeSize:70 Lat:110 SizeLat:86 for: call void @llvm.experimental.vector.histogram.umin.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) +; AVX512-NEXT: Cost Model: Found costs of RThru:108 CodeSize:140 Lat:220 SizeLat:172 for: call void @llvm.experimental.vector.histogram.umin.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) +; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void +; + call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) + call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) + call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) + call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) + call void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) + call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) + call void @llvm.experimental.vector.histogram.uadd.sat.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) + call void @llvm.experimental.vector.histogram.uadd.sat.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) + call void @llvm.experimental.vector.histogram.umax.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) + call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) + call void @llvm.experimental.vector.histogram.umax.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) + call void @llvm.experimental.vector.histogram.umax.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) + call void @llvm.experimental.vector.histogram.umin.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison) + call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison) + call void @llvm.experimental.vector.histogram.umin.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison) + call void @llvm.experimental.vector.histogram.umin.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison) + ret void +} +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; AVX: {{.*}} +; SSE: {{.*}} diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll index 3c1854e7fff25..994f2f6dcbf61 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll @@ -8,10 +8,13 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 % ; CHECK-LABEL: define void @simple_histogram( ; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: iter.check: -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2 +; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]] +; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP2]], 1 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.main.loop.iter.check: -; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2 ; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP3]] ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH1:%.*]] @@ -34,21 +37,23 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 % ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]] ; CHECK: vec.epilog.iter.check: -; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 2 +; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP4]] ; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]] ; CHECK: vec.epilog.ph: ; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ] -; CHECK-NEXT: [[N_MOD_VF2:%.*]] = and i64 [[N]], 1 +; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP9]], 1 +; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], [[TMP10]] ; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]] ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: vec.epilog.vector.body: ; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[FOR_BODY]] ] ; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX4]] -; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <2 x i32>, ptr [[TMP18]], align 4 -; CHECK-NEXT: [[TMP16:%.*]] = zext <2 x i32> [[WIDE_LOAD5]] to <2 x i64> -; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <2 x i64> [[TMP16]] -; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.v2p0.i32(<2 x ptr> [[TMP13]], i32 1, <2 x i1> splat (i1 true)) -; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], 2 +; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load , ptr [[TMP18]], align 4 +; CHECK-NEXT: [[TMP12:%.*]] = zext [[WIDE_LOAD5]] to +; CHECK-NEXT: [[WIDE_GEP6:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], [[TMP12]] +; CHECK-NEXT: call void @llvm.experimental.vector.histogram.add.nxv2p0.i32( [[WIDE_GEP6]], i32 1, splat (i1 true)) +; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], [[TMP10]] ; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]] ; CHECK-NEXT: br i1 [[TMP21]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: vec.epilog.middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/X86/histograms.ll b/llvm/test/Transforms/LoopVectorize/X86/histograms.ll new file mode 100644 index 0000000000000..ced8718dbcfd8 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/X86/histograms.ll @@ -0,0 +1,71 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6 +; RUN: opt < %s -mcpu=x86-64 -passes=loop-vectorize -enable-histogram-loop-vectorization -S | FileCheck %s --check-prefixes=SSE,SSE2 +; RUN: opt < %s -mcpu=x86-64-v2 -passes=loop-vectorize -enable-histogram-loop-vectorization -S | FileCheck %s --check-prefixes=SSE,SSE42 +; RUN: opt < %s -mcpu=x86-64-v3 -passes=loop-vectorize -enable-histogram-loop-vectorization -S | FileCheck %s --check-prefixes=AVX,AVX2 +; RUN: opt < %s -mcpu=x86-64-v4 -passes=loop-vectorize -enable-histogram-loop-vectorization -S | FileCheck %s --check-prefixes=AVX,AVX512 + +target triple = "x86_64-unknown-linux-gnu" + +define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices) { +; SSE-LABEL: define void @simple_histogram( +; SSE-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]]) #[[ATTR0:[0-9]+]] { +; SSE-NEXT: [[ENTRY:.*]]: +; SSE-NEXT: br label %[[LOOP:.*]] +; SSE: [[LOOP]]: +; SSE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; SSE-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]] +; SSE-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4 +; SSE-NEXT: [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64 +; SSE-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]] +; SSE-NEXT: [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4 +; SSE-NEXT: [[INC:%.*]] = add nsw i32 [[L_BUCKET]], 1 +; SSE-NEXT: store i32 [[INC]], ptr [[GEP_BUCKET]], align 4 +; SSE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; SSE-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000 +; SSE-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]] +; SSE: [[EXIT]]: +; SSE-NEXT: ret void +; +; AVX-LABEL: define void @simple_histogram( +; AVX-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]]) #[[ATTR0:[0-9]+]] { +; AVX-NEXT: [[ENTRY:.*]]: +; AVX-NEXT: br label %[[LOOP:.*]] +; AVX: [[LOOP]]: +; AVX-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; AVX-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]] +; AVX-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4 +; AVX-NEXT: [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64 +; AVX-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]] +; AVX-NEXT: [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4 +; AVX-NEXT: [[INC:%.*]] = add nsw i32 [[L_BUCKET]], 1 +; AVX-NEXT: store i32 [[INC]], ptr [[GEP_BUCKET]], align 4 +; AVX-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; AVX-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000 +; AVX-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]] +; AVX: [[EXIT]]: +; AVX-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv + %l.idx = load i32, ptr %gep.indices, align 4 + %idxprom1 = zext i32 %l.idx to i64 + %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1 + %l.bucket = load i32, ptr %gep.bucket, align 4 + %inc = add nsw i32 %l.bucket, 1 + store i32 %inc, ptr %gep.bucket, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, 1000 + br i1 %exitcond, label %exit, label %loop + +exit: + ret void +} +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; AVX2: {{.*}} +; AVX512: {{.*}} +; SSE2: {{.*}} +; SSE42: {{.*}}