Speed-up range operations in vector<bool>

Author: winner245

libcxx/include/__algorithm/copy.h

@@ -13,10 +13,13 @@
 #include <__algorithm/for_each_segment.h>
 #include <__algorithm/min.h>
 #include <__config>
+#include <__fwd/bit_reference.h>
+#include <__iterator/distance.h>
 #include <__iterator/iterator_traits.h>
 #include <__iterator/segmented_iterator.h>
 #include <__type_traits/common_type.h>
 #include <__type_traits/enable_if.h>
+#include <__type_traits/is_convertible.h>
 #include <__utility/move.h>
 #include <__utility/pair.h>
 
@@ -32,6 +35,25 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 template <class _InIter, class _Sent, class _OutIter>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter> __copy(_InIter, _Sent, _OutIter);
 
+template <class _InIter,
+          class _Sent,
+          __enable_if_t<__has_input_iterator_category<_InIter>::value &&
+                            !__has_random_access_iterator_category<_InIter>::value,
+                        int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 typename iterator_traits<_InIter>::difference_type
+__iter_sent_distance(_InIter __first, _Sent __last) {
+  typename iterator_traits<_InIter>::difference_type __r(0);
+  for (; __first != __last; ++__first)
+    ++__r;
+  return __r;
+}
+
+template <class _InIter, class _Sent, __enable_if_t<__has_random_access_iterator_category<_InIter>::value, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 typename iterator_traits<_InIter>::difference_type
+__iter_sent_distance(_InIter __first, _Sent __last) {
+  return static_cast<typename iterator_traits<_InIter>::difference_type>(__last - __first);
+}
+
 struct __copy_impl {
   template <class _InIter, class _Sent, class _OutIter>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
@@ -95,6 +117,64 @@ struct __copy_impl {
     }
   }
 
+  template <class _InIter,
+            class _Sent,
+            class _Cp,
+            __enable_if_t<(__has_forward_iterator_category<_InIter>::value ||
+                           __has_iterator_concept_convertible_to<_InIter, forward_iterator_tag>::value) &&
+                              is_convertible<typename iterator_traits<_InIter>::value_type, bool>::value,
+                          int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, __bit_iterator<_Cp, false> >
+  operator()(_InIter __first, _Sent __last, __bit_iterator<_Cp, false> __result) const {
+    using _It            = __bit_iterator<_Cp, false>;
+    using __storage_type = typename _It::__storage_type;
+#if _LIBCPP_STD_VER >= 20
+    __storage_type __n = static_cast<__storage_type>(std::ranges::distance(__first, __last));
+#else
+    __storage_type __n = static_cast<__storage_type>(std::__iter_sent_distance(__first, __last));
+#endif
+    const unsigned __bits_per_word = _It::__bits_per_word;
+
+    if (__first != __last) {
+      // do first partial word, if present
+      if (__result.__ctz_ != 0) {
+        __storage_type __clz = static_cast<__storage_type>(__bits_per_word - __result.__ctz_);
+        __storage_type __dn  = std::min(__clz, __n);
+        __storage_type __w   = *__result.__seg_;
+        __storage_type __m   = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz - __dn));
+        __w &= ~__m;
+        for (__storage_type __i = 0; __i < __dn; ++__i, ++__first)
+          __w |= static_cast<__storage_type>(*__first) << __result.__ctz_++;
+        *__result.__seg_ = __w;
+        if (__result.__ctz_ == __bits_per_word) {
+          __result.__ctz_ = 0;
+          ++__result.__seg_;
+        }
+        __n -= __dn;
+      }
+    }
+    // do middle whole words, if present
+    __storage_type __nw = __n / __bits_per_word;
+    __n -= __nw * __bits_per_word;
+    for (; __nw; --__nw) {
+      __storage_type __w = 0;
+      for (__storage_type __i = 0; __i < __bits_per_word; ++__i, ++__first)
+        __w |= static_cast<__storage_type>(*__first) << __i;
+      *__result.__seg_++ = __w;
+    }
+    // do last partial word, if present
+    if (__n) {
+      __storage_type __w = 0;
+      for (__storage_type __i = 0; __i < __n; ++__i, ++__first)
+        __w |= static_cast<__storage_type>(*__first) << __i;
+      __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
+      *__result.__seg_ &= ~__m;
+      *__result.__seg_ |= __w;
+      __result.__ctz_ = __n;
+    }
+    return std::make_pair(std::move(__first), std::move(__result));
+  }
+
   // At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer.
   template <class _In, class _Out, __enable_if_t<__can_lower_copy_assignment_to_memmove<_In, _Out>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*>

libcxx/include/__bit_reference

@@ -10,6 +10,7 @@
 #ifndef _LIBCPP___BIT_REFERENCE
 #define _LIBCPP___BIT_REFERENCE
 
+#include <__algorithm/copy.h>
 #include <__algorithm/copy_n.h>
 #include <__algorithm/min.h>
 #include <__bit/countr.h>
@@ -22,8 +23,11 @@
 #include <__memory/construct_at.h>
 #include <__memory/pointer_traits.h>
 #include <__type_traits/conditional.h>
+#include <__type_traits/enable_if.h>
 #include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_convertible.h>
 #include <__type_traits/void_t.h>
+#include <__utility/pair.h>
 #include <__utility/swap.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -978,6 +982,16 @@ private:
   template <class _Dp>
   friend struct __bit_array;
 
+  template <class _InIter,
+            class _Sent,
+            class _Dp,
+            __enable_if_t<(__has_forward_iterator_category<_InIter>::value ||
+                           __has_iterator_concept_convertible_to<_InIter, forward_iterator_tag>::value) &&
+                              is_convertible<typename iterator_traits<_InIter>::value_type, bool>::value,
+                          int> >
+  _LIBCPP_CONSTEXPR_SINCE_CXX14 friend pair<_InIter, __bit_iterator<_Dp, false> >
+  __copy_impl::operator()(_InIter __first, _Sent __last, __bit_iterator<_Dp, false> __result) const;
+
   template <bool _FillVal, class _Dp>
   _LIBCPP_CONSTEXPR_SINCE_CXX20 friend void
   __fill_n_bool(__bit_iterator<_Dp, false> __first, typename __size_difference_type_traits<_Dp>::size_type __n);

libcxx/include/__vector/vector_bool.h

@@ -171,7 +171,6 @@ class _LIBCPP_TEMPLATE_VIS vector<bool, _Allocator> {
     if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
       auto __n = static_cast<size_type>(ranges::distance(__range));
       __init_with_size(ranges::begin(__range), ranges::end(__range), __n);
-
     } else {
       __init_with_sentinel(ranges::begin(__range), ranges::end(__range));
     }

libcxx/test/benchmarks/containers/ContainerBenchmarks.h

@@ -10,11 +10,13 @@
 #ifndef BENCHMARK_CONTAINER_BENCHMARKS_H
 #define BENCHMARK_CONTAINER_BENCHMARKS_H
 
+#include <__type_traits/type_identity.h>
 #include <cassert>
 #include <iterator>
 #include <utility>
 
 #include "benchmark/benchmark.h"
+#include "../../std/containers/from_range_helpers.h"
 #include "../Utilities.h"
 #include "test_iterators.h"
 
@@ -51,16 +53,57 @@ void BM_Assignment(benchmark::State& st, Container) {
   }
 }
 
+template <class Container, class Generator, class InputIter = decltype(std::declval<Generator>()(0).begin())>
+void BM_AssignIterIter(benchmark::State& st, Generator gen, InputIter = {}) {
+  using T   = typename Container::value_type;
+  auto size = st.range(0);
+  auto in1  = gen(size);
+  auto in2  = gen(size);
+  DoNotOptimizeData(in1);
+  DoNotOptimizeData(in2);
+  Container c(in1.begin(), in1.end());
+  DoNotOptimizeData(c);
+  bool toggle = false;
+  for (auto _ : st) {
+    std::vector<T>& in = toggle ? in1 : in2;
+    auto first         = in.begin();
+    auto last          = in.end();
+    c.assign(InputIter(first), InputIter(last));
+    toggle = !toggle;
+    DoNotOptimizeData(c);
+  }
+}
+
+template <typename Container, class Generator, class Range = std::__type_identity_t<Container>>
+void BM_AssignRange(benchmark::State& st, Generator gen, Range = {}) {
+  auto size = st.range(0);
+  auto in1  = gen(size);
+  auto in2  = gen(size);
+  DoNotOptimizeData(in1);
+  DoNotOptimizeData(in2);
+  Range rg1(std::ranges::begin(in1), std::ranges::end(in1));
+  Range rg2(std::ranges::begin(in2), std::ranges::end(in2));
+  Container c(std::from_range, rg1);
+  DoNotOptimizeData(c);
+  bool toggle = false;
+  for (auto _ : st) {
+    auto& rg = toggle ? rg1 : rg2;
+    c.assign_range(rg);
+    toggle = !toggle;
+    DoNotOptimizeData(c);
+  }
+}
+
 template <std::size_t... sz, typename Container, typename GenInputs>
 void BM_AssignInputIterIter(benchmark::State& st, Container c, GenInputs gen) {
   auto v = gen(1, sz...);
   c.resize(st.range(0), v[0]);
   auto in = gen(st.range(1), sz...);
-  benchmark::DoNotOptimize(&in);
-  benchmark::DoNotOptimize(&c);
+  DoNotOptimizeData(in);
+  DoNotOptimizeData(c);
   for (auto _ : st) {
     c.assign(cpp17_input_iterator(in.begin()), cpp17_input_iterator(in.end()));
-    benchmark::ClobberMemory();
+    DoNotOptimizeData(c);
   }
 }
 
@@ -73,24 +116,25 @@ void BM_ConstructSizeValue(benchmark::State& st, Container, typename Container::
   }
 }
 
-template <class Container, class GenInputs>
-void BM_ConstructIterIter(benchmark::State& st, Container, GenInputs gen) {
-  auto in          = gen(st.range(0));
-  const auto begin = in.begin();
-  const auto end   = in.end();
-  benchmark::DoNotOptimize(&in);
+template <class Container, class GenInputs, class InputIter = decltype(std::declval<GenInputs>()(0).begin())>
+void BM_ConstructIterIter(benchmark::State& st, GenInputs gen, InputIter = {}) {
+  auto in = gen(st.range(0));
+  DoNotOptimizeData(in);
+  const auto begin = InputIter(in.begin());
+  const auto end   = InputIter(in.end());
   while (st.KeepRunning()) {
-    Container c(begin, end);
+    Container c(begin, end); // we assume the destructor doesn't dominate the benchmark
     DoNotOptimizeData(c);
   }
 }
 
-template <class Container, class GenInputs>
-void BM_ConstructFromRange(benchmark::State& st, Container, GenInputs gen) {
+template <class Container, class GenInputs, class Range = std::__type_identity_t<Container>>
+void BM_ConstructFromRange(benchmark::State& st, GenInputs gen, Range = {}) {
   auto in = gen(st.range(0));
-  benchmark::DoNotOptimize(&in);
+  DoNotOptimizeData(in);
+  Range rg(std::ranges::begin(in), std::ranges::end(in));
   while (st.KeepRunning()) {
-    Container c(std::from_range, in);
+    Container c(std::from_range, rg); // we assume the destructor doesn't dominate the benchmark
     DoNotOptimizeData(c);
   }
 }
@@ -108,6 +152,52 @@ void BM_Pushback_no_grow(benchmark::State& state, Container c) {
   }
 }
 
+template <class Container, class GenInputs, class InputIter = decltype(std::declval<GenInputs>()(0).begin())>
+void BM_InsertIterIterIter(benchmark::State& st, GenInputs gen, InputIter = {}) {
+  auto in = gen(st.range(0));
+  DoNotOptimizeData(in);
+  const auto beg      = InputIter(in.begin());
+  const auto end      = InputIter(in.end());
+  const unsigned size = 100;
+  Container c(size);
+  DoNotOptimizeData(c);
+  for (auto _ : st) {
+    c.insert(c.begin(), beg, end);
+    DoNotOptimizeData(c);
+    c.erase(c.begin() + size, c.end()); // avoid growing indefinitely
+  }
+}
+
+template <class Container, class GenInputs, class Range = std::__type_identity_t<Container>>
+void BM_InsertRange(benchmark::State& st, GenInputs gen, Range = {}) {
+  auto in = gen(st.range(0));
+  DoNotOptimizeData(in);
+  Range rg(std::ranges::begin(in), std::ranges::end(in));
+  const unsigned size = 100;
+  Container c(size);
+  DoNotOptimizeData(c);
+  for (auto _ : st) {
+    c.insert_range(c.begin(), rg);
+    DoNotOptimizeData(c);
+    c.erase(c.begin() + size, c.end()); // avoid growing indefinitely
+  }
+}
+
+template <class Container, class GenInputs, class Range = std::__type_identity_t<Container>>
+void BM_AppendRange(benchmark::State& st, GenInputs gen, Range = {}) {
+  auto in = gen(st.range(0));
+  DoNotOptimizeData(in);
+  Range rg(std::ranges::begin(in), std::ranges::end(in));
+  const unsigned size = 100;
+  Container c(size);
+  DoNotOptimizeData(c);
+  for (auto _ : st) {
+    c.append_range(rg);
+    DoNotOptimizeData(c);
+    c.erase(c.begin() + size, c.end()); // avoid growing indefinitely
+  }
+}
+
 template <class Container, class GenInputs>
 void BM_InsertValue(benchmark::State& st, Container c, GenInputs gen) {
   auto in        = gen(st.range(0));

libcxx/test/benchmarks/containers/deque.bench.cpp

@@ -24,22 +24,22 @@ BENCHMARK_CAPTURE(BM_ConstructSize, deque_byte, std::deque<unsigned char>{})->Ar
 
 BENCHMARK_CAPTURE(BM_ConstructSizeValue, deque_byte, std::deque<unsigned char>{}, 0)->Arg(5140480);
 
-BENCHMARK_CAPTURE(BM_ConstructIterIter, deque_char, std::deque<char>{}, getRandomIntegerInputs<char>)
+BENCHMARK_CAPTURE(BM_ConstructIterIter<std::deque<char>>, deque_char, getRandomIntegerInputs<char>)
     ->Arg(TestNumInputs);
 
-BENCHMARK_CAPTURE(BM_ConstructIterIter, deque_size_t, std::deque<size_t>{}, getRandomIntegerInputs<size_t>)
+BENCHMARK_CAPTURE(BM_ConstructIterIter<std::deque<size_t>>, deque_size_t, getRandomIntegerInputs<size_t>)
     ->Arg(TestNumInputs);
 
-BENCHMARK_CAPTURE(BM_ConstructIterIter, deque_string, std::deque<std::string>{}, getRandomStringInputs)
+BENCHMARK_CAPTURE(BM_ConstructIterIter<std::deque<std::string>>, deque_string, getRandomStringInputs)
     ->Arg(TestNumInputs);
 
-BENCHMARK_CAPTURE(BM_ConstructFromRange, deque_char, std::deque<char>{}, getRandomIntegerInputs<char>)
+BENCHMARK_CAPTURE(BM_ConstructFromRange<std::deque<char>>, deque_char, getRandomIntegerInputs<char>)
     ->Arg(TestNumInputs);
 
-BENCHMARK_CAPTURE(BM_ConstructFromRange, deque_size_t, std::deque<size_t>{}, getRandomIntegerInputs<size_t>)
+BENCHMARK_CAPTURE(BM_ConstructFromRange<std::deque<size_t>>, deque_size_t, getRandomIntegerInputs<size_t>)
     ->Arg(TestNumInputs);
 
-BENCHMARK_CAPTURE(BM_ConstructFromRange, deque_string, std::deque<std::string>{}, getRandomStringInputs)
+BENCHMARK_CAPTURE(BM_ConstructFromRange<std::deque<std::string>>, deque_string, getRandomStringInputs)
     ->Arg(TestNumInputs);
 
 BENCHMARK_CAPTURE(BM_erase_iter_in_middle, deque_int, std::deque<int>{}, getRandomIntegerInputs<int>)