d0cee6939a
This patch optimizes `std::fill`, `std::fill_n`, `std::ranges::fill`, and `std::ranges::fill_n` for segmented iterators, achieving substantial performance improvements. Specifically, for `deque<int>` iterators, the performance improvements are above 10x for all these algorithms. The optimization also enables filling segmented memory of `deque<int>` to approach the performance of filling contiguous memory of `vector<int>`. Benchmark results comparing the before and after implementations are provided below. For additional context, we’ve included `vector<int>` results, which remain unchanged, as this patch specifically targets segmented iterators and leaves non-segmented iterator behavior untouched. Fixes two subtasks outlined in #102817. #### `fill_n` ``` ----------------------------------------------------------------------------- Benchmark Before After Speedup ----------------------------------------------------------------------------- std::fill_n(deque<int>)/32 11.4 ns 2.28 ns 5.0x std::fill_n(deque<int>)/50 19.7 ns 3.40 ns 5.8x std::fill_n(deque<int>)/1024 391 ns 37.3 ns 10.5x std::fill_n(deque<int>)/8192 3174 ns 301 ns 10.5x std::fill_n(deque<int>)/65536 26504 ns 2951 ns 9.0x std::fill_n(deque<int>)/1048576 407960 ns 80658 ns 5.1x rng::fill_n(deque<int>)/32 14.3 ns 2.15 ns 6.6x rng::fill_n(deque<int>)/50 20.2 ns 3.22 ns 6.3x rng::fill_n(deque<int>)/1024 381 ns 37.8 ns 10.1x rng::fill_n(deque<int>)/8192 3101 ns 294 ns 10.5x rng::fill_n(deque<int>)/65536 25098 ns 2926 ns 8.6x rng::fill_n(deque<int>)/1048576 394342 ns 78874 ns 5.0x std::fill_n(vector<int>)/32 1.76 ns 1.72 ns 1.0x std::fill_n(vector<int>)/50 3.00 ns 2.73 ns 1.1x std::fill_n(vector<int>)/1024 38.4 ns 37.9 ns 1.0x std::fill_n(vector<int>)/8192 258 ns 252 ns 1.0x std::fill_n(vector<int>)/65536 2993 ns 2889 ns 1.0x std::fill_n(vector<int>)/1048576 80328 ns 80468 ns 1.0x rng::fill_n(vector<int>)/32 1.99 ns 1.35 ns 1.5x rng::fill_n(vector<int>)/50 2.66 ns 2.12 ns 1.3x rng::fill_n(vector<int>)/1024 37.7 ns 35.8 ns 1.1x rng::fill_n(vector<int>)/8192 253 ns 250 ns 1.0x rng::fill_n(vector<int>)/65536 2922 ns 2930 ns 1.0x rng::fill_n(vector<int>)/1048576 79739 ns 79742 ns 1.0x ``` #### `fill` ``` -------------------------------------------------------------------------- Benchmark Before After Speedup -------------------------------------------------------------------------- std::fill(deque<int>)/32 13.7 ns 2.45 ns 5.6x std::fill(deque<int>)/50 21.7 ns 4.57 ns 4.7x std::fill(deque<int>)/1024 367 ns 38.5 ns 9.5x std::fill(deque<int>)/8192 2896 ns 247 ns 11.7x std::fill(deque<int>)/65536 23723 ns 2907 ns 8.2x std::fill(deque<int>)/1048576 379043 ns 79885 ns 4.7x rng::fill(deque<int>)/32 13.6 ns 2.70 ns 5.0x rng::fill(deque<int>)/50 23.4 ns 3.94 ns 5.9x rng::fill(deque<int>)/1024 377 ns 37.9 ns 9.9x rng::fill(deque<int>)/8192 2914 ns 286 ns 10.2x rng::fill(deque<int>)/65536 23612 ns 2939 ns 8.0x rng::fill(deque<int>)/1048576 379841 ns 80079 ns 4.7x std::fill(vector<int>)/32 1.99 ns 1.79 ns 1.1x std::fill(vector<int>)/50 3.05 ns 3.06 ns 1.0x std::fill(vector<int>)/1024 37.6 ns 38.0 ns 1.0x std::fill(vector<int>)/8192 255 ns 257 ns 1.0x std::fill(vector<int>)/65536 2966 ns 2981 ns 1.0x std::fill(vector<int>)/1048576 78300 ns 80348 ns 1.0x rng::fill(vector<int>)/32 1.77 ns 1.75 ns 1.0x rng::fill(vector<int>)/50 4.85 ns 2.31 ns 2.1x rng::fill(vector<int>)/1024 39.6 ns 36.1 ns 1.1x rng::fill(vector<int>)/8192 238 ns 251 ns 0.9x rng::fill(vector<int>)/65536 2941 ns 2918 ns 1.0x rng::fill(vector<int>)/1048576 80497 ns 80442 ns 1.0x ``` --------- Co-authored-by: Louis Dionne <ldionne.2@gmail.com> Co-authored-by: A. Jiang <de34@live.cn>
118 lines
4.3 KiB
C++
118 lines
4.3 KiB
C++
//===----------------------------------------------------------------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#ifndef _LIBCPP___ALGORITHM_FILL_N_H
|
|
#define _LIBCPP___ALGORITHM_FILL_N_H
|
|
|
|
#include <__algorithm/for_each_n_segment.h>
|
|
#include <__algorithm/min.h>
|
|
#include <__config>
|
|
#include <__fwd/bit_reference.h>
|
|
#include <__iterator/iterator_traits.h>
|
|
#include <__iterator/segmented_iterator.h>
|
|
#include <__memory/pointer_traits.h>
|
|
#include <__type_traits/conjunction.h>
|
|
#include <__type_traits/enable_if.h>
|
|
#include <__type_traits/integral_constant.h>
|
|
#include <__type_traits/negation.h>
|
|
#include <__utility/convert_to_integral.h>
|
|
|
|
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
|
|
# pragma GCC system_header
|
|
#endif
|
|
|
|
_LIBCPP_PUSH_MACROS
|
|
#include <__undef_macros>
|
|
|
|
_LIBCPP_BEGIN_NAMESPACE_STD
|
|
|
|
// fill_n isn't specialized for std::memset, because the compiler already optimizes the loop to a call to std::memset.
|
|
|
|
template <class _OutputIterator,
|
|
class _Size,
|
|
class _Tp
|
|
#ifndef _LIBCPP_CXX03_LANG
|
|
,
|
|
__enable_if_t<!_And<_BoolConstant<__is_segmented_iterator_v<_OutputIterator>>,
|
|
__has_random_access_local_iterator<_OutputIterator>>::value,
|
|
int> = 0
|
|
#endif
|
|
>
|
|
inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
|
|
__fill_n(_OutputIterator __first, _Size __n, const _Tp& __value) {
|
|
for (; __n > 0; ++__first, (void)--__n)
|
|
*__first = __value;
|
|
return __first;
|
|
}
|
|
|
|
#ifndef _LIBCPP_CXX03_LANG
|
|
template < class _OutputIterator,
|
|
class _Size,
|
|
class _Tp,
|
|
__enable_if_t<_And<_BoolConstant<__is_segmented_iterator_v<_OutputIterator>>,
|
|
__has_random_access_local_iterator<_OutputIterator>>::value,
|
|
int> = 0>
|
|
inline _LIBCPP_HIDE_FROM_ABI
|
|
_LIBCPP_CONSTEXPR_SINCE_CXX14 _OutputIterator __fill_n(_OutputIterator __first, _Size __n, const _Tp& __value) {
|
|
using __local_iterator_t = typename __segmented_iterator_traits<_OutputIterator>::__local_iterator;
|
|
return std::__for_each_n_segment(__first, __n, [&](__local_iterator_t __lfirst, __local_iterator_t __llast) {
|
|
std::__fill_n(__lfirst, __llast - __lfirst, __value);
|
|
});
|
|
}
|
|
#endif // !_LIBCPP_CXX03_LANG
|
|
|
|
template <bool _FillVal, class _Cp>
|
|
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
|
|
__fill_n_bool(__bit_iterator<_Cp, false> __first, typename __size_difference_type_traits<_Cp>::size_type __n) {
|
|
using _It = __bit_iterator<_Cp, false>;
|
|
using __storage_type = typename _It::__storage_type;
|
|
|
|
const int __bits_per_word = _It::__bits_per_word;
|
|
// do first partial word
|
|
if (__first.__ctz_ != 0) {
|
|
__storage_type __clz_f = static_cast<__storage_type>(__bits_per_word - __first.__ctz_);
|
|
__storage_type __dn = std::min(__clz_f, __n);
|
|
std::__fill_masked_range(std::__to_address(__first.__seg_), __clz_f - __dn, __first.__ctz_, _FillVal);
|
|
__n -= __dn;
|
|
++__first.__seg_;
|
|
}
|
|
// do middle whole words
|
|
__storage_type __nw = __n / __bits_per_word;
|
|
std::__fill_n(std::__to_address(__first.__seg_), __nw, _FillVal ? static_cast<__storage_type>(-1) : 0);
|
|
__n -= __nw * __bits_per_word;
|
|
// do last partial word
|
|
if (__n > 0) {
|
|
__first.__seg_ += __nw;
|
|
std::__fill_masked_range(std::__to_address(__first.__seg_), __bits_per_word - __n, 0u, _FillVal);
|
|
}
|
|
}
|
|
|
|
template <class _Cp, class _Size>
|
|
inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator<_Cp, false>
|
|
__fill_n(__bit_iterator<_Cp, false> __first, _Size __n, const bool& __value) {
|
|
if (__n > 0) {
|
|
if (__value)
|
|
std::__fill_n_bool<true>(__first, __n);
|
|
else
|
|
std::__fill_n_bool<false>(__first, __n);
|
|
}
|
|
return __first + __n;
|
|
}
|
|
|
|
template <class _OutputIterator, class _Size, class _Tp>
|
|
inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
|
|
fill_n(_OutputIterator __first, _Size __n, const _Tp& __value) {
|
|
return std::__fill_n(__first, std::__convert_to_integral(__n), __value);
|
|
}
|
|
|
|
_LIBCPP_END_NAMESPACE_STD
|
|
|
|
_LIBCPP_POP_MACROS
|
|
|
|
#endif // _LIBCPP___ALGORITHM_FILL_N_H
|