b68e2eba0b
``` --------------------------------------------------- Benchmark old new --------------------------------------------------- bm_mismatch<char>/1 0.835 ns 2.37 ns bm_mismatch<char>/2 1.44 ns 2.60 ns bm_mismatch<char>/3 2.06 ns 2.83 ns bm_mismatch<char>/4 2.60 ns 3.29 ns bm_mismatch<char>/5 3.15 ns 3.77 ns bm_mismatch<char>/6 3.82 ns 4.17 ns bm_mismatch<char>/7 4.29 ns 4.52 ns bm_mismatch<char>/8 4.78 ns 4.86 ns bm_mismatch<char>/16 9.06 ns 7.54 ns bm_mismatch<char>/64 31.7 ns 19.1 ns bm_mismatch<char>/512 249 ns 8.16 ns bm_mismatch<char>/4096 1956 ns 44.2 ns bm_mismatch<char>/32768 15498 ns 501 ns bm_mismatch<char>/262144 123965 ns 4479 ns bm_mismatch<char>/1048576 495668 ns 21306 ns bm_mismatch<short>/1 0.710 ns 2.12 ns bm_mismatch<short>/2 1.03 ns 2.66 ns bm_mismatch<short>/3 1.29 ns 3.56 ns bm_mismatch<short>/4 1.68 ns 4.29 ns bm_mismatch<short>/5 1.96 ns 5.18 ns bm_mismatch<short>/6 2.59 ns 5.91 ns bm_mismatch<short>/7 2.86 ns 6.63 ns bm_mismatch<short>/8 3.19 ns 7.33 ns bm_mismatch<short>/16 5.48 ns 13.0 ns bm_mismatch<short>/64 16.6 ns 4.06 ns bm_mismatch<short>/512 130 ns 13.8 ns bm_mismatch<short>/4096 985 ns 93.8 ns bm_mismatch<short>/32768 7846 ns 1002 ns bm_mismatch<short>/262144 63217 ns 10637 ns bm_mismatch<short>/1048576 251782 ns 42471 ns bm_mismatch<int>/1 0.716 ns 1.91 ns bm_mismatch<int>/2 1.21 ns 2.49 ns bm_mismatch<int>/3 1.38 ns 3.46 ns bm_mismatch<int>/4 1.71 ns 4.04 ns bm_mismatch<int>/5 2.00 ns 4.98 ns bm_mismatch<int>/6 2.43 ns 5.67 ns bm_mismatch<int>/7 3.05 ns 6.38 ns bm_mismatch<int>/8 3.22 ns 7.09 ns bm_mismatch<int>/16 5.18 ns 12.8 ns bm_mismatch<int>/64 16.6 ns 5.28 ns bm_mismatch<int>/512 129 ns 25.2 ns bm_mismatch<int>/4096 1009 ns 201 ns bm_mismatch<int>/32768 7776 ns 2144 ns bm_mismatch<int>/262144 62371 ns 20551 ns bm_mismatch<int>/1048576 254750 ns 90097 ns ```
124 lines
4.5 KiB
C++
124 lines
4.5 KiB
C++
//===----------------------------------------------------------------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#ifndef _LIBCPP___ALGORITHM_SIMD_UTILS_H
|
|
#define _LIBCPP___ALGORITHM_SIMD_UTILS_H
|
|
|
|
#include <__bit/bit_cast.h>
|
|
#include <__bit/countr.h>
|
|
#include <__config>
|
|
#include <__type_traits/is_arithmetic.h>
|
|
#include <__type_traits/is_same.h>
|
|
#include <__utility/integer_sequence.h>
|
|
#include <cstddef>
|
|
#include <cstdint>
|
|
|
|
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
|
|
# pragma GCC system_header
|
|
#endif
|
|
|
|
// TODO: Find out how altivec changes things and allow vectorizations there too.
|
|
#if _LIBCPP_STD_VER >= 14 && defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 1700 && !defined(__ALTIVEC__)
|
|
# define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 1
|
|
#else
|
|
# define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 0
|
|
#endif
|
|
|
|
#if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS && !defined(__OPTIMIZE_SIZE__)
|
|
# define _LIBCPP_VECTORIZE_ALGORITHMS 1
|
|
#else
|
|
# define _LIBCPP_VECTORIZE_ALGORITHMS 0
|
|
#endif
|
|
|
|
#if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS
|
|
|
|
_LIBCPP_BEGIN_NAMESPACE_STD
|
|
|
|
// This isn't specialized for 64 byte vectors on purpose. They have the potential to significantly reduce performance
|
|
// in mixed simd/non-simd workloads and don't provide any performance improvement for currently vectorized algorithms
|
|
// as far as benchmarks are concerned.
|
|
# if defined(__AVX__)
|
|
template <class _Tp>
|
|
inline constexpr size_t __native_vector_size = 32 / sizeof(_Tp);
|
|
# elif defined(__SSE__) || defined(__ARM_NEON__)
|
|
template <class _Tp>
|
|
inline constexpr size_t __native_vector_size = 16 / sizeof(_Tp);
|
|
# elif defined(__MMX__)
|
|
template <class _Tp>
|
|
inline constexpr size_t __native_vector_size = 8 / sizeof(_Tp);
|
|
# else
|
|
template <class _Tp>
|
|
inline constexpr size_t __native_vector_size = 1;
|
|
# endif
|
|
|
|
template <class _ArithmeticT, size_t _Np>
|
|
using __simd_vector __attribute__((__ext_vector_type__(_Np))) = _ArithmeticT;
|
|
|
|
template <class _VecT>
|
|
inline constexpr size_t __simd_vector_size_v = []<bool _False = false>() -> size_t {
|
|
static_assert(_False, "Not a vector!");
|
|
}();
|
|
|
|
template <class _Tp, size_t _Np>
|
|
inline constexpr size_t __simd_vector_size_v<__simd_vector<_Tp, _Np>> = _Np;
|
|
|
|
template <class _Tp, size_t _Np>
|
|
_LIBCPP_HIDE_FROM_ABI _Tp __simd_vector_underlying_type_impl(__simd_vector<_Tp, _Np>) {
|
|
return _Tp{};
|
|
}
|
|
|
|
template <class _VecT>
|
|
using __simd_vector_underlying_type_t = decltype(std::__simd_vector_underlying_type_impl(_VecT{}));
|
|
|
|
// This isn't inlined without always_inline when loading chars.
|
|
template <class _VecT, class _Tp>
|
|
_LIBCPP_NODISCARD _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _VecT __load_vector(const _Tp* __ptr) noexcept {
|
|
return [=]<size_t... _Indices>(index_sequence<_Indices...>) _LIBCPP_ALWAYS_INLINE noexcept {
|
|
return _VecT{__ptr[_Indices]...};
|
|
}(make_index_sequence<__simd_vector_size_v<_VecT>>{});
|
|
}
|
|
|
|
template <class _Tp, size_t _Np>
|
|
_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool __all_of(__simd_vector<_Tp, _Np> __vec) noexcept {
|
|
return __builtin_reduce_and(__builtin_convertvector(__vec, __simd_vector<bool, _Np>));
|
|
}
|
|
|
|
template <class _Tp, size_t _Np>
|
|
_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_set(__simd_vector<_Tp, _Np> __vec) noexcept {
|
|
using __mask_vec = __simd_vector<bool, _Np>;
|
|
|
|
// This has MSan disabled du to https://github.com/llvm/llvm-project/issues/85876
|
|
auto __impl = [&]<class _MaskT>(_MaskT) _LIBCPP_NO_SANITIZE("memory") noexcept {
|
|
return std::__countr_zero(__builtin_bit_cast(_MaskT, __builtin_convertvector(__vec, __mask_vec)));
|
|
};
|
|
|
|
if constexpr (sizeof(__mask_vec) == sizeof(uint8_t)) {
|
|
return __impl(uint8_t{});
|
|
} else if constexpr (sizeof(__mask_vec) == sizeof(uint16_t)) {
|
|
return __impl(uint16_t{});
|
|
} else if constexpr (sizeof(__mask_vec) == sizeof(uint32_t)) {
|
|
return __impl(uint32_t{});
|
|
} else if constexpr (sizeof(__mask_vec) == sizeof(uint64_t)) {
|
|
return __impl(uint64_t{});
|
|
} else {
|
|
static_assert(sizeof(__mask_vec) == 0, "unexpected required size for mask integer type");
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
template <class _Tp, size_t _Np>
|
|
_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_not_set(__simd_vector<_Tp, _Np> __vec) noexcept {
|
|
return std::__find_first_set(~__vec);
|
|
}
|
|
|
|
_LIBCPP_END_NAMESPACE_STD
|
|
|
|
#endif // _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS
|
|
|
|
#endif // _LIBCPP___ALGORITHM_SIMD_UTILS_H
|