From e9788cc04c2560da812c7eca726c08bcc69dfb30 Mon Sep 17 00:00:00 2001 From: Bartosz Taudul Date: Fri, 26 Jun 2020 00:51:03 +0200 Subject: [PATCH] Bump robin_hood to 3.7.0. --- server/tracy_robin_hood.h | 557 ++++++++++++++++++++------------------ 1 file changed, 300 insertions(+), 257 deletions(-) diff --git a/server/tracy_robin_hood.h b/server/tracy_robin_hood.h index 46cc4d52..81afbea4 100644 --- a/server/tracy_robin_hood.h +++ b/server/tracy_robin_hood.h @@ -6,7 +6,7 @@ // _/_____/ // // Fast & memory efficient hashtable based on robin hood hashing for C++11/14/17/20 -// version 3.5.0 +// version 3.7.0 // https://github.com/martinus/robin-hood-hashing // // Licensed under the MIT License . @@ -36,7 +36,7 @@ // see https://semver.org/ #define ROBIN_HOOD_VERSION_MAJOR 3 // for incompatible API changes -#define ROBIN_HOOD_VERSION_MINOR 5 // for adding functionality in a backwards-compatible manner +#define ROBIN_HOOD_VERSION_MINOR 7 // for adding functionality in a backwards-compatible manner #define ROBIN_HOOD_VERSION_PATCH 0 // for backwards-compatible bug fixes #include @@ -128,7 +128,7 @@ static Counts& counts() { #endif // count leading/trailing bits -#if ( ( defined __i386 || defined __x86_64__ ) && defined __BMI__ ) || defined _M_IX86 || defined _M_X64 +#if ((defined __i386 || defined __x86_64__) && defined __BMI__) || defined _M_IX86 || defined _M_X64 # ifdef _MSC_VER # include # else @@ -142,7 +142,7 @@ static Counts& counts() { # if defined __AVX2__ || defined __BMI__ # define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) ROBIN_HOOD(CTZ)(x) # else -# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) ((x) ? ROBIN_HOOD(CTZ)(x) : ROBIN_HOOD(BITNESS)) +# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) ROBIN_HOOD(CTZ)(x) # endif #elif defined _MSC_VER # if ROBIN_HOOD(BITNESS) == 32 @@ -290,38 +290,10 @@ using index_sequence_for = make_index_sequence; namespace detail { -// umul -#if defined(__SIZEOF_INT128__) -# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_UMUL128() 1 -# if defined(__GNUC__) || defined(__clang__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wpedantic" -using uint128_t = unsigned __int128; -# pragma GCC diagnostic pop -# endif -inline uint64_t umul128(uint64_t a, uint64_t b, uint64_t* high) noexcept { - auto result = static_cast(a) * static_cast(b); - *high = static_cast(result >> 64U); - return static_cast(result); +template +T rotr(T x, unsigned k) { + return (x >> k) | (x << (8U * sizeof(T) - k)); } -#elif (defined(_MSC_VER) && ROBIN_HOOD(BITNESS) == 64) -# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_UMUL128() 1 -# include // for __umulh -# pragma intrinsic(__umulh) -# ifndef _M_ARM64 -# pragma intrinsic(_umul128) -# endif -inline uint64_t umul128(uint64_t a, uint64_t b, uint64_t* high) noexcept { -# ifdef _M_ARM64 - *high = __umulh(a, b); - return ((uint64_t)(a)) * (b); -# else - return _umul128(a, b, high); -# endif -} -#else -# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_UMUL128() 0 -#endif // This cast gets rid of warnings like "cast from 'uint8_t*' {aka 'unsigned char*'} to // 'uint64_t*' {aka 'long unsigned int*'} increases required alignment of target type". Use with @@ -489,10 +461,10 @@ private: mListForFree = data; // create linked list for newly allocated data - auto const headT = + auto* const headT = reinterpret_cast_no_cast_align_warning(reinterpret_cast(ptr) + ALIGNMENT); - auto const head = reinterpret_cast(headT); + auto* const head = reinterpret_cast(headT); // Visual Studio compiler automatically unrolls this loop, which is pretty cool for (size_t i = 0; i < numElements; ++i) { @@ -541,7 +513,7 @@ private: T** mListForFree{nullptr}; }; -template +template struct NodeAllocator; // dummy allocator that does nothing @@ -568,12 +540,18 @@ struct identity_hash { // c++14 doesn't have is_nothrow_swappable, and clang++ 6.0.1 doesn't like it either, so I'm making // my own here. namespace swappable { +#if ROBIN_HOOD(CXX) < ROBIN_HOOD(CXX17) using std::swap; template struct nothrow { static const bool value = noexcept(swap(std::declval(), std::declval())); }; - +#else +template +struct nothrow { + static const bool value = std::is_nothrow_swappable::value; +}; +#endif } // namespace swappable } // namespace detail @@ -602,20 +580,19 @@ struct pair { , second(o.second) {} // pair constructors are explicit so we don't accidentally call this ctor when we don't have to. - explicit constexpr pair(std::pair&& o) noexcept( - noexcept(T1(std::move(std::declval()))) && - noexcept(T2(std::move(std::declval())))) + explicit constexpr pair(std::pair&& o) noexcept(noexcept( + T1(std::move(std::declval()))) && noexcept(T2(std::move(std::declval())))) : first(std::move(o.first)) , second(std::move(o.second)) {} - constexpr pair(T1&& a, T2&& b) noexcept(noexcept(T1(std::move(std::declval()))) && - noexcept(T2(std::move(std::declval())))) + constexpr pair(T1&& a, T2&& b) noexcept(noexcept( + T1(std::move(std::declval()))) && noexcept(T2(std::move(std::declval())))) : first(std::move(a)) , second(std::move(b)) {} template - constexpr pair(U1&& a, U2&& b) noexcept(noexcept(T1(std::forward(std::declval()))) && - noexcept(T2(std::forward(std::declval())))) + constexpr pair(U1&& a, U2&& b) noexcept(noexcept(T1(std::forward( + std::declval()))) && noexcept(T2(std::forward(std::declval())))) : first(std::forward(a)) , second(std::forward(b)) {} @@ -631,15 +608,12 @@ struct pair { // constructor called from the std::piecewise_construct_t ctor template - pair(std::tuple& a, std::tuple& b, - ROBIN_HOOD_STD::index_sequence /*unused*/, - ROBIN_HOOD_STD::index_sequence< - I2...> /*unused*/) noexcept(noexcept(T1(std:: - forward(std::get( - std::declval< - std::tuple&>()))...)) && - noexcept(T2(std::forward( - std::get(std::declval&>()))...))) + pair(std::tuple& a, std::tuple& b, ROBIN_HOOD_STD::index_sequence /*unused*/, ROBIN_HOOD_STD::index_sequence /*unused*/) noexcept( + noexcept(T1(std::forward(std::get( + std::declval&>()))...)) && noexcept(T2(std:: + forward(std::get( + std::declval&>()))...))) : first(std::forward(std::get(a))...) , second(std::forward(std::get(b))...) { // make visual studio compiler happy about warning about unused a & b. @@ -648,19 +622,6 @@ struct pair { (void)b; } - ROBIN_HOOD(NODISCARD) first_type& getFirst() noexcept { - return first; - } - ROBIN_HOOD(NODISCARD) first_type const& getFirst() const noexcept { - return first; - } - ROBIN_HOOD(NODISCARD) second_type& getSecond() noexcept { - return second; - } - ROBIN_HOOD(NODISCARD) second_type const& getSecond() const noexcept { - return second; - } - void swap(pair& o) noexcept((detail::swappable::nothrow::value) && (detail::swappable::nothrow::value)) { using std::swap; @@ -673,19 +634,44 @@ struct pair { }; template -void swap(pair& a, pair& b) noexcept( +inline void swap(pair& a, pair& b) noexcept( noexcept(std::declval&>().swap(std::declval&>()))) { a.swap(b); } -// Hash an arbitrary amount of bytes. This is basically Murmur2 hash without caring about big -// endianness. TODO(martinus) add a fallback for very large strings? +template +inline constexpr bool operator==(pair const& x, pair const& y) { + return (x.first == y.first) && (x.second == y.second); +} +template +inline constexpr bool operator!=(pair const& x, pair const& y) { + return !(x == y); +} +template +inline constexpr bool operator<(pair const& x, pair const& y) noexcept(noexcept( + std::declval() < std::declval()) && noexcept(std::declval() < + std::declval())) { + return x.first < y.first || (!(y.first < x.first) && x.second < y.second); +} +template +inline constexpr bool operator>(pair const& x, pair const& y) { + return y < x; +} +template +inline constexpr bool operator<=(pair const& x, pair const& y) { + return !(x > y); +} +template +inline constexpr bool operator>=(pair const& x, pair const& y) { + return !(x < y); +} + static size_t hash_bytes(void const* ptr, size_t const len) noexcept { static constexpr uint64_t m = UINT64_C(0xc6a4a7935bd1e995); static constexpr uint64_t seed = UINT64_C(0xe17a1465); static constexpr unsigned int r = 47; - auto const data64 = static_cast(ptr); + auto const* const data64 = static_cast(ptr); uint64_t h = seed ^ (len * m); size_t const n_blocks = len / 8; @@ -700,7 +686,7 @@ static size_t hash_bytes(void const* ptr, size_t const len) noexcept { h *= m; } - auto const data8 = reinterpret_cast(data64 + n_blocks); + auto const* const data8 = reinterpret_cast(data64 + n_blocks); switch (len & 7U) { case 7: h ^= static_cast(data8[6]) << 48U; @@ -734,28 +720,18 @@ static size_t hash_bytes(void const* ptr, size_t const len) noexcept { return static_cast(h); } -inline size_t hash_int(uint64_t obj) noexcept { -#if ROBIN_HOOD(HAS_UMUL128) - // 167079903232 masksum, 120428523 ops best: 0xde5fb9d2630458e9 - static constexpr uint64_t k = UINT64_C(0xde5fb9d2630458e9); - uint64_t h; - uint64_t l = detail::umul128(obj, k, &h); - return h + l; -#elif ROBIN_HOOD(BITNESS) == 32 - uint64_t const r = obj * UINT64_C(0xca4bcaa75ec3f625); - auto h = static_cast(r >> 32U); - auto l = static_cast(r); - return h + l; -#else - // murmurhash 3 finalizer - uint64_t h = obj; - h ^= h >> 33; - h *= 0xff51afd7ed558ccd; - h ^= h >> 33; - h *= 0xc4ceb9fe1a85ec53; - h ^= h >> 33; +inline size_t hash_int(uint64_t x) noexcept { + // inspired by lemire's strongly universal hashing + // https://lemire.me/blog/2018/08/15/fast-strongly-universal-64-bit-hashing-everywhere/ + // + // Instead of shifts, we use rotations so we don't lose any bits. + // + // Added a final multiplcation with a constant for more mixing. It is most important that the + // lower bits are well mixed. + auto h1 = x * UINT64_C(0xA24BAED4963EE407); + auto h2 = detail::rotr(x, 32U) * UINT64_C(0x9FB21C651E98DF25); + auto h = detail::rotr(h1 + h2, 32U); return static_cast(h); -#endif } // A thin wrapper around std::hash, performing an additional simple mixing step of the result. @@ -817,8 +793,14 @@ ROBIN_HOOD_HASH_INT(unsigned long long); #endif namespace detail { -// using wrapper classes for hash and key_equal prevents the diamond problem when the same type is -// used. see https://stackoverflow.com/a/28771920/48181 +template +struct has_is_transparent : public std::false_type {}; + +template +struct has_is_transparent : public std::true_type {}; + +// using wrapper classes for hash and key_equal prevents the diamond problem when the same type +// is used. see https://stackoverflow.com/a/28771920/48181 template struct WrapHash : public T { WrapHash() = default; @@ -835,8 +817,8 @@ struct WrapKeyEqual : public T { // A highly optimized hashmap implementation, using the Robin Hood algorithm. // -// In most cases, this map should be usable as a drop-in replacement for std::unordered_map, but be -// about 2x faster in most cases and require much less allocations. +// In most cases, this map should be usable as a drop-in replacement for std::unordered_map, but +// be about 2x faster in most cases and require much less allocations. // // This implementation uses the following memory layout: // @@ -844,8 +826,8 @@ struct WrapKeyEqual : public T { // // * Node: either a DataNode that directly has the std::pair as member, // or a DataNode with a pointer to std::pair. Which DataNode representation to use -// depends on how fast the swap() operation is. Heuristically, this is automatically choosen based -// on sizeof(). there are always 2^n Nodes. +// depends on how fast the swap() operation is. Heuristically, this is automatically choosen +// based on sizeof(). there are always 2^n Nodes. // // * info: Each Node in the map has a corresponding info byte, so there are 2^n info bytes. // Each byte is initialized to 0, meaning the corresponding Node is empty. Set to 1 means the @@ -853,34 +835,38 @@ struct WrapKeyEqual : public T { // actually belongs to the previous position and was pushed out because that place is already // taken. // -// * infoSentinel: Sentinel byte set to 1, so that iterator's ++ can stop at end() without the need -// for a idx -// variable. +// * infoSentinel: Sentinel byte set to 1, so that iterator's ++ can stop at end() without the +// need for a idx variable. // -// According to STL, order of templates has effect on throughput. That's why I've moved the boolean -// to the front. +// According to STL, order of templates has effect on throughput. That's why I've moved the +// boolean to the front. // https://www.reddit.com/r/cpp/comments/ahp6iu/compile_time_binary_size_reductions_and_cs_future/eeguck4/ -template -class Table : public WrapHash, - public WrapKeyEqual, - detail::NodeAllocator< - typename std::conditional< - std::is_void::value, Key, - tracy::pair::type, - T>>::type, - 4, 16384, IsFlatMap> { +class Table + : public WrapHash, + public WrapKeyEqual, + detail::NodeAllocator< + typename std::conditional< + std::is_void::value, Key, + tracy::pair::type, T>>::type, + 4, 16384, IsFlat> { public: + static constexpr bool is_flat = IsFlat; + static constexpr bool is_map = !std::is_void::value; + static constexpr bool is_set = !is_map; + static constexpr bool is_transparent = + has_is_transparent::value && has_is_transparent::value; + using key_type = Key; using mapped_type = T; using value_type = typename std::conditional< - std::is_void::value, Key, - tracy::pair::type, T>>::type; + is_set, Key, + tracy::pair::type, T>>::type; using size_type = size_t; using hasher = Hash; using key_equal = KeyEqual; - using Self = Table; - static constexpr bool is_flat_map = IsFlatMap; + using Self = Table; private: static_assert(MaxLoadFactor100 > 10 && MaxLoadFactor100 < 100, @@ -896,7 +882,7 @@ private: static constexpr uint32_t InitialInfoNumBits = 5; static constexpr uint8_t InitialInfoInc = 1U << InitialInfoNumBits; static constexpr uint8_t InitialInfoHashShift = sizeof(size_t) * 8 - InitialInfoNumBits; - using DataPool = detail::NodeAllocator; + using DataPool = detail::NodeAllocator; // type needs to be wider than uint8_t. using InfoType = uint32_t; @@ -904,8 +890,8 @@ private: // DataNode //////////////////////////////////////////////////////// // Primary template for the data node. We have special implementations for small and big - // objects. For large objects it is assumed that swap() is fairly slow, so we allocate these on - // the heap so swap merely swaps a pointer. + // objects. For large objects it is assumed that swap() is fairly slow, so we allocate these + // on the heap so swap merely swaps a pointer. template class DataNode {}; @@ -941,40 +927,38 @@ private: return mData; } - template + template ROBIN_HOOD(NODISCARD) - typename std::enable_if::value, typename V::first_type&>::type - getFirst() noexcept { + typename std::enable_if::type getFirst() noexcept { return mData.first; } - template + template ROBIN_HOOD(NODISCARD) - typename std::enable_if::value, V&>::type getFirst() noexcept { + typename std::enable_if::type getFirst() noexcept { return mData; } - template + template ROBIN_HOOD(NODISCARD) - typename std::enable_if::value, typename V::first_type const&>::type + typename std::enable_if::type getFirst() const noexcept { return mData.first; } - template + template ROBIN_HOOD(NODISCARD) - typename std::enable_if::value, V const&>::type getFirst() const noexcept { + typename std::enable_if::type getFirst() const noexcept { return mData; } - template + template ROBIN_HOOD(NODISCARD) - typename std::enable_if::value, Q&>::type getSecond() noexcept { + typename std::enable_if::type getSecond() noexcept { return mData.second; } - template + template ROBIN_HOOD(NODISCARD) - typename std::enable_if::value, Q const&>::type getSecond() const - noexcept { + typename std::enable_if::type getSecond() const noexcept { return mData.second; } @@ -1026,40 +1010,38 @@ private: return *mData; } - template + template ROBIN_HOOD(NODISCARD) - typename std::enable_if::value, typename V::first_type&>::type - getFirst() noexcept { + typename std::enable_if::type getFirst() noexcept { return mData->first; } - template + template ROBIN_HOOD(NODISCARD) - typename std::enable_if::value, V&>::type getFirst() noexcept { + typename std::enable_if::type getFirst() noexcept { return *mData; } - template + template ROBIN_HOOD(NODISCARD) - typename std::enable_if::value, typename V::first_type const&>::type + typename std::enable_if::type getFirst() const noexcept { return mData->first; } - template + template ROBIN_HOOD(NODISCARD) - typename std::enable_if::value, V const&>::type getFirst() const noexcept { + typename std::enable_if::type getFirst() const noexcept { return *mData; } - template + template ROBIN_HOOD(NODISCARD) - typename std::enable_if::value, Q&>::type getSecond() noexcept { + typename std::enable_if::type getSecond() noexcept { return mData->second; } - template + template ROBIN_HOOD(NODISCARD) - typename std::enable_if::value, Q const&>::type getSecond() const - noexcept { + typename std::enable_if::type getSecond() const noexcept { return mData->second; } @@ -1072,7 +1054,7 @@ private: value_type* mData; }; - using Node = DataNode; + using Node = DataNode; // helpers for doInsert: extract first entry (only const required) ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(Node const& n) const noexcept { @@ -1102,8 +1084,8 @@ private: template struct Cloner { void operator()(M const& source, M& target) const { - auto src = reinterpret_cast(source.mKeyVals); - auto tgt = reinterpret_cast(target.mKeyVals); + auto const* const src = reinterpret_cast(source.mKeyVals); + auto* tgt = reinterpret_cast(target.mKeyVals); auto const numElementsWithBuffer = target.calcNumElementsWithBuffer(target.mMask + 1); std::copy(src, src + target.calcNumBytesTotal(numElementsWithBuffer), tgt); } @@ -1125,7 +1107,7 @@ private: // Destroyer /////////////////////////////////////////////////////// - template + template struct Destroyer {}; template @@ -1191,8 +1173,8 @@ private: // compared to end(). Iter() = default; - // Rule of zero: nothing specified. The conversion constructor is only enabled for iterator - // to const_iterator, so it doesn't accidentally work as a copy ctor. + // Rule of zero: nothing specified. The conversion constructor is only enabled for + // iterator to const_iterator, so it doesn't accidentally work as a copy ctor. // Conversion constructor from iterator to const_iterator. template (mInfo); + size_t n = 0; + while (0U == (n = detail::unaligned_load(mInfo))) { + mInfo += sizeof(size_t); + mKeyVals += sizeof(size_t); + } #if ROBIN_HOOD(LITTLE_ENDIAN) - inc = ROBIN_HOOD_COUNT_TRAILING_ZEROES(n) / 8; + auto inc = ROBIN_HOOD_COUNT_TRAILING_ZEROES(n) / 8; #else - inc = ROBIN_HOOD_COUNT_LEADING_ZEROES(n) / 8; + auto inc = ROBIN_HOOD_COUNT_LEADING_ZEROES(n) / 8; #endif - mInfo += inc; - mKeyVals += inc; - } while (inc == static_cast(sizeof(size_t))); + mInfo += inc; + mKeyVals += inc; } - friend class Table; + friend class Table; NodePtr mKeyVals{nullptr}; uint8_t const* mInfo{nullptr}; }; @@ -1275,9 +1264,9 @@ private: // The upper 1-5 bits need to be a reasonable good hash, to save comparisons. template void keyToIdx(HashKey&& key, size_t* idx, InfoType* info) const { - // for a user-specified hash that is *not* robin_hood::hash, apply robin_hood::hash as an - // additional mixing step. This serves as a bad hash prevention, if the given data is badly - // mixed. + // for a user-specified hash that is *not* robin_hood::hash, apply robin_hood::hash as + // an additional mixing step. This serves as a bad hash prevention, if the given data is + // badly mixed. using Mix = typename std::conditional, hasher>::value, ::tracy::detail::identity_hash, @@ -1313,7 +1302,7 @@ private: idx = startIdx; while (idx != insertion_idx) { - ROBIN_HOOD_COUNT(shiftUp); + ROBIN_HOOD_COUNT(shiftUp) mInfo[idx] = static_cast(mInfo[idx - 1] + mInfoInc); if (ROBIN_HOOD_UNLIKELY(mInfo[idx] + mInfoInc > 0xFF)) { mMaxNumElementsAllowed = 0; @@ -1324,12 +1313,13 @@ private: void shiftDown(size_t idx) noexcept(std::is_nothrow_move_assignable::value) { // until we find one that is either empty or has zero offset. - // TODO(martinus) we don't need to move everything, just the last one for the same bucket. + // TODO(martinus) we don't need to move everything, just the last one for the same + // bucket. mKeyVals[idx].destroy(*this); // until we find one that is either empty or has zero offset. while (mInfo[idx + 1] >= 2 * mInfoInc) { - ROBIN_HOOD_COUNT(shiftDown); + ROBIN_HOOD_COUNT(shiftDown) mInfo[idx] = static_cast(mInfo[idx + 1] - mInfoInc); mKeyVals[idx] = std::move(mKeyVals[idx + 1]); ++idx; @@ -1345,8 +1335,8 @@ private: template ROBIN_HOOD(NODISCARD) size_t findIdx(Other const& key) const { - size_t idx; - InfoType info; + size_t idx{}; + InfoType info{}; keyToIdx(key, &idx, &info); do { @@ -1370,7 +1360,7 @@ private: } void cloneData(const Table& o) { - Cloner()(o, *this); + Cloner()(o, *this); } // inserts a keyval that is guaranteed to be new, e.g. when the hashmap is resized. @@ -1382,8 +1372,8 @@ private: throwOverflowError(); // impossible to reach LCOV_EXCL_LINE } - size_t idx; - InfoType info; + size_t idx{}; + InfoType info{}; keyToIdx(keyval.getFirst(), &idx, &info); // skip forward. Use <= because we are certain that the element is not there. @@ -1423,17 +1413,17 @@ public: using iterator = Iter; using const_iterator = Iter; - // Creates an empty hash map. Nothing is allocated yet, this happens at the first insert. This - // tremendously speeds up ctor & dtor of a map that never receives an element. The penalty is - // payed at the first insert, and not before. Lookup of this empty map works because everybody - // points to DummyInfoByte::b. parameter bucket_count is dictated by the standard, but we can - // ignore it. - explicit Table(size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, const Hash& h = Hash{}, - const KeyEqual& equal = KeyEqual{}) noexcept(noexcept(Hash(h)) && - noexcept(KeyEqual(equal))) + // Creates an empty hash map. Nothing is allocated yet, this happens at the first insert. + // This tremendously speeds up ctor & dtor of a map that never receives an element. The + // penalty is payed at the first insert, and not before. Lookup of this empty map works + // because everybody points to DummyInfoByte::b. parameter bucket_count is dictated by the + // standard, but we can ignore it. + explicit Table( + size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, const Hash& h = Hash{}, + const KeyEqual& equal = KeyEqual{}) noexcept(noexcept(Hash(h)) && noexcept(KeyEqual(equal))) : WHash(h) , WKeyEqual(equal) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) } template @@ -1441,7 +1431,7 @@ public: const Hash& h = Hash{}, const KeyEqual& equal = KeyEqual{}) : WHash(h) , WKeyEqual(equal) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) insert(first, last); } @@ -1450,7 +1440,7 @@ public: const KeyEqual& equal = KeyEqual{}) : WHash(h) , WKeyEqual(equal) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) insert(initlist.begin(), initlist.end()); } @@ -1458,7 +1448,7 @@ public: : WHash(std::move(static_cast(o))) , WKeyEqual(std::move(static_cast(o))) , DataPool(std::move(static_cast(o))) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) if (o.mMask) { mKeyVals = std::move(o.mKeyVals); mInfo = std::move(o.mInfo); @@ -1473,7 +1463,7 @@ public: } Table& operator=(Table&& o) noexcept { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) if (&o != this) { if (o.mMask) { // only move stuff if the other map actually has some data @@ -1503,7 +1493,7 @@ public: : WHash(static_cast(o)) , WKeyEqual(static_cast(o)) , DataPool(static_cast(o)) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) if (!o.empty()) { // not empty: create an exact copy. it is also possible to just iterate through all // elements and insert them, but copying is probably faster. @@ -1523,15 +1513,17 @@ public: } // Creates a copy of the given map. Copy constructor of each entry is used. + // Not sure why clang-tidy thinks this doesn't handle self assignment, it does + // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp) Table& operator=(Table const& o) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) if (&o == this) { // prevent assigning of itself return *this; } - // we keep using the old allocator and not assign the new one, because we want to keep the - // memory available. when it is the same size. + // we keep using the old allocator and not assign the new one, because we want to keep + // the memory available. when it is the same size. if (o.empty()) { if (0 == mMask) { // nothing to do, we are empty too @@ -1550,7 +1542,7 @@ public: } // clean up old stuff - Destroyer::value>{}.nodes(*this); + Destroyer::value>{}.nodes(*this); if (mMask != o.mMask) { // no luck: we don't have the same array size allocated, so we need to realloc. @@ -1582,21 +1574,21 @@ public: // Swaps everything between the two maps. void swap(Table& o) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) using std::swap; swap(o, *this); } // Clears all data, without resizing. void clear() { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) if (empty()) { - // don't do anything! also important because we don't want to write to DummyInfoByte::b, - // even though we would just write 0 to it. + // don't do anything! also important because we don't want to write to + // DummyInfoByte::b, even though we would just write 0 to it. return; } - Destroyer::value>{}.nodes(*this); + Destroyer::value>{}.nodes(*this); auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); // clear everything, then set the sentinel again @@ -1610,19 +1602,18 @@ public: // Destroys the map and all it's contents. ~Table() { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) destroy(); } - // Checks if both maps contain the same entries. Order is irrelevant. + // Checks if both tables contain the same entries. Order is irrelevant. bool operator==(const Table& other) const { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) if (other.size() != size()) { return false; } for (auto const& otherEntry : other) { - auto const myIt = find(otherEntry.first); - if (myIt == end() || !(myIt->second == otherEntry.second)) { + if (!has(otherEntry)) { return false; } } @@ -1631,19 +1622,19 @@ public: } bool operator!=(const Table& other) const { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return !operator==(other); } template typename std::enable_if::value, Q&>::type operator[](const key_type& key) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return doCreateByKey(key); } template typename std::enable_if::value, Q&>::type operator[](key_type&& key) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return doCreateByKey(std::move(key)); } @@ -1657,7 +1648,7 @@ public: template std::pair emplace(Args&&... args) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) Node n{*this, std::forward(args)...}; auto r = doInsert(std::move(n)); if (!r.second) { @@ -1669,7 +1660,7 @@ public: } std::pair insert(const value_type& keyval) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return doInsert(keyval); } @@ -1679,7 +1670,7 @@ public: // Returns 1 if key is found, 0 otherwise. size_t count(const key_type& key) const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) auto kv = mKeyVals + findIdx(key); if (kv != reinterpret_cast_no_cast_align_warning(mInfo)) { return 1; @@ -1687,12 +1678,33 @@ public: return 0; } + template + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if::type count(const OtherKey& key) const { + ROBIN_HOOD_TRACE(this) + auto kv = mKeyVals + findIdx(key); + if (kv != reinterpret_cast_no_cast_align_warning(mInfo)) { + return 1; + } + return 0; + } + + bool contains(const key_type& key) const { // NOLINT(modernize-use-nodiscard) + return 1U == count(key); + } + + template + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if::type contains(const OtherKey& key) const { + return 1U == count(key); + } + // Returns a reference to the value found for key. // Throws std::out_of_range if element cannot be found template // NOLINTNEXTLINE(modernize-use-nodiscard) typename std::enable_if::value, Q&>::type at(key_type const& key) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) auto kv = mKeyVals + findIdx(key); if (kv == reinterpret_cast_no_cast_align_warning(mInfo)) { doThrow("key not found"); @@ -1705,7 +1717,7 @@ public: template // NOLINTNEXTLINE(modernize-use-nodiscard) typename std::enable_if::value, Q const&>::type at(key_type const& key) const { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) auto kv = mKeyVals + findIdx(key); if (kv == reinterpret_cast_no_cast_align_warning(mInfo)) { doThrow("key not found"); @@ -1714,44 +1726,60 @@ public: } const_iterator find(const key_type& key) const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) const size_t idx = findIdx(key); return const_iterator{mKeyVals + idx, mInfo + idx}; } template const_iterator find(const OtherKey& key, is_transparent_tag /*unused*/) const { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return const_iterator{mKeyVals + idx, mInfo + idx}; + } + + template + typename std::enable_if::type // NOLINT(modernize-use-nodiscard) + find(const OtherKey& key) const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) const size_t idx = findIdx(key); return const_iterator{mKeyVals + idx, mInfo + idx}; } iterator find(const key_type& key) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) const size_t idx = findIdx(key); return iterator{mKeyVals + idx, mInfo + idx}; } template iterator find(const OtherKey& key, is_transparent_tag /*unused*/) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return iterator{mKeyVals + idx, mInfo + idx}; + } + + template + typename std::enable_if::type find(const OtherKey& key) { + ROBIN_HOOD_TRACE(this) const size_t idx = findIdx(key); return iterator{mKeyVals + idx, mInfo + idx}; } iterator begin() { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) if (empty()) { return end(); } return iterator(mKeyVals, mInfo, fast_forward_tag{}); } const_iterator begin() const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return cbegin(); } const_iterator cbegin() const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) if (empty()) { return cend(); } @@ -1759,22 +1787,22 @@ public: } iterator end() { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) // no need to supply valid info pointer: end() must not be dereferenced, and only node // pointer is compared. return iterator{reinterpret_cast_no_cast_align_warning(mInfo), nullptr}; } const_iterator end() const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return cend(); } const_iterator cend() const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return const_iterator{reinterpret_cast_no_cast_align_warning(mInfo), nullptr}; } iterator erase(const_iterator pos) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) // its safe to perform const cast here // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) return erase(iterator{const_cast(pos.mKeyVals), const_cast(pos.mInfo)}); @@ -1782,7 +1810,7 @@ public: // Erases element at pos, returns iterator to the next element. iterator erase(iterator pos) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) // we assume that pos always points to a valid entry, and not end(). auto const idx = static_cast(pos.mKeyVals - mKeyVals); @@ -1799,9 +1827,9 @@ public: } size_t erase(const key_type& key) { - ROBIN_HOOD_TRACE(this); - size_t idx; - InfoType info; + ROBIN_HOOD_TRACE(this) + size_t idx{}; + InfoType info{}; keyToIdx(key, &idx, &info); // check while info matches with the source idx @@ -1827,7 +1855,7 @@ public: // reserves space for the specified number of elements. Makes sure the old data fits. // Exactly the same as resize(c). Use resize(0) to shrink to fit. void reserve(size_t c) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) auto const minElementsAllowed = (std::max)(c, mNumElements); auto newSize = InitialNumElements; while (calcMaxNumElementsAllowed(newSize) < minElementsAllowed && newSize != 0) { @@ -1841,33 +1869,33 @@ public: } size_type size() const noexcept { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return mNumElements; } size_type max_size() const noexcept { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return static_cast(-1); } ROBIN_HOOD(NODISCARD) bool empty() const noexcept { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return 0 == mNumElements; } float max_load_factor() const noexcept { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return MaxLoadFactor100 / 100.0F; } // Average number of elements per bucket. Since we allow only 1 per bucket float load_factor() const noexcept { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return static_cast(size()) / static_cast(mMask + 1); } ROBIN_HOOD(NODISCARD) size_t mask() const noexcept { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return mMask; } @@ -1913,10 +1941,25 @@ public: } private: + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::value, bool>::type has(const value_type& e) const { + ROBIN_HOOD_TRACE(this) + auto it = find(e.first); + return it != end() && it->second == e.second; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::value, bool>::type has(const value_type& e) const { + ROBIN_HOOD_TRACE(this) + return find(e) != end(); + } + // reserves space for at least the specified number of elements. // only works if numBuckets if power of two void rehashPowerOfTwo(size_t numBuckets) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) Node* const oldKeyVals = mKeyVals; uint8_t const* const oldInfo = mInfo; @@ -1969,13 +2012,13 @@ private: template typename std::enable_if::value, Q&>::type doCreateByKey(Arg&& key) { while (true) { - size_t idx; - InfoType info; + size_t idx{}; + InfoType info{}; keyToIdx(key, &idx, &info); nextWhileLess(&info, &idx); - // while we potentially have a match. Can't do a do-while here because when mInfo is 0 - // we don't want to skip forward + // while we potentially have a match. Can't do a do-while here because when mInfo is + // 0 we don't want to skip forward while (info == mInfo[idx]) { if (WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) { // key already exists, do not insert. @@ -2004,8 +2047,8 @@ private: auto& l = mKeyVals[insertion_idx]; if (idx == insertion_idx) { - // put at empty spot. This forwards all arguments into the node where the object is - // constructed exactly where it is needed. + // put at empty spot. This forwards all arguments into the node where the object + // is constructed exactly where it is needed. ::new (static_cast(&l)) Node(*this, std::piecewise_construct, std::forward_as_tuple(std::forward(key)), std::forward_as_tuple()); @@ -2027,8 +2070,8 @@ private: template std::pair doInsert(Arg&& keyval) { while (true) { - size_t idx; - InfoType info; + size_t idx{}; + InfoType info{}; keyToIdx(getFirstConst(keyval), &idx, &info); nextWhileLess(&info, &idx); @@ -2080,7 +2123,7 @@ private: bool try_increase_info() { ROBIN_HOOD_LOG("mInfoInc=" << mInfoInc << ", numElements=" << mNumElements << ", maxNumElementsAllowed=" - << calcMaxNumElementsAllowed(mMask + 1)); + << calcMaxNumElementsAllowed(mMask + 1)) if (mInfoInc <= 2) { // need to be > 2 so that shift works (otherwise undefined behavior!) return false; @@ -2120,7 +2163,7 @@ private: ROBIN_HOOD_LOG("mNumElements=" << mNumElements << ", maxNumElementsAllowed=" << maxNumElementsAllowed << ", load=" << (static_cast(mNumElements) * 100.0 / - (static_cast(mMask) + 1))); + (static_cast(mMask) + 1))) // it seems we have a really bad hash function! don't try to resize again if (mNumElements * 2 < calcMaxNumElementsAllowed(mMask + 1)) { throwOverflowError(); @@ -2135,20 +2178,20 @@ private: return; } - Destroyer::value>{} + Destroyer::value>{} .nodesDoNotDeallocate(*this); // This protection against not deleting mMask shouldn't be needed as it's sufficiently // protected with the 0==mMask check, but I have this anyways because g++ 7 otherwise // reports a compile error: attempt to free a non-heap object ‘fm’ // [-Werror=free-nonheap-object] - if (mKeyVals != reinterpret_cast(&mMask)) { + if (mKeyVals != reinterpret_cast_no_cast_align_warning(&mMask)) { free(mKeyVals); } } void init() noexcept { - mKeyVals = reinterpret_cast(&mMask); + mKeyVals = reinterpret_cast_no_cast_align_warning(&mMask); mInfo = reinterpret_cast(&mMask); mNumElements = 0; mMask = 0; @@ -2158,14 +2201,14 @@ private: } // members are sorted so no padding occurs - Node* mKeyVals = reinterpret_cast(&mMask); // 8 byte 8 - uint8_t* mInfo = reinterpret_cast(&mMask); // 8 byte 16 - size_t mNumElements = 0; // 8 byte 24 - size_t mMask = 0; // 8 byte 32 - size_t mMaxNumElementsAllowed = 0; // 8 byte 40 - InfoType mInfoInc = InitialInfoInc; // 4 byte 44 - InfoType mInfoHashShift = InitialInfoHashShift; // 4 byte 48 - // 16 byte 56 if NodeAllocator + Node* mKeyVals = reinterpret_cast_no_cast_align_warning(&mMask); // 8 byte 8 + uint8_t* mInfo = reinterpret_cast(&mMask); // 8 byte 16 + size_t mNumElements = 0; // 8 byte 24 + size_t mMask = 0; // 8 byte 32 + size_t mMaxNumElementsAllowed = 0; // 8 byte 40 + InfoType mInfoInc = InitialInfoInc; // 4 byte 44 + InfoType mInfoHashShift = InitialInfoHashShift; // 4 byte 48 + // 16 byte 56 if NodeAllocator }; } // namespace detail @@ -2198,8 +2241,8 @@ template , typename KeyEqual = std::equa size_t MaxLoadFactor100 = 80> using unordered_node_set = detail::Table; -template , - typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> +template , typename KeyEqual = std::equal_to, + size_t MaxLoadFactor100 = 80> using unordered_set = detail::Table::value && std::is_nothrow_move_assignable::value,