[libc++] Introduce _LIBCPP_COMPRESSED_ELEMENT (#134253)

We have multiple classes with an empty base optimization that contains
just a single type. This patch introduces `_LIBCPP_COMPRESSED_ELEMENT`
to refactor these classes to avoid having them essentially twice,
reducing the amount of code significantly.

libcxx/docs/ReleaseNotes/22.rst

@@ -58,9 +58,10 @@ Improvements and New Features
 - The performance of ``insert(iterator, iterator)`` of ``map``, ``set``, ``multimap`` and ``multiset`` has been improved
   by up to 2.5x
 - The performance of ``map::insert_or_assign`` has been improved by up to 2x
-
 - ``ofstream::write`` has been optimized to pass through large strings to system calls directly instead of copying them
   in chunks into a buffer.
+- Multiple internal types have been refactored to use ``[[no_unique_address]]``, resulting in faster compile times and
+  reduced debug information.
 
 Deprecations and Removals
 -------------------------
@@ -88,5 +89,10 @@ ABI Affecting Changes
   ``_LIBCPP_ABI_NO_ITERATOR_BASES``. If you are using this flag and care about ABI stability, you should set
   ``_LIBCPP_ABI_NO_REVERSE_ITERATOR_SECOND_MEMBER`` as well.
 
+- The internal types ``__map_value_compare``, ``__unordered_map_hasher``, ``__unordered_map_equal``,
+  ``__hash_map_hasher`` and ``__hash_map_equal`` have been refactored to use ``_LIBCPP_COMPRESSED_ELEMENT`` instead of
+  potentially inheriting from the types they wrap. At this point in time we are not aware of any ABI changes caused by
+  this.
+
 Build System Changes
 --------------------

libcxx/include/__memory/compressed_pair.h

@@ -80,6 +80,10 @@ class __compressed_pair_padding {
 template <class _ToPad>
 class __compressed_pair_padding<_ToPad, true> {};
 
+#  define _LIBCPP_COMPRESSED_ELEMENT(T1, Initializer1)                                                                 \
+    _LIBCPP_NO_UNIQUE_ADDRESS T1 Initializer1;                                                                         \
+    _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T1> _LIBCPP_CONCAT3(__padding_, __LINE__, _)
+
 // TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
 #  ifdef _LIBCPP_COMPILER_GCC
 #    define _LIBCPP_COMPRESSED_PAIR(T1, Initializer1, T2, Initializer2)                                                \
@@ -121,6 +125,8 @@ class __compressed_pair_padding<_ToPad, true> {};
 #  endif
 
 #else
+#  define _LIBCPP_COMPRESSED_ELEMENT(T1, Initializer1) _LIBCPP_NO_UNIQUE_ADDRESS T1 Initializer1
+
 #  define _LIBCPP_COMPRESSED_PAIR(T1, Name1, T2, Name2)                                                                \
     _LIBCPP_NO_UNIQUE_ADDRESS T1 Name1;                                                                                \
     _LIBCPP_NO_UNIQUE_ADDRESS T2 Name2

libcxx/include/ext/hash_map

@@ -206,6 +206,7 @@ template <class Key, class T, class Hash, class Pred, class Alloc>
 #else
 #  include <__config>
 #  include <__hash_table>
+#  include <__memory/compressed_pair.h>
 #  include <algorithm>
 #  include <ext/__hash>
 #  include <functional>
@@ -224,21 +225,9 @@ _LIBCPP_WARNING("Use of the header <ext/hash_map> is deprecated.  Migrate to <un
 
 namespace __gnu_cxx {
 
-template <class _Tp, class _Hash, bool = std::is_empty<_Hash>::value && !std::__libcpp_is_final<_Hash>::value >
-class __hash_map_hasher : private _Hash {
-public:
-  _LIBCPP_HIDE_FROM_ABI __hash_map_hasher() : _Hash() {}
-  _LIBCPP_HIDE_FROM_ABI __hash_map_hasher(const _Hash& __h) : _Hash(__h) {}
-  _LIBCPP_HIDE_FROM_ABI const _Hash& hash_function() const { return *this; }
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(const _Tp& __x) const { return static_cast<const _Hash&>(*this)(__x.first); }
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(const typename _Tp::first_type& __x) const {
-    return static_cast<const _Hash&>(*this)(__x);
-  }
-};
-
 template <class _Tp, class _Hash>
-class __hash_map_hasher<_Tp, _Hash, false> {
-  _Hash __hash_;
+class __hash_map_hasher {
+  _LIBCPP_COMPRESSED_ELEMENT(_Hash, __hash_);
 
 public:
   _LIBCPP_HIDE_FROM_ABI __hash_map_hasher() : __hash_() {}
@@ -248,30 +237,9 @@ public:
   _LIBCPP_HIDE_FROM_ABI size_t operator()(const typename _Tp::first_type& __x) const { return __hash_(__x); }
 };
 
-template <class _Tp, class _Pred, bool = std::is_empty<_Pred>::value && !std::__libcpp_is_final<_Pred>::value >
-class __hash_map_equal : private _Pred {
-public:
-  _LIBCPP_HIDE_FROM_ABI __hash_map_equal() : _Pred() {}
-  _LIBCPP_HIDE_FROM_ABI __hash_map_equal(const _Pred& __p) : _Pred(__p) {}
-  _LIBCPP_HIDE_FROM_ABI const _Pred& key_eq() const { return *this; }
-  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const {
-    return static_cast<const _Pred&>(*this)(__x.first, __y.first);
-  }
-  _LIBCPP_HIDE_FROM_ABI bool operator()(const typename _Tp::first_type& __x, const _Tp& __y) const {
-    return static_cast<const _Pred&>(*this)(__x, __y.first);
-  }
-  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const typename _Tp::first_type& __y) const {
-    return static_cast<const _Pred&>(*this)(__x.first, __y);
-  }
-  _LIBCPP_HIDE_FROM_ABI bool
-  operator()(const typename _Tp::first_type& __x, const typename _Tp::first_type& __y) const {
-    return static_cast<const _Pred&>(*this)(__x, __y);
-  }
-};
-
 template <class _Tp, class _Pred>
-class __hash_map_equal<_Tp, _Pred, false> {
-  _Pred __pred_;
+class __hash_map_equal {
+  _LIBCPP_COMPRESSED_ELEMENT(_Pred, __pred_);
 
 public:
   _LIBCPP_HIDE_FROM_ABI __hash_map_equal() : __pred_() {}

libcxx/include/map

@@ -589,6 +589,7 @@ erase_if(multimap<Key, T, Compare, Allocator>& c, Predicate pred);  // C++20
 #  include <__memory/addressof.h>
 #  include <__memory/allocator.h>
 #  include <__memory/allocator_traits.h>
+#  include <__memory/compressed_pair.h>
 #  include <__memory/pointer_traits.h>
 #  include <__memory/unique_ptr.h>
 #  include <__memory_resource/polymorphic_allocator.h>
@@ -633,47 +634,9 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Key,
-          class _CP,
-          class _Compare,
-          bool = is_empty<_Compare>::value && !__libcpp_is_final<_Compare>::value>
-class __map_value_compare : private _Compare {
-public:
-  _LIBCPP_HIDE_FROM_ABI __map_value_compare() _NOEXCEPT_(is_nothrow_default_constructible<_Compare>::value)
-      : _Compare() {}
-  _LIBCPP_HIDE_FROM_ABI __map_value_compare(_Compare __c) _NOEXCEPT_(is_nothrow_copy_constructible<_Compare>::value)
-      : _Compare(__c) {}
-  _LIBCPP_HIDE_FROM_ABI const _Compare& key_comp() const _NOEXCEPT { return *this; }
-  _LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _CP& __y) const {
-    return static_cast<const _Compare&>(*this)(__x.first, __y.first);
-  }
-  _LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _Key& __y) const {
-    return static_cast<const _Compare&>(*this)(__x.first, __y);
-  }
-  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _CP& __y) const {
-    return static_cast<const _Compare&>(*this)(__x, __y.first);
-  }
-  _LIBCPP_HIDE_FROM_ABI void swap(__map_value_compare& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Compare>) {
-    using std::swap;
-    swap(static_cast<_Compare&>(*this), static_cast<_Compare&>(__y));
-  }
-
-#  if _LIBCPP_STD_VER >= 14
-  template <typename _K2>
-  _LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _CP& __y) const {
-    return static_cast<const _Compare&>(*this)(__x, __y.first);
-  }
-
-  template <typename _K2>
-  _LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _K2& __y) const {
-    return static_cast<const _Compare&>(*this)(__x.first, __y);
-  }
-#  endif
-};
-
 template <class _Key, class _CP, class _Compare>
-class __map_value_compare<_Key, _CP, _Compare, false> {
-  _Compare __comp_;
+class __map_value_compare {
+  _LIBCPP_COMPRESSED_ELEMENT(_Compare, __comp_);
 
 public:
   _LIBCPP_HIDE_FROM_ABI __map_value_compare() _NOEXCEPT_(is_nothrow_default_constructible<_Compare>::value)
@@ -685,7 +648,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _CP& __y) const { return __comp_(__x.first, __y.first); }
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _Key& __y) const { return __comp_(__x.first, __y); }
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _CP& __y) const { return __comp_(__x, __y.first); }
-  void swap(__map_value_compare& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Compare>) {
+  _LIBCPP_HIDE_FROM_ABI void swap(__map_value_compare& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Compare>) {
     using std::swap;
     swap(__comp_, __y.__comp_);
   }
@@ -747,9 +710,9 @@ struct __lazy_synth_three_way_comparator<__map_value_compare<_Key, _MapValueT, _
 };
 #  endif // _LIBCPP_STD_VER >= 14
 
-template <class _Key, class _CP, class _Compare, bool __b>
+template <class _Key, class _CP, class _Compare>
 inline _LIBCPP_HIDE_FROM_ABI void
-swap(__map_value_compare<_Key, _CP, _Compare, __b>& __x, __map_value_compare<_Key, _CP, _Compare, __b>& __y)
+swap(__map_value_compare<_Key, _CP, _Compare>& __x, __map_value_compare<_Key, _CP, _Compare>& __y)
     _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
   __x.swap(__y);
 }

libcxx/include/unordered_map

@@ -600,6 +600,7 @@ template <class Key, class T, class Hash, class Pred, class Alloc>
 #  include <__memory/addressof.h>
 #  include <__memory/allocator.h>
 #  include <__memory/allocator_traits.h>
+#  include <__memory/compressed_pair.h>
 #  include <__memory/pointer_traits.h>
 #  include <__memory/unique_ptr.h>
 #  include <__memory_resource/polymorphic_allocator.h>
@@ -643,34 +644,9 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Key,
-          class _Cp,
-          class _Hash,
-          class _Pred,
-          bool = is_empty<_Hash>::value && !__libcpp_is_final<_Hash>::value>
-class __unordered_map_hasher : private _Hash {
-public:
-  _LIBCPP_HIDE_FROM_ABI __unordered_map_hasher() _NOEXCEPT_(is_nothrow_default_constructible<_Hash>::value) : _Hash() {}
-  _LIBCPP_HIDE_FROM_ABI __unordered_map_hasher(const _Hash& __h) _NOEXCEPT_(is_nothrow_copy_constructible<_Hash>::value)
-      : _Hash(__h) {}
-  _LIBCPP_HIDE_FROM_ABI const _Hash& hash_function() const _NOEXCEPT { return *this; }
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(const _Cp& __x) const { return static_cast<const _Hash&>(*this)(__x.first); }
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(const _Key& __x) const { return static_cast<const _Hash&>(*this)(__x); }
-#  if _LIBCPP_STD_VER >= 20
-  template <typename _K2>
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(const _K2& __x) const {
-    return static_cast<const _Hash&>(*this)(__x);
-  }
-#  endif
-  _LIBCPP_HIDE_FROM_ABI void swap(__unordered_map_hasher& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Hash>) {
-    using std::swap;
-    swap(static_cast<_Hash&>(*this), static_cast<_Hash&>(__y));
-  }
-};
-
 template <class _Key, class _Cp, class _Hash, class _Pred>
-class __unordered_map_hasher<_Key, _Cp, _Hash, _Pred, false> {
-  _Hash __hash_;
+class __unordered_map_hasher {
+  _LIBCPP_COMPRESSED_ELEMENT(_Hash, __hash_);
 
 public:
   _LIBCPP_HIDE_FROM_ABI __unordered_map_hasher() _NOEXCEPT_(is_nothrow_default_constructible<_Hash>::value)
@@ -692,60 +668,16 @@ public:
   }
 };
 
-template <class _Key, class _Cp, class _Hash, class _Pred, bool __b>
+template <class _Key, class _Cp, class _Hash, class _Pred>
 inline _LIBCPP_HIDE_FROM_ABI void
-swap(__unordered_map_hasher<_Key, _Cp, _Hash, _Pred, __b>& __x,
-     __unordered_map_hasher<_Key, _Cp, _Hash, _Pred, __b>& __y) _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
+swap(__unordered_map_hasher<_Key, _Cp, _Hash, _Pred>& __x, __unordered_map_hasher<_Key, _Cp, _Hash, _Pred>& __y)
+    _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
   __x.swap(__y);
 }
 
-template <class _Key,
-          class _Cp,
-          class _Pred,
-          class _Hash,
-          bool = is_empty<_Pred>::value && !__libcpp_is_final<_Pred>::value>
-class __unordered_map_equal : private _Pred {
-public:
-  _LIBCPP_HIDE_FROM_ABI __unordered_map_equal() _NOEXCEPT_(is_nothrow_default_constructible<_Pred>::value) : _Pred() {}
-  _LIBCPP_HIDE_FROM_ABI __unordered_map_equal(const _Pred& __p) _NOEXCEPT_(is_nothrow_copy_constructible<_Pred>::value)
-      : _Pred(__p) {}
-  _LIBCPP_HIDE_FROM_ABI const _Pred& key_eq() const _NOEXCEPT { return *this; }
-  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _Cp& __y) const {
-    return static_cast<const _Pred&>(*this)(__x.first, __y.first);
-  }
-  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _Key& __y) const {
-    return static_cast<const _Pred&>(*this)(__x.first, __y);
-  }
-  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _Cp& __y) const {
-    return static_cast<const _Pred&>(*this)(__x, __y.__get_value().first);
-  }
-#  if _LIBCPP_STD_VER >= 20
-  template <typename _K2>
-  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _K2& __y) const {
-    return static_cast<const _Pred&>(*this)(__x.first, __y);
-  }
-  template <typename _K2>
-  _LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _Cp& __y) const {
-    return static_cast<const _Pred&>(*this)(__x, __y.__get_value().first);
-  }
-  template <typename _K2>
-  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _K2& __y) const {
-    return static_cast<const _Pred&>(*this)(__x, __y);
-  }
-  template <typename _K2>
-  _LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _Key& __y) const {
-    return static_cast<const _Pred&>(*this)(__x, __y);
-  }
-#  endif
-  _LIBCPP_HIDE_FROM_ABI void swap(__unordered_map_equal& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Pred>) {
-    using std::swap;
-    swap(static_cast<_Pred&>(*this), static_cast<_Pred&>(__y));
-  }
-};
-
 template <class _Key, class _Cp, class _Pred, class _Hash>
-class __unordered_map_equal<_Key, _Cp, _Pred, _Hash, false> {
-  _Pred __pred_;
+class __unordered_map_equal {
+  _LIBCPP_COMPRESSED_ELEMENT(_Pred, __pred_);
 
 public:
   _LIBCPP_HIDE_FROM_ABI __unordered_map_equal() _NOEXCEPT_(is_nothrow_default_constructible<_Pred>::value)
@@ -780,9 +712,9 @@ public:
   }
 };
 
-template <class _Key, class _Cp, class _Pred, class _Hash, bool __b>
+template <class _Key, class _Cp, class _Pred, class _Hash>
 inline _LIBCPP_HIDE_FROM_ABI void
-swap(__unordered_map_equal<_Key, _Cp, _Pred, _Hash, __b>& __x, __unordered_map_equal<_Key, _Cp, _Pred, _Hash, __b>& __y)
+swap(__unordered_map_equal<_Key, _Cp, _Pred, _Hash>& __x, __unordered_map_equal<_Key, _Cp, _Pred, _Hash>& __y)
     _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
   __x.swap(__y);
 }