diff options
Diffstat (limited to 'absl/container')
42 files changed, 5185 insertions, 1055 deletions
diff --git a/absl/container/BUILD.bazel b/absl/container/BUILD.bazel index 0ba2fa76..b00c30fd 100644 --- a/absl/container/BUILD.bazel +++ b/absl/container/BUILD.bazel @@ -108,7 +108,7 @@ cc_test( cc_binary( name = "fixed_array_benchmark", - testonly = 1, + testonly = True, srcs = ["fixed_array_benchmark.cc"], copts = ABSL_TEST_COPTS + ["$(STACK_FRAME_UNLIMITED)"], linkopts = ABSL_DEFAULT_LINKOPTS, @@ -126,6 +126,7 @@ cc_library( linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ ":compressed_tuple", + "//absl/base:base_internal", "//absl/base:config", "//absl/base:core_headers", "//absl/memory", @@ -151,7 +152,7 @@ cc_library( cc_library( name = "test_allocator", - testonly = 1, + testonly = True, copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, textual_hdrs = ["internal/test_allocator.h"], @@ -181,7 +182,7 @@ cc_test( cc_binary( name = "inlined_vector_benchmark", - testonly = 1, + testonly = True, srcs = ["inlined_vector_benchmark.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -210,7 +211,7 @@ cc_test( cc_library( name = "test_instance_tracker", - testonly = 1, + testonly = True, srcs = ["internal/test_instance_tracker.cc"], hdrs = ["internal/test_instance_tracker.h"], copts = ABSL_DEFAULT_COPTS, @@ -247,11 +248,11 @@ cc_library( linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ ":container_memory", - ":hash_function_defaults", + ":hash_container_defaults", ":raw_hash_map", "//absl/algorithm:container", "//absl/base:core_headers", - "//absl/memory", + "//absl/meta:type_traits", ], ) @@ -264,10 +265,13 @@ cc_test( deps = [ ":flat_hash_map", ":hash_generator_testing", + ":hash_policy_testing", + ":test_allocator", ":unordered_map_constructor_test", ":unordered_map_lookup_test", ":unordered_map_members_test", ":unordered_map_modifiers_test", + "//absl/base:config", "//absl/log:check", "//absl/meta:type_traits", "//absl/types:any", @@ -283,11 +287,12 @@ cc_library( linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ ":container_memory", - ":hash_function_defaults", + ":hash_container_defaults", ":raw_hash_set", "//absl/algorithm:container", "//absl/base:core_headers", "//absl/memory", + "//absl/meta:type_traits", ], ) @@ -301,6 +306,7 @@ cc_test( ":container_memory", ":flat_hash_set", ":hash_generator_testing", + ":test_allocator", ":unordered_set_constructor_test", ":unordered_set_lookup_test", ":unordered_set_members_test", @@ -321,12 +327,13 @@ cc_library( linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ ":container_memory", - ":hash_function_defaults", + ":hash_container_defaults", ":node_slot_policy", ":raw_hash_map", "//absl/algorithm:container", "//absl/base:core_headers", "//absl/memory", + "//absl/meta:type_traits", ], ) @@ -337,13 +344,14 @@ cc_test( linkopts = ABSL_DEFAULT_LINKOPTS, tags = ["no_test_loonix"], deps = [ - ":hash_generator_testing", + ":hash_policy_testing", ":node_hash_map", ":tracked", ":unordered_map_constructor_test", ":unordered_map_lookup_test", ":unordered_map_members_test", ":unordered_map_modifiers_test", + "//absl/base:config", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], @@ -355,12 +363,14 @@ cc_library( copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ - ":hash_function_defaults", + ":container_memory", + ":hash_container_defaults", ":node_slot_policy", ":raw_hash_set", "//absl/algorithm:container", "//absl/base:core_headers", "//absl/memory", + "//absl/meta:type_traits", ], ) @@ -371,11 +381,15 @@ cc_test( linkopts = ABSL_DEFAULT_LINKOPTS, tags = ["no_test_loonix"], deps = [ + ":hash_generator_testing", + ":hash_policy_testing", ":node_hash_set", ":unordered_set_constructor_test", ":unordered_set_lookup_test", ":unordered_set_members_test", ":unordered_set_modifiers_test", + "//absl/base:config", + "//absl/memory", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], @@ -420,13 +434,26 @@ cc_library( "//visibility:private", ], deps = [ + ":common", "//absl/base:config", "//absl/hash", + "//absl/meta:type_traits", "//absl/strings", "//absl/strings:cord", ], ) +cc_library( + name = "hash_container_defaults", + hdrs = ["hash_container_defaults.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":hash_function_defaults", + "//absl/base:config", + ], +) + cc_test( name = "hash_function_defaults_test", srcs = ["internal/hash_function_defaults_test.cc"], @@ -434,6 +461,8 @@ cc_test( linkopts = ABSL_DEFAULT_LINKOPTS, tags = NOTEST_TAGS_MOBILE + ["no_test_loonix"], deps = [ + ":flat_hash_map", + ":flat_hash_set", ":hash_function_defaults", "//absl/hash", "//absl/random", @@ -447,7 +476,7 @@ cc_test( cc_library( name = "hash_generator_testing", - testonly = 1, + testonly = True, srcs = ["internal/hash_generator_testing.cc"], hdrs = ["internal/hash_generator_testing.h"], copts = ABSL_TEST_COPTS, @@ -463,7 +492,7 @@ cc_library( cc_library( name = "hash_policy_testing", - testonly = 1, + testonly = True, hdrs = ["internal/hash_policy_testing.h"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -502,6 +531,7 @@ cc_test( copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ + ":container_memory", ":hash_policy_traits", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", @@ -563,6 +593,7 @@ cc_library( "//absl/base", "//absl/base:config", "//absl/base:core_headers", + "//absl/base:no_destructor", "//absl/base:raw_logging_internal", "//absl/debugging:stacktrace", "//absl/memory", @@ -686,14 +717,18 @@ cc_test( ":hash_policy_testing", ":hashtable_debug", ":hashtablez_sampler", + ":node_hash_set", ":raw_hash_set", ":test_allocator", + ":test_instance_tracker", "//absl/base", "//absl/base:config", "//absl/base:core_headers", "//absl/base:prefetch", + "//absl/functional:function_ref", "//absl/hash", "//absl/log", + "//absl/log:check", "//absl/memory", "//absl/meta:type_traits", "//absl/strings", @@ -704,16 +739,18 @@ cc_test( cc_binary( name = "raw_hash_set_benchmark", - testonly = 1, + testonly = True, srcs = ["internal/raw_hash_set_benchmark.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, tags = ["benchmark"], visibility = ["//visibility:private"], deps = [ + ":container_memory", ":hash_function_defaults", ":raw_hash_set", "//absl/base:raw_logging_internal", + "//absl/random", "//absl/strings:str_format", "@com_github_google_benchmark//:benchmark_main", ], @@ -721,7 +758,7 @@ cc_binary( cc_binary( name = "raw_hash_set_probe_benchmark", - testonly = 1, + testonly = True, srcs = ["internal/raw_hash_set_probe_benchmark.cc"], copts = ABSL_TEST_COPTS, linkopts = select({ @@ -750,6 +787,7 @@ cc_test( copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ + ":container_memory", ":raw_hash_set", ":tracked", "//absl/base:config", @@ -795,7 +833,7 @@ cc_test( cc_binary( name = "layout_benchmark", - testonly = 1, + testonly = True, srcs = ["internal/layout_benchmark.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -811,7 +849,7 @@ cc_binary( cc_library( name = "tracked", - testonly = 1, + testonly = True, hdrs = ["internal/tracked.h"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -822,7 +860,7 @@ cc_library( cc_library( name = "unordered_map_constructor_test", - testonly = 1, + testonly = True, hdrs = ["internal/unordered_map_constructor_test.h"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -835,7 +873,7 @@ cc_library( cc_library( name = "unordered_map_lookup_test", - testonly = 1, + testonly = True, hdrs = ["internal/unordered_map_lookup_test.h"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -848,7 +886,7 @@ cc_library( cc_library( name = "unordered_map_modifiers_test", - testonly = 1, + testonly = True, hdrs = ["internal/unordered_map_modifiers_test.h"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -861,7 +899,7 @@ cc_library( cc_library( name = "unordered_set_constructor_test", - testonly = 1, + testonly = True, hdrs = ["internal/unordered_set_constructor_test.h"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -875,7 +913,7 @@ cc_library( cc_library( name = "unordered_set_members_test", - testonly = 1, + testonly = True, hdrs = ["internal/unordered_set_members_test.h"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -887,7 +925,7 @@ cc_library( cc_library( name = "unordered_map_members_test", - testonly = 1, + testonly = True, hdrs = ["internal/unordered_map_members_test.h"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -899,7 +937,7 @@ cc_library( cc_library( name = "unordered_set_lookup_test", - testonly = 1, + testonly = True, hdrs = ["internal/unordered_set_lookup_test.h"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -912,7 +950,7 @@ cc_library( cc_library( name = "unordered_set_modifiers_test", - testonly = 1, + testonly = True, hdrs = ["internal/unordered_set_modifiers_test.h"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -991,6 +1029,7 @@ cc_library( ":compressed_tuple", ":container_memory", ":layout", + "//absl/base:config", "//absl/base:core_headers", "//absl/base:raw_logging_internal", "//absl/base:throw_delegate", @@ -999,13 +1038,12 @@ cc_library( "//absl/strings", "//absl/strings:cord", "//absl/types:compare", - "//absl/utility", ], ) cc_library( name = "btree_test_common", - testonly = 1, + testonly = True, hdrs = ["btree_test.h"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -1056,7 +1094,7 @@ cc_test( cc_binary( name = "btree_benchmark", - testonly = 1, + testonly = True, srcs = [ "btree_benchmark.cc", ], diff --git a/absl/container/CMakeLists.txt b/absl/container/CMakeLists.txt index 128cc0e9..25831d5f 100644 --- a/absl/container/CMakeLists.txt +++ b/absl/container/CMakeLists.txt @@ -27,10 +27,11 @@ absl_cc_library( LINKOPTS ${ABSL_DEFAULT_LINKOPTS} DEPS - absl::container_common absl::common_policy_traits absl::compare absl::compressed_tuple + absl::config + absl::container_common absl::container_memory absl::cord absl::core_headers @@ -40,7 +41,6 @@ absl_cc_library( absl::strings absl::throw_delegate absl::type_traits - absl::utility ) # Internal-only target, do not depend on directly. @@ -176,6 +176,7 @@ absl_cc_library( COPTS ${ABSL_DEFAULT_COPTS} DEPS + absl::base_internal absl::compressed_tuple absl::config absl::core_headers @@ -213,6 +214,7 @@ absl_cc_library( DEPS absl::config GTest::gmock + TESTONLY ) absl_cc_test( @@ -287,10 +289,10 @@ absl_cc_library( DEPS absl::container_memory absl::core_headers - absl::hash_function_defaults + absl::hash_container_defaults absl::raw_hash_map absl::algorithm_container - absl::memory + absl::type_traits PUBLIC ) @@ -304,8 +306,11 @@ absl_cc_test( DEPS absl::any absl::check + absl::config absl::flat_hash_map absl::hash_generator_testing + absl::hash_policy_testing + absl::test_allocator absl::type_traits absl::unordered_map_constructor_test absl::unordered_map_lookup_test @@ -323,11 +328,12 @@ absl_cc_library( ${ABSL_DEFAULT_COPTS} DEPS absl::container_memory - absl::hash_function_defaults + absl::hash_container_defaults absl::raw_hash_set absl::algorithm_container absl::core_headers absl::memory + absl::type_traits PUBLIC ) @@ -347,6 +353,7 @@ absl_cc_test( absl::hash_generator_testing absl::memory absl::strings + absl::test_allocator absl::unordered_set_constructor_test absl::unordered_set_lookup_test absl::unordered_set_members_test @@ -364,11 +371,12 @@ absl_cc_library( DEPS absl::container_memory absl::core_headers - absl::hash_function_defaults + absl::hash_container_defaults absl::node_slot_policy absl::raw_hash_map absl::algorithm_container absl::memory + absl::type_traits PUBLIC ) @@ -380,7 +388,8 @@ absl_cc_test( COPTS ${ABSL_TEST_COPTS} DEPS - absl::hash_generator_testing + absl::config + absl::hash_policy_testing absl::node_hash_map absl::tracked absl::unordered_map_constructor_test @@ -398,12 +407,14 @@ absl_cc_library( COPTS ${ABSL_DEFAULT_COPTS} DEPS + absl::container_memory absl::core_headers - absl::hash_function_defaults + absl::hash_container_defaults absl::node_slot_policy absl::raw_hash_set absl::algorithm_container absl::memory + absl::type_traits PUBLIC ) @@ -417,7 +428,10 @@ absl_cc_test( "-DUNORDERED_SET_CXX17" DEPS absl::hash_generator_testing + absl::hash_policy_testing + absl::memory absl::node_hash_set + absl::type_traits absl::unordered_set_constructor_test absl::unordered_set_lookup_test absl::unordered_set_members_test @@ -425,6 +439,19 @@ absl_cc_test( GTest::gmock_main ) +absl_cc_library( + NAME + hash_container_defaults + HDRS + "hash_container_defaults.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::hash_function_defaults + PUBLIC +) + # Internal-only target, do not depend on directly. absl_cc_library( NAME @@ -467,9 +494,11 @@ absl_cc_library( ${ABSL_DEFAULT_COPTS} DEPS absl::config + absl::container_common absl::cord absl::hash absl::strings + absl::type_traits PUBLIC ) @@ -483,6 +512,8 @@ absl_cc_test( DEPS absl::cord absl::cord_test_helpers + absl::flat_hash_map + absl::flat_hash_set absl::hash_function_defaults absl::hash absl::random_random @@ -557,6 +588,7 @@ absl_cc_test( COPTS ${ABSL_TEST_COPTS} DEPS + absl::container_memory absl::hash_policy_traits GTest::gmock_main ) @@ -602,6 +634,7 @@ absl_cc_library( absl::base absl::config absl::exponential_biased + absl::no_destructor absl::raw_logging_internal absl::sample_recorder absl::synchronization @@ -743,11 +776,13 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::base + absl::check absl::config absl::container_memory absl::core_headers absl::flat_hash_map absl::flat_hash_set + absl::function_ref absl::hash absl::hash_function_defaults absl::hash_policy_testing @@ -755,10 +790,12 @@ absl_cc_test( absl::hashtablez_sampler absl::log absl::memory + absl::node_hash_set absl::prefetch absl::raw_hash_set absl::strings absl::test_allocator + absl::test_instance_tracker absl::type_traits GTest::gmock_main ) @@ -772,6 +809,7 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::config + absl::container_memory absl::raw_hash_set absl::tracked GTest::gmock_main diff --git a/absl/container/btree_map.h b/absl/container/btree_map.h index 0f62f0bd..b959b674 100644 --- a/absl/container/btree_map.h +++ b/absl/container/btree_map.h @@ -49,6 +49,8 @@ // // Another API difference is that btree iterators can be subtracted, and this // is faster than using std::distance. +// +// B-tree maps are not exception-safe. #ifndef ABSL_CONTAINER_BTREE_MAP_H_ #define ABSL_CONTAINER_BTREE_MAP_H_ @@ -85,7 +87,7 @@ struct map_params; // template <typename Key, typename Value, typename Compare = std::less<Key>, typename Alloc = std::allocator<std::pair<const Key, Value>>> -class btree_map +class ABSL_INTERNAL_ATTRIBUTE_OWNER btree_map : public container_internal::btree_map_container< container_internal::btree<container_internal::map_params< Key, Value, Compare, Alloc, /*TargetNodeSize=*/256, @@ -523,7 +525,7 @@ typename btree_map<K, V, C, A>::size_type erase_if( // template <typename Key, typename Value, typename Compare = std::less<Key>, typename Alloc = std::allocator<std::pair<const Key, Value>>> -class btree_multimap +class ABSL_INTERNAL_ATTRIBUTE_OWNER btree_multimap : public container_internal::btree_multimap_container< container_internal::btree<container_internal::map_params< Key, Value, Compare, Alloc, /*TargetNodeSize=*/256, diff --git a/absl/container/btree_set.h b/absl/container/btree_set.h index 51dc42b7..986d27da 100644 --- a/absl/container/btree_set.h +++ b/absl/container/btree_set.h @@ -48,10 +48,13 @@ // // Another API difference is that btree iterators can be subtracted, and this // is faster than using std::distance. +// +// B-tree sets are not exception-safe. #ifndef ABSL_CONTAINER_BTREE_SET_H_ #define ABSL_CONTAINER_BTREE_SET_H_ +#include "absl/base/attributes.h" #include "absl/container/internal/btree.h" // IWYU pragma: export #include "absl/container/internal/btree_container.h" // IWYU pragma: export @@ -86,7 +89,7 @@ struct set_params; // template <typename Key, typename Compare = std::less<Key>, typename Alloc = std::allocator<Key>> -class btree_set +class ABSL_INTERNAL_ATTRIBUTE_OWNER btree_set : public container_internal::btree_set_container< container_internal::btree<container_internal::set_params< Key, Compare, Alloc, /*TargetNodeSize=*/256, @@ -442,7 +445,7 @@ typename btree_set<K, C, A>::size_type erase_if(btree_set<K, C, A> &set, // template <typename Key, typename Compare = std::less<Key>, typename Alloc = std::allocator<Key>> -class btree_multiset +class ABSL_INTERNAL_ATTRIBUTE_OWNER btree_multiset : public container_internal::btree_multiset_container< container_internal::btree<container_internal::set_params< Key, Compare, Alloc, /*TargetNodeSize=*/256, diff --git a/absl/container/flat_hash_map.h b/absl/container/flat_hash_map.h index acd013b0..ebd9ed67 100644 --- a/absl/container/flat_hash_map.h +++ b/absl/container/flat_hash_map.h @@ -26,21 +26,24 @@ // // In most cases, your default choice for a hash map should be a map of type // `flat_hash_map`. +// +// `flat_hash_map` is not exception-safe. #ifndef ABSL_CONTAINER_FLAT_HASH_MAP_H_ #define ABSL_CONTAINER_FLAT_HASH_MAP_H_ #include <cstddef> -#include <new> +#include <memory> #include <type_traits> #include <utility> #include "absl/algorithm/container.h" +#include "absl/base/attributes.h" #include "absl/base/macros.h" +#include "absl/container/hash_container_defaults.h" #include "absl/container/internal/container_memory.h" -#include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export #include "absl/container/internal/raw_hash_map.h" // IWYU pragma: export -#include "absl/memory/memory.h" +#include "absl/meta/type_traits.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -62,7 +65,7 @@ struct FlatHashMapPolicy; // * Requires values that are MoveConstructible // * Supports heterogeneous lookup, through `find()`, `operator[]()` and // `insert()`, provided that the map is provided a compatible heterogeneous -// hashing function and equality operator. +// hashing function and equality operator. See below for details. // * Invalidates any references and pointers to elements within the table after // `rehash()` and when the table is moved. // * Contains a `capacity()` member function indicating the number of element @@ -80,6 +83,19 @@ struct FlatHashMapPolicy; // libraries (e.g. .dll, .so) is unsupported due to way `absl::Hash` values may // be randomized across dynamically loaded libraries. // +// To achieve heterogeneous lookup for custom types either `Hash` and `Eq` type +// parameters can be used or `T` should have public inner types +// `absl_container_hash` and (optionally) `absl_container_eq`. In either case, +// `typename Hash::is_transparent` and `typename Eq::is_transparent` should be +// well-formed. Both types are basically functors: +// * `Hash` should support `size_t operator()(U val) const` that returns a hash +// for the given `val`. +// * `Eq` should support `bool operator()(U lhs, V rhs) const` that returns true +// if `lhs` is equal to `rhs`. +// +// In most cases `T` needs only to provide the `absl_container_hash`. In this +// case `std::equal_to<void>` will be used instead of `eq` part. +// // NOTE: A `flat_hash_map` stores its value types directly inside its // implementation array to avoid memory indirection. Because a `flat_hash_map` // is designed to move data when rehashed, map values will not retain pointer @@ -106,13 +122,13 @@ struct FlatHashMapPolicy; // if (result != ducks.end()) { // std::cout << "Result: " << result->second << std::endl; // } -template <class K, class V, - class Hash = absl::container_internal::hash_default_hash<K>, - class Eq = absl::container_internal::hash_default_eq<K>, +template <class K, class V, class Hash = DefaultHashContainerHash<K>, + class Eq = DefaultHashContainerEq<K>, class Allocator = std::allocator<std::pair<const K, V>>> -class flat_hash_map : public absl::container_internal::raw_hash_map< - absl::container_internal::FlatHashMapPolicy<K, V>, - Hash, Eq, Allocator> { +class ABSL_INTERNAL_ATTRIBUTE_OWNER flat_hash_map + : public absl::container_internal::raw_hash_map< + absl::container_internal::FlatHashMapPolicy<K, V>, Hash, Eq, + Allocator> { using Base = typename flat_hash_map::raw_hash_map; public: @@ -560,6 +576,38 @@ typename flat_hash_map<K, V, H, E, A>::size_type erase_if( namespace container_internal { +// c_for_each_fast(flat_hash_map<>, Function) +// +// Container-based version of the <algorithm> `std::for_each()` function to +// apply a function to a container's elements. +// There is no guarantees on the order of the function calls. +// Erasure and/or insertion of elements in the function is not allowed. +template <typename K, typename V, typename H, typename E, typename A, + typename Function> +decay_t<Function> c_for_each_fast(const flat_hash_map<K, V, H, E, A>& c, + Function&& f) { + container_internal::ForEach(f, &c); + return f; +} +template <typename K, typename V, typename H, typename E, typename A, + typename Function> +decay_t<Function> c_for_each_fast(flat_hash_map<K, V, H, E, A>& c, + Function&& f) { + container_internal::ForEach(f, &c); + return f; +} +template <typename K, typename V, typename H, typename E, typename A, + typename Function> +decay_t<Function> c_for_each_fast(flat_hash_map<K, V, H, E, A>&& c, + Function&& f) { + container_internal::ForEach(f, &c); + return f; +} + +} // namespace container_internal + +namespace container_internal { + template <class K, class V> struct FlatHashMapPolicy { using slot_policy = container_internal::map_slot_policy<K, V>; @@ -573,9 +621,10 @@ struct FlatHashMapPolicy { slot_policy::construct(alloc, slot, std::forward<Args>(args)...); } + // Returns std::true_type in case destroy is trivial. template <class Allocator> - static void destroy(Allocator* alloc, slot_type* slot) { - slot_policy::destroy(alloc, slot); + static auto destroy(Allocator* alloc, slot_type* slot) { + return slot_policy::destroy(alloc, slot); } template <class Allocator> @@ -592,6 +641,13 @@ struct FlatHashMapPolicy { std::forward<Args>(args)...); } + template <class Hash> + static constexpr HashSlotFn get_hash_slot_fn() { + return memory_internal::IsLayoutCompatible<K, V>::value + ? &TypeErasedApplyToSlotFn<Hash, K> + : nullptr; + } + static size_t space_used(const slot_type*) { return 0; } static std::pair<const K, V>& element(slot_type* slot) { return slot->value; } diff --git a/absl/container/flat_hash_map_test.cc b/absl/container/flat_hash_map_test.cc index d90fe9d5..08915e20 100644 --- a/absl/container/flat_hash_map_test.cc +++ b/absl/container/flat_hash_map_test.cc @@ -16,12 +16,17 @@ #include <cstddef> #include <memory> +#include <string> #include <type_traits> #include <utility> #include <vector> +#include "gmock/gmock.h" #include "gtest/gtest.h" +#include "absl/base/config.h" #include "absl/container/internal/hash_generator_testing.h" +#include "absl/container/internal/hash_policy_testing.h" +#include "absl/container/internal/test_allocator.h" #include "absl/container/internal/unordered_map_constructor_test.h" #include "absl/container/internal/unordered_map_lookup_test.h" #include "absl/container/internal/unordered_map_members_test.h" @@ -40,6 +45,7 @@ using ::testing::_; using ::testing::IsEmpty; using ::testing::Pair; using ::testing::UnorderedElementsAre; +using ::testing::UnorderedElementsAreArray; // Check that absl::flat_hash_map works in a global constructor. struct BeforeMain { @@ -302,6 +308,58 @@ TEST(FlatHashMap, EraseIf) { } } +TEST(FlatHashMap, CForEach) { + flat_hash_map<int, int> m; + std::vector<std::pair<int, int>> expected; + for (int i = 0; i < 100; ++i) { + { + SCOPED_TRACE("mutable object iteration"); + std::vector<std::pair<int, int>> v; + absl::container_internal::c_for_each_fast( + m, [&v](std::pair<const int, int>& p) { v.push_back(p); }); + EXPECT_THAT(v, UnorderedElementsAreArray(expected)); + } + { + SCOPED_TRACE("const object iteration"); + std::vector<std::pair<int, int>> v; + const flat_hash_map<int, int>& cm = m; + absl::container_internal::c_for_each_fast( + cm, [&v](const std::pair<const int, int>& p) { v.push_back(p); }); + EXPECT_THAT(v, UnorderedElementsAreArray(expected)); + } + { + SCOPED_TRACE("const object iteration"); + std::vector<std::pair<int, int>> v; + absl::container_internal::c_for_each_fast( + flat_hash_map<int, int>(m), + [&v](std::pair<const int, int>& p) { v.push_back(p); }); + EXPECT_THAT(v, UnorderedElementsAreArray(expected)); + } + m[i] = i; + expected.emplace_back(i, i); + } +} + +TEST(FlatHashMap, CForEachMutate) { + flat_hash_map<int, int> s; + std::vector<std::pair<int, int>> expected; + for (int i = 0; i < 100; ++i) { + std::vector<std::pair<int, int>> v; + absl::container_internal::c_for_each_fast( + s, [&v](std::pair<const int, int>& p) { + v.push_back(p); + p.second++; + }); + EXPECT_THAT(v, UnorderedElementsAreArray(expected)); + for (auto& p : expected) { + p.second++; + } + EXPECT_THAT(s, UnorderedElementsAreArray(expected)); + s[i] = i; + expected.emplace_back(i, i); + } +} + // This test requires std::launder for mutable key access in node handles. #if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606 TEST(FlatHashMap, NodeHandleMutableKeyAccess) { @@ -351,6 +409,49 @@ TEST(FlatHashMap, RecursiveTypeCompiles) { t.m[0] = RecursiveType{}; } +TEST(FlatHashMap, FlatHashMapPolicyDestroyReturnsTrue) { + EXPECT_TRUE( + (decltype(FlatHashMapPolicy<int, char>::destroy<std::allocator<char>>( + nullptr, nullptr))())); + EXPECT_FALSE( + (decltype(FlatHashMapPolicy<int, char>::destroy<CountingAllocator<char>>( + nullptr, nullptr))())); + EXPECT_FALSE((decltype(FlatHashMapPolicy<int, std::unique_ptr<int>>::destroy< + std::allocator<char>>(nullptr, nullptr))())); +} + +struct InconsistentHashEqType { + InconsistentHashEqType(int v1, int v2) : v1(v1), v2(v2) {} + template <typename H> + friend H AbslHashValue(H h, InconsistentHashEqType t) { + return H::combine(std::move(h), t.v1); + } + bool operator==(InconsistentHashEqType t) const { return v2 == t.v2; } + int v1, v2; +}; + +TEST(Iterator, InconsistentHashEqFunctorsValidation) { + if (!IsAssertEnabled()) GTEST_SKIP() << "Assertions not enabled."; + + absl::flat_hash_map<InconsistentHashEqType, int> m; + for (int i = 0; i < 10; ++i) m[{i, i}] = 1; + // We need to insert multiple times to guarantee that we get the assertion + // because it's possible for the hash to collide with the inserted element + // that has v2==0. In those cases, the new element won't be inserted. + auto insert_conflicting_elems = [&] { + for (int i = 100; i < 20000; ++i) { + EXPECT_EQ((m[{i, 0}]), 1); + } + }; + + const char* crash_message = "hash/eq functors are inconsistent."; +#if defined(__arm__) || defined(__aarch64__) + // On ARM, the crash message is garbled so don't expect a specific message. + crash_message = ""; +#endif + EXPECT_DEATH_IF_SUPPORTED(insert_conflicting_elems(), crash_message); +} + } // namespace } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/flat_hash_set.h b/absl/container/flat_hash_set.h index a94a82a0..a3e36e05 100644 --- a/absl/container/flat_hash_set.h +++ b/absl/container/flat_hash_set.h @@ -26,18 +26,25 @@ // // In most cases, your default choice for a hash set should be a set of type // `flat_hash_set`. +// +// `flat_hash_set` is not exception-safe. + #ifndef ABSL_CONTAINER_FLAT_HASH_SET_H_ #define ABSL_CONTAINER_FLAT_HASH_SET_H_ +#include <cstddef> +#include <memory> #include <type_traits> #include <utility> #include "absl/algorithm/container.h" +#include "absl/base/attributes.h" #include "absl/base/macros.h" +#include "absl/container/hash_container_defaults.h" #include "absl/container/internal/container_memory.h" -#include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export #include "absl/container/internal/raw_hash_set.h" // IWYU pragma: export #include "absl/memory/memory.h" +#include "absl/meta/type_traits.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -58,7 +65,7 @@ struct FlatHashSetPolicy; // * Requires keys that are CopyConstructible // * Supports heterogeneous lookup, through `find()` and `insert()`, provided // that the set is provided a compatible heterogeneous hashing function and -// equality operator. +// equality operator. See below for details. // * Invalidates any references and pointers to elements within the table after // `rehash()` and when the table is moved. // * Contains a `capacity()` member function indicating the number of element @@ -76,6 +83,19 @@ struct FlatHashSetPolicy; // libraries (e.g. .dll, .so) is unsupported due to way `absl::Hash` values may // be randomized across dynamically loaded libraries. // +// To achieve heterogeneous lookup for custom types either `Hash` and `Eq` type +// parameters can be used or `T` should have public inner types +// `absl_container_hash` and (optionally) `absl_container_eq`. In either case, +// `typename Hash::is_transparent` and `typename Eq::is_transparent` should be +// well-formed. Both types are basically functors: +// * `Hash` should support `size_t operator()(U val) const` that returns a hash +// for the given `val`. +// * `Eq` should support `bool operator()(U lhs, V rhs) const` that returns true +// if `lhs` is equal to `rhs`. +// +// In most cases `T` needs only to provide the `absl_container_hash`. In this +// case `std::equal_to<void>` will be used instead of `eq` part. +// // NOTE: A `flat_hash_set` stores its keys directly inside its implementation // array to avoid memory indirection. Because a `flat_hash_set` is designed to // move data when rehashed, set keys will not retain pointer stability. If you @@ -99,10 +119,10 @@ struct FlatHashSetPolicy; // if (ducks.contains("dewey")) { // std::cout << "We found dewey!" << std::endl; // } -template <class T, class Hash = absl::container_internal::hash_default_hash<T>, - class Eq = absl::container_internal::hash_default_eq<T>, +template <class T, class Hash = DefaultHashContainerHash<T>, + class Eq = DefaultHashContainerEq<T>, class Allocator = std::allocator<T>> -class flat_hash_set +class ABSL_INTERNAL_ATTRIBUTE_OWNER flat_hash_set : public absl::container_internal::raw_hash_set< absl::container_internal::FlatHashSetPolicy<T>, Hash, Eq, Allocator> { using Base = typename flat_hash_set::raw_hash_set; @@ -460,6 +480,33 @@ typename flat_hash_set<T, H, E, A>::size_type erase_if( namespace container_internal { +// c_for_each_fast(flat_hash_set<>, Function) +// +// Container-based version of the <algorithm> `std::for_each()` function to +// apply a function to a container's elements. +// There is no guarantees on the order of the function calls. +// Erasure and/or insertion of elements in the function is not allowed. +template <typename T, typename H, typename E, typename A, typename Function> +decay_t<Function> c_for_each_fast(const flat_hash_set<T, H, E, A>& c, + Function&& f) { + container_internal::ForEach(f, &c); + return f; +} +template <typename T, typename H, typename E, typename A, typename Function> +decay_t<Function> c_for_each_fast(flat_hash_set<T, H, E, A>& c, Function&& f) { + container_internal::ForEach(f, &c); + return f; +} +template <typename T, typename H, typename E, typename A, typename Function> +decay_t<Function> c_for_each_fast(flat_hash_set<T, H, E, A>&& c, Function&& f) { + container_internal::ForEach(f, &c); + return f; +} + +} // namespace container_internal + +namespace container_internal { + template <class T> struct FlatHashSetPolicy { using slot_type = T; @@ -473,9 +520,11 @@ struct FlatHashSetPolicy { std::forward<Args>(args)...); } + // Return std::true_type in case destroy is trivial. template <class Allocator> - static void destroy(Allocator* alloc, slot_type* slot) { + static auto destroy(Allocator* alloc, slot_type* slot) { absl::allocator_traits<Allocator>::destroy(*alloc, slot); + return IsDestructionTrivial<Allocator, slot_type>(); } static T& element(slot_type* slot) { return *slot; } @@ -489,6 +538,11 @@ struct FlatHashSetPolicy { } static size_t space_used(const T*) { return 0; } + + template <class Hash> + static constexpr HashSlotFn get_hash_slot_fn() { + return &TypeErasedApplyToSlotFn<Hash, T>; + } }; } // namespace container_internal diff --git a/absl/container/flat_hash_set_test.cc b/absl/container/flat_hash_set_test.cc index a60b4bf5..0dd43269 100644 --- a/absl/container/flat_hash_set_test.cc +++ b/absl/container/flat_hash_set_test.cc @@ -16,6 +16,7 @@ #include <cstdint> #include <memory> +#include <type_traits> #include <utility> #include <vector> @@ -24,6 +25,7 @@ #include "absl/base/config.h" #include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_generator_testing.h" +#include "absl/container/internal/test_allocator.h" #include "absl/container/internal/unordered_set_constructor_test.h" #include "absl/container/internal/unordered_set_lookup_test.h" #include "absl/container/internal/unordered_set_members_test.h" @@ -179,15 +181,46 @@ TEST(FlatHashSet, EraseIf) { } } -class PoisonInline { +TEST(FlatHashSet, CForEach) { + using ValueType = std::pair<int, int>; + flat_hash_set<ValueType> s; + std::vector<ValueType> expected; + for (int i = 0; i < 100; ++i) { + { + SCOPED_TRACE("mutable object iteration"); + std::vector<ValueType> v; + absl::container_internal::c_for_each_fast( + s, [&v](const ValueType& p) { v.push_back(p); }); + ASSERT_THAT(v, UnorderedElementsAreArray(expected)); + } + { + SCOPED_TRACE("const object iteration"); + std::vector<ValueType> v; + const flat_hash_set<ValueType>& cs = s; + absl::container_internal::c_for_each_fast( + cs, [&v](const ValueType& p) { v.push_back(p); }); + ASSERT_THAT(v, UnorderedElementsAreArray(expected)); + } + { + SCOPED_TRACE("temporary object iteration"); + std::vector<ValueType> v; + absl::container_internal::c_for_each_fast( + flat_hash_set<ValueType>(s), + [&v](const ValueType& p) { v.push_back(p); }); + ASSERT_THAT(v, UnorderedElementsAreArray(expected)); + } + s.emplace(i, i); + expected.emplace_back(i, i); + } +} + +class PoisonSoo { int64_t data_; public: - explicit PoisonInline(int64_t d) : data_(d) { - SanitizerPoisonObject(&data_); - } - PoisonInline(const PoisonInline& that) : PoisonInline(*that) {} - ~PoisonInline() { SanitizerUnpoisonObject(&data_); } + explicit PoisonSoo(int64_t d) : data_(d) { SanitizerPoisonObject(&data_); } + PoisonSoo(const PoisonSoo& that) : PoisonSoo(*that) {} + ~PoisonSoo() { SanitizerUnpoisonObject(&data_); } int64_t operator*() const { SanitizerUnpoisonObject(&data_); @@ -196,45 +229,66 @@ class PoisonInline { return ret; } template <typename H> - friend H AbslHashValue(H h, const PoisonInline& pi) { + friend H AbslHashValue(H h, const PoisonSoo& pi) { return H::combine(std::move(h), *pi); } - bool operator==(const PoisonInline& rhs) const { return **this == *rhs; } + bool operator==(const PoisonSoo& rhs) const { return **this == *rhs; } }; -// Tests that we don't touch the poison_ member of PoisonInline. -TEST(FlatHashSet, PoisonInline) { - PoisonInline a(0), b(1); - { // basic usage - flat_hash_set<PoisonInline> set; - set.insert(a); - EXPECT_THAT(set, UnorderedElementsAre(a)); - set.insert(b); - EXPECT_THAT(set, UnorderedElementsAre(a, b)); - set.erase(a); - EXPECT_THAT(set, UnorderedElementsAre(b)); - set.rehash(0); // shrink to inline - EXPECT_THAT(set, UnorderedElementsAre(b)); - } - { // test move constructor from inline to inline - flat_hash_set<PoisonInline> set; - set.insert(a); - flat_hash_set<PoisonInline> set2(std::move(set)); - EXPECT_THAT(set2, UnorderedElementsAre(a)); - } - { // test move assignment from inline to inline - flat_hash_set<PoisonInline> set, set2; - set.insert(a); - set2 = std::move(set); - EXPECT_THAT(set2, UnorderedElementsAre(a)); - } - { // test alloc move constructor from inline to inline - flat_hash_set<PoisonInline> set; - set.insert(a); - flat_hash_set<PoisonInline> set2(std::move(set), - std::allocator<PoisonInline>()); - EXPECT_THAT(set2, UnorderedElementsAre(a)); - } +TEST(FlatHashSet, PoisonSooBasic) { + PoisonSoo a(0), b(1); + flat_hash_set<PoisonSoo> set; + set.insert(a); + EXPECT_THAT(set, UnorderedElementsAre(a)); + set.insert(b); + EXPECT_THAT(set, UnorderedElementsAre(a, b)); + set.erase(a); + EXPECT_THAT(set, UnorderedElementsAre(b)); + set.rehash(0); // Shrink to SOO. + EXPECT_THAT(set, UnorderedElementsAre(b)); +} + +TEST(FlatHashSet, PoisonSooMoveConstructSooToSoo) { + PoisonSoo a(0); + flat_hash_set<PoisonSoo> set; + set.insert(a); + flat_hash_set<PoisonSoo> set2(std::move(set)); + EXPECT_THAT(set2, UnorderedElementsAre(a)); +} + +TEST(FlatHashSet, PoisonSooAllocMoveConstructSooToSoo) { + PoisonSoo a(0); + flat_hash_set<PoisonSoo> set; + set.insert(a); + flat_hash_set<PoisonSoo> set2(std::move(set), std::allocator<PoisonSoo>()); + EXPECT_THAT(set2, UnorderedElementsAre(a)); +} + +TEST(FlatHashSet, PoisonSooMoveAssignFullSooToEmptySoo) { + PoisonSoo a(0); + flat_hash_set<PoisonSoo> set, set2; + set.insert(a); + set2 = std::move(set); + EXPECT_THAT(set2, UnorderedElementsAre(a)); +} + +TEST(FlatHashSet, PoisonSooMoveAssignFullSooToFullSoo) { + PoisonSoo a(0), b(1); + flat_hash_set<PoisonSoo> set, set2; + set.insert(a); + set2.insert(b); + set2 = std::move(set); + EXPECT_THAT(set2, UnorderedElementsAre(a)); +} + +TEST(FlatHashSet, FlatHashSetPolicyDestroyReturnsTrue) { + EXPECT_TRUE((decltype(FlatHashSetPolicy<int>::destroy<std::allocator<int>>( + nullptr, nullptr))())); + EXPECT_FALSE( + (decltype(FlatHashSetPolicy<int>::destroy<CountingAllocator<int>>( + nullptr, nullptr))())); + EXPECT_FALSE((decltype(FlatHashSetPolicy<std::unique_ptr<int>>::destroy< + std::allocator<int>>(nullptr, nullptr))())); } } // namespace diff --git a/absl/container/hash_container_defaults.h b/absl/container/hash_container_defaults.h new file mode 100644 index 00000000..eb944a7c --- /dev/null +++ b/absl/container/hash_container_defaults.h @@ -0,0 +1,45 @@ +// Copyright 2024 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_CONTAINER_HASH_CONTAINER_DEFAULTS_H_ +#define ABSL_CONTAINER_HASH_CONTAINER_DEFAULTS_H_ + +#include "absl/base/config.h" +#include "absl/container/internal/hash_function_defaults.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// DefaultHashContainerHash is a convenience alias for the functor that is used +// by default by Abseil hash-based (unordered) containers for hashing when +// `Hash` type argument is not explicitly specified. +// +// This type alias can be used by generic code that wants to provide more +// flexibility for defining underlying containers. +template <typename T> +using DefaultHashContainerHash = absl::container_internal::hash_default_hash<T>; + +// DefaultHashContainerEq is a convenience alias for the functor that is used by +// default by Abseil hash-based (unordered) containers for equality check when +// `Eq` type argument is not explicitly specified. +// +// This type alias can be used by generic code that wants to provide more +// flexibility for defining underlying containers. +template <typename T> +using DefaultHashContainerEq = absl::container_internal::hash_default_eq<T>; + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_HASH_CONTAINER_DEFAULTS_H_ diff --git a/absl/container/inlined_vector.h b/absl/container/inlined_vector.h index 04e2c385..974b6521 100644 --- a/absl/container/inlined_vector.h +++ b/absl/container/inlined_vector.h @@ -775,7 +775,20 @@ class InlinedVector { ABSL_HARDENING_ASSERT(pos >= begin()); ABSL_HARDENING_ASSERT(pos < end()); + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102329#c2 + // It appears that GCC thinks that since `pos` is a const pointer and may + // point to uninitialized memory at this point, a warning should be + // issued. But `pos` is actually only used to compute an array index to + // write to. +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#pragma GCC diagnostic ignored "-Wuninitialized" +#endif return storage_.Erase(pos, pos + 1); +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic pop +#endif } // Overload of `InlinedVector::erase(...)` that erases every element in the diff --git a/absl/container/inlined_vector_test.cc b/absl/container/inlined_vector_test.cc index 241389ae..6954262e 100644 --- a/absl/container/inlined_vector_test.cc +++ b/absl/container/inlined_vector_test.cc @@ -304,6 +304,86 @@ TEST(UniquePtr, MoveAssign) { } } +// Swapping containers of unique pointers should work fine, with no +// leaks, despite the fact that unique pointers are trivially relocatable but +// not trivially destructible. +// TODO(absl-team): Using unique_ptr here is technically correct, but +// a trivially relocatable struct would be less semantically confusing. +TEST(UniquePtr, Swap) { + for (size_t size1 = 0; size1 < 5; ++size1) { + for (size_t size2 = 0; size2 < 5; ++size2) { + absl::InlinedVector<std::unique_ptr<size_t>, 2> a; + absl::InlinedVector<std::unique_ptr<size_t>, 2> b; + for (size_t i = 0; i < size1; ++i) { + a.push_back(std::make_unique<size_t>(i + 10)); + } + for (size_t i = 0; i < size2; ++i) { + b.push_back(std::make_unique<size_t>(i + 20)); + } + a.swap(b); + ASSERT_THAT(a, SizeIs(size2)); + ASSERT_THAT(b, SizeIs(size1)); + for (size_t i = 0; i < a.size(); ++i) { + ASSERT_THAT(a[i], Pointee(i + 20)); + } + for (size_t i = 0; i < b.size(); ++i) { + ASSERT_THAT(b[i], Pointee(i + 10)); + } + } + } +} + +// Erasing from a container of unique pointers should work fine, with no +// leaks, despite the fact that unique pointers are trivially relocatable but +// not trivially destructible. +// TODO(absl-team): Using unique_ptr here is technically correct, but +// a trivially relocatable struct would be less semantically confusing. +TEST(UniquePtr, EraseSingle) { + for (size_t size = 4; size < 16; ++size) { + absl::InlinedVector<std::unique_ptr<size_t>, 8> a; + for (size_t i = 0; i < size; ++i) { + a.push_back(std::make_unique<size_t>(i)); + } + a.erase(a.begin()); + ASSERT_THAT(a, SizeIs(size - 1)); + for (size_t i = 0; i < size - 1; ++i) { + ASSERT_THAT(a[i], Pointee(i + 1)); + } + a.erase(a.begin() + 2); + ASSERT_THAT(a, SizeIs(size - 2)); + ASSERT_THAT(a[0], Pointee(1)); + ASSERT_THAT(a[1], Pointee(2)); + for (size_t i = 2; i < size - 2; ++i) { + ASSERT_THAT(a[i], Pointee(i + 2)); + } + } +} + +// Erasing from a container of unique pointers should work fine, with no +// leaks, despite the fact that unique pointers are trivially relocatable but +// not trivially destructible. +// TODO(absl-team): Using unique_ptr here is technically correct, but +// a trivially relocatable struct would be less semantically confusing. +TEST(UniquePtr, EraseMulti) { + for (size_t size = 5; size < 16; ++size) { + absl::InlinedVector<std::unique_ptr<size_t>, 8> a; + for (size_t i = 0; i < size; ++i) { + a.push_back(std::make_unique<size_t>(i)); + } + a.erase(a.begin(), a.begin() + 2); + ASSERT_THAT(a, SizeIs(size - 2)); + for (size_t i = 0; i < size - 2; ++i) { + ASSERT_THAT(a[i], Pointee(i + 2)); + } + a.erase(a.begin() + 1, a.begin() + 3); + ASSERT_THAT(a, SizeIs(size - 4)); + ASSERT_THAT(a[0], Pointee(2)); + for (size_t i = 1; i < size - 4; ++i) { + ASSERT_THAT(a[i], Pointee(i + 4)); + } + } +} + // At the end of this test loop, the elements between [erase_begin, erase_end) // should have reference counts == 0, and all others elements should have // reference counts == 1. @@ -783,7 +863,9 @@ TEST(OverheadTest, Storage) { // The union should be absorbing some of the allocation bookkeeping overhead // in the larger vectors, leaving only the size_ field as overhead. - struct T { void* val; }; + struct T { + void* val; + }; size_t expected_overhead = sizeof(T); EXPECT_EQ((2 * expected_overhead), diff --git a/absl/container/internal/btree.h b/absl/container/internal/btree.h index 91df57a3..689e71a5 100644 --- a/absl/container/internal/btree.h +++ b/absl/container/internal/btree.h @@ -53,11 +53,11 @@ #include <functional> #include <iterator> #include <limits> -#include <new> #include <string> #include <type_traits> #include <utility> +#include "absl/base/config.h" #include "absl/base/internal/raw_logging.h" #include "absl/base/macros.h" #include "absl/container/internal/common.h" @@ -70,7 +70,6 @@ #include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "absl/types/compare.h" -#include "absl/utility/utility.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -78,9 +77,10 @@ namespace container_internal { #ifdef ABSL_BTREE_ENABLE_GENERATIONS #error ABSL_BTREE_ENABLE_GENERATIONS cannot be directly set -#elif defined(ABSL_HAVE_ADDRESS_SANITIZER) || \ - defined(ABSL_HAVE_HWADDRESS_SANITIZER) || \ - defined(ABSL_HAVE_MEMORY_SANITIZER) +#elif (defined(ABSL_HAVE_ADDRESS_SANITIZER) || \ + defined(ABSL_HAVE_HWADDRESS_SANITIZER) || \ + defined(ABSL_HAVE_MEMORY_SANITIZER)) && \ + !defined(NDEBUG_SANITIZER) // If defined, performance is important. // When compiled in sanitizer mode, we add generation integers to the nodes and // iterators. When iterators are used, we validate that the container has not // been mutated since the iterator was constructed. @@ -475,7 +475,7 @@ struct SearchResult { // useful information. template <typename V> struct SearchResult<V, false> { - SearchResult() {} + SearchResult() = default; explicit SearchResult(V v) : value(v) {} SearchResult(V v, MatchKind /*match*/) : value(v) {} @@ -580,14 +580,12 @@ class btree_node { using layout_type = absl::container_internal::Layout<btree_node *, uint32_t, field_type, slot_type, btree_node *>; + using leaf_layout_type = typename layout_type::template WithStaticSizes< + /*parent*/ 1, + /*generation*/ BtreeGenerationsEnabled() ? 1 : 0, + /*position, start, finish, max_count*/ 4>; constexpr static size_type SizeWithNSlots(size_type n) { - return layout_type( - /*parent*/ 1, - /*generation*/ BtreeGenerationsEnabled() ? 1 : 0, - /*position, start, finish, max_count*/ 4, - /*slots*/ n, - /*children*/ 0) - .AllocSize(); + return leaf_layout_type(/*slots*/ n, /*children*/ 0).AllocSize(); } // A lower bound for the overhead of fields other than slots in a leaf node. constexpr static size_type MinimumOverhead() { @@ -619,27 +617,22 @@ class btree_node { constexpr static size_type kNodeSlots = kNodeTargetSlots >= kMinNodeSlots ? kNodeTargetSlots : kMinNodeSlots; + using internal_layout_type = typename layout_type::template WithStaticSizes< + /*parent*/ 1, + /*generation*/ BtreeGenerationsEnabled() ? 1 : 0, + /*position, start, finish, max_count*/ 4, /*slots*/ kNodeSlots, + /*children*/ kNodeSlots + 1>; + // The node is internal (i.e. is not a leaf node) if and only if `max_count` // has this value. constexpr static field_type kInternalNodeMaxCount = 0; - constexpr static layout_type Layout(const size_type slot_count, - const size_type child_count) { - return layout_type( - /*parent*/ 1, - /*generation*/ BtreeGenerationsEnabled() ? 1 : 0, - /*position, start, finish, max_count*/ 4, - /*slots*/ slot_count, - /*children*/ child_count); - } // Leaves can have less than kNodeSlots values. - constexpr static layout_type LeafLayout( + constexpr static leaf_layout_type LeafLayout( const size_type slot_count = kNodeSlots) { - return Layout(slot_count, 0); - } - constexpr static layout_type InternalLayout() { - return Layout(kNodeSlots, kNodeSlots + 1); + return leaf_layout_type(slot_count, 0); } + constexpr static auto InternalLayout() { return internal_layout_type(); } constexpr static size_type LeafSize(const size_type slot_count = kNodeSlots) { return LeafLayout(slot_count).AllocSize(); } @@ -1407,9 +1400,9 @@ class btree { copy_or_move_values_in_order(other); } btree(btree &&other) noexcept - : root_(absl::exchange(other.root_, EmptyNode())), + : root_(std::exchange(other.root_, EmptyNode())), rightmost_(std::move(other.rightmost_)), - size_(absl::exchange(other.size_, 0u)) { + size_(std::exchange(other.size_, 0u)) { other.mutable_rightmost() = EmptyNode(); } btree(btree &&other, const allocator_type &alloc) diff --git a/absl/container/internal/common_policy_traits.h b/absl/container/internal/common_policy_traits.h index 57eac678..c521f612 100644 --- a/absl/container/internal/common_policy_traits.h +++ b/absl/container/internal/common_policy_traits.h @@ -45,9 +45,10 @@ struct common_policy_traits { // PRECONDITION: `slot` is INITIALIZED // POSTCONDITION: `slot` is UNINITIALIZED + // Returns std::true_type in case destroy is trivial. template <class Alloc> - static void destroy(Alloc* alloc, slot_type* slot) { - Policy::destroy(alloc, slot); + static auto destroy(Alloc* alloc, slot_type* slot) { + return Policy::destroy(alloc, slot); } // Transfers the `old_slot` to `new_slot`. Any memory allocated by the @@ -63,7 +64,7 @@ struct common_policy_traits { // UNINITIALIZED template <class Alloc> static void transfer(Alloc* alloc, slot_type* new_slot, slot_type* old_slot) { - transfer_impl(alloc, new_slot, old_slot, Rank0{}); + transfer_impl(alloc, new_slot, old_slot, Rank2{}); } // PRECONDITION: `slot` is INITIALIZED @@ -82,23 +83,31 @@ struct common_policy_traits { static constexpr bool transfer_uses_memcpy() { return std::is_same<decltype(transfer_impl<std::allocator<char>>( - nullptr, nullptr, nullptr, Rank0{})), + nullptr, nullptr, nullptr, Rank2{})), + std::true_type>::value; + } + + // Returns true if destroy is trivial and can be omitted. + template <class Alloc> + static constexpr bool destroy_is_trivial() { + return std::is_same<decltype(destroy<Alloc>(nullptr, nullptr)), std::true_type>::value; } private: - // To rank the overloads below for overload resolution. Rank0 is preferred. - struct Rank2 {}; - struct Rank1 : Rank2 {}; - struct Rank0 : Rank1 {}; + // Use go/ranked-overloads for dispatching. + struct Rank0 {}; + struct Rank1 : Rank0 {}; + struct Rank2 : Rank1 {}; // Use auto -> decltype as an enabler. // P::transfer returns std::true_type if transfer uses memcpy (e.g. in // node_slot_policy). template <class Alloc, class P = Policy> static auto transfer_impl(Alloc* alloc, slot_type* new_slot, - slot_type* old_slot, Rank0) - -> decltype(P::transfer(alloc, new_slot, old_slot)) { + slot_type* old_slot, + Rank2) -> decltype(P::transfer(alloc, new_slot, + old_slot)) { return P::transfer(alloc, new_slot, old_slot); } #if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606 @@ -121,7 +130,7 @@ struct common_policy_traits { template <class Alloc> static void transfer_impl(Alloc* alloc, slot_type* new_slot, - slot_type* old_slot, Rank2) { + slot_type* old_slot, Rank0) { construct(alloc, new_slot, std::move(element(old_slot))); destroy(alloc, old_slot); } diff --git a/absl/container/internal/common_policy_traits_test.cc b/absl/container/internal/common_policy_traits_test.cc index faee3e7a..8d8f8baa 100644 --- a/absl/container/internal/common_policy_traits_test.cc +++ b/absl/container/internal/common_policy_traits_test.cc @@ -39,44 +39,59 @@ struct PolicyWithoutOptionalOps { using key_type = Slot; using init_type = Slot; - static std::function<void(void*, Slot*, Slot)> construct; - static std::function<void(void*, Slot*)> destroy; + struct PolicyFunctions { + std::function<void(void*, Slot*, Slot)> construct; + std::function<void(void*, Slot*)> destroy; + std::function<Slot&(Slot*)> element; + }; + + static PolicyFunctions* functions() { + static PolicyFunctions* functions = new PolicyFunctions(); + return functions; + } - static std::function<Slot&(Slot*)> element; + static void construct(void* a, Slot* b, Slot c) { + functions()->construct(a, b, c); + } + static void destroy(void* a, Slot* b) { functions()->destroy(a, b); } + static Slot& element(Slot* b) { return functions()->element(b); } }; -std::function<void(void*, Slot*, Slot)> PolicyWithoutOptionalOps::construct; -std::function<void(void*, Slot*)> PolicyWithoutOptionalOps::destroy; - -std::function<Slot&(Slot*)> PolicyWithoutOptionalOps::element; - struct PolicyWithOptionalOps : PolicyWithoutOptionalOps { - static std::function<void(void*, Slot*, Slot*)> transfer; + struct TransferFunctions { + std::function<void(void*, Slot*, Slot*)> transfer; + }; + + static TransferFunctions* transfer_fn() { + static TransferFunctions* transfer_fn = new TransferFunctions(); + return transfer_fn; + } + static void transfer(void* a, Slot* b, Slot* c) { + transfer_fn()->transfer(a, b, c); + } }; -std::function<void(void*, Slot*, Slot*)> PolicyWithOptionalOps::transfer; -struct PolicyWithMemcpyTransfer : PolicyWithoutOptionalOps { - static std::function<std::true_type(void*, Slot*, Slot*)> transfer; +struct PolicyWithMemcpyTransferAndTrivialDestroy : PolicyWithoutOptionalOps { + static std::true_type transfer(void*, Slot*, Slot*) { return {}; } + static std::true_type destroy(void*, Slot*) { return {}; } }; -std::function<std::true_type(void*, Slot*, Slot*)> - PolicyWithMemcpyTransfer::transfer; struct Test : ::testing::Test { Test() { - PolicyWithoutOptionalOps::construct = [&](void* a1, Slot* a2, Slot a3) { + PolicyWithoutOptionalOps::functions()->construct = [&](void* a1, Slot* a2, + Slot a3) { construct.Call(a1, a2, std::move(a3)); }; - PolicyWithoutOptionalOps::destroy = [&](void* a1, Slot* a2) { + PolicyWithoutOptionalOps::functions()->destroy = [&](void* a1, Slot* a2) { destroy.Call(a1, a2); }; - PolicyWithoutOptionalOps::element = [&](Slot* a1) -> Slot& { + PolicyWithoutOptionalOps::functions()->element = [&](Slot* a1) -> Slot& { return element.Call(a1); }; - PolicyWithOptionalOps::transfer = [&](void* a1, Slot* a2, Slot* a3) { - return transfer.Call(a1, a2, a3); - }; + PolicyWithOptionalOps::transfer_fn()->transfer = + [&](void* a1, Slot* a2, Slot* a3) { return transfer.Call(a1, a2, a3); }; } std::allocator<Slot> alloc; @@ -125,7 +140,15 @@ TEST(TransferUsesMemcpy, Basic) { EXPECT_FALSE( common_policy_traits<PolicyWithOptionalOps>::transfer_uses_memcpy()); EXPECT_TRUE( - common_policy_traits<PolicyWithMemcpyTransfer>::transfer_uses_memcpy()); + common_policy_traits< + PolicyWithMemcpyTransferAndTrivialDestroy>::transfer_uses_memcpy()); +} + +TEST(DestroyIsTrivial, Basic) { + EXPECT_FALSE(common_policy_traits<PolicyWithOptionalOps>::destroy_is_trivial< + std::allocator<char>>()); + EXPECT_TRUE(common_policy_traits<PolicyWithMemcpyTransferAndTrivialDestroy>:: + destroy_is_trivial<std::allocator<char>>()); } } // namespace diff --git a/absl/container/internal/compressed_tuple.h b/absl/container/internal/compressed_tuple.h index 59e70eb2..6db0468d 100644 --- a/absl/container/internal/compressed_tuple.h +++ b/absl/container/internal/compressed_tuple.h @@ -87,11 +87,11 @@ struct Storage { constexpr Storage() = default; template <typename V> explicit constexpr Storage(absl::in_place_t, V&& v) - : value(absl::forward<V>(v)) {} + : value(std::forward<V>(v)) {} constexpr const T& get() const& { return value; } - T& get() & { return value; } - constexpr const T&& get() const&& { return absl::move(*this).value; } - T&& get() && { return std::move(*this).value; } + constexpr T& get() & { return value; } + constexpr const T&& get() const&& { return std::move(*this).value; } + constexpr T&& get() && { return std::move(*this).value; } }; template <typename T, size_t I> @@ -99,13 +99,12 @@ struct ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC Storage<T, I, true> : T { constexpr Storage() = default; template <typename V> - explicit constexpr Storage(absl::in_place_t, V&& v) - : T(absl::forward<V>(v)) {} + explicit constexpr Storage(absl::in_place_t, V&& v) : T(std::forward<V>(v)) {} constexpr const T& get() const& { return *this; } - T& get() & { return *this; } - constexpr const T&& get() const&& { return absl::move(*this); } - T&& get() && { return std::move(*this); } + constexpr T& get() & { return *this; } + constexpr const T&& get() const&& { return std::move(*this); } + constexpr T&& get() && { return std::move(*this); } }; template <typename D, typename I, bool ShouldAnyUseBase> @@ -123,7 +122,7 @@ struct ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTupleImpl< constexpr CompressedTupleImpl() = default; template <typename... Vs> explicit constexpr CompressedTupleImpl(absl::in_place_t, Vs&&... args) - : Storage<Ts, I>(absl::in_place, absl::forward<Vs>(args))... {} + : Storage<Ts, I>(absl::in_place, std::forward<Vs>(args))... {} friend CompressedTuple<Ts...>; }; @@ -135,7 +134,7 @@ struct ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTupleImpl< constexpr CompressedTupleImpl() = default; template <typename... Vs> explicit constexpr CompressedTupleImpl(absl::in_place_t, Vs&&... args) - : Storage<Ts, I, false>(absl::in_place, absl::forward<Vs>(args))... {} + : Storage<Ts, I, false>(absl::in_place, std::forward<Vs>(args))... {} friend CompressedTuple<Ts...>; }; @@ -234,11 +233,11 @@ class ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple bool> = true> explicit constexpr CompressedTuple(First&& first, Vs&&... base) : CompressedTuple::CompressedTupleImpl(absl::in_place, - absl::forward<First>(first), - absl::forward<Vs>(base)...) {} + std::forward<First>(first), + std::forward<Vs>(base)...) {} template <int I> - ElemT<I>& get() & { + constexpr ElemT<I>& get() & { return StorageT<I>::get(); } @@ -248,13 +247,13 @@ class ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple } template <int I> - ElemT<I>&& get() && { + constexpr ElemT<I>&& get() && { return std::move(*this).StorageT<I>::get(); } template <int I> constexpr const ElemT<I>&& get() const&& { - return absl::move(*this).StorageT<I>::get(); + return std::move(*this).StorageT<I>::get(); } }; diff --git a/absl/container/internal/compressed_tuple_test.cc b/absl/container/internal/compressed_tuple_test.cc index 74111f97..c3edf542 100644 --- a/absl/container/internal/compressed_tuple_test.cc +++ b/absl/container/internal/compressed_tuple_test.cc @@ -15,7 +15,11 @@ #include "absl/container/internal/compressed_tuple.h" #include <memory> +#include <set> #include <string> +#include <type_traits> +#include <utility> +#include <vector> #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -27,14 +31,22 @@ // These are declared at global scope purely so that error messages // are smaller and easier to understand. -enum class CallType { kConstRef, kConstMove }; +enum class CallType { kMutableRef, kConstRef, kMutableMove, kConstMove }; template <int> struct Empty { + constexpr CallType value() & { return CallType::kMutableRef; } constexpr CallType value() const& { return CallType::kConstRef; } + constexpr CallType value() && { return CallType::kMutableMove; } constexpr CallType value() const&& { return CallType::kConstMove; } }; +// Unconditionally return an lvalue reference to `t`. +template <typename T> +constexpr T& AsLValue(T&& t) { + return t; +} + template <typename T> struct NotEmpty { T value; @@ -54,6 +66,7 @@ namespace { using absl::test_internal::CopyableMovableInstance; using absl::test_internal::InstanceTracker; +using ::testing::Each; TEST(CompressedTupleTest, Sizeof) { EXPECT_EQ(sizeof(int), sizeof(CompressedTuple<int>)); @@ -70,6 +83,30 @@ TEST(CompressedTupleTest, Sizeof) { sizeof(CompressedTuple<int, Empty<0>, NotEmpty<double>, Empty<1>>)); } +TEST(CompressedTupleTest, PointerToEmpty) { + auto to_void_ptrs = [](const auto&... objs) { + return std::vector<const void*>{static_cast<const void*>(&objs)...}; + }; + { + using Tuple = CompressedTuple<int, Empty<0>>; + EXPECT_EQ(sizeof(int), sizeof(Tuple)); + Tuple t; + EXPECT_THAT(to_void_ptrs(t.get<1>()), Each(&t)); + } + { + using Tuple = CompressedTuple<int, Empty<0>, Empty<1>>; + EXPECT_EQ(sizeof(int), sizeof(Tuple)); + Tuple t; + EXPECT_THAT(to_void_ptrs(t.get<1>(), t.get<2>()), Each(&t)); + } + { + using Tuple = CompressedTuple<int, Empty<0>, Empty<1>, Empty<2>>; + EXPECT_EQ(sizeof(int), sizeof(Tuple)); + Tuple t; + EXPECT_THAT(to_void_ptrs(t.get<1>(), t.get<2>(), t.get<3>()), Each(&t)); + } +} + TEST(CompressedTupleTest, OneMoveOnRValueConstructionTemp) { InstanceTracker tracker; CompressedTuple<CopyableMovableInstance> x1(CopyableMovableInstance(1)); @@ -346,8 +383,24 @@ TEST(CompressedTupleTest, Constexpr) { constexpr int value() const { return v; } int v; }; - constexpr CompressedTuple<int, double, CompressedTuple<int>, Empty<0>> x( - 7, 1.25, CompressedTuple<int>(5), {}); + + using Tuple = CompressedTuple<int, double, CompressedTuple<int>, Empty<0>>; + + constexpr int r0 = + AsLValue(Tuple(1, 0.75, CompressedTuple<int>(9), {})).get<0>(); + constexpr double r1 = + AsLValue(Tuple(1, 0.75, CompressedTuple<int>(9), {})).get<1>(); + constexpr int r2 = + AsLValue(Tuple(1, 0.75, CompressedTuple<int>(9), {})).get<2>().get<0>(); + constexpr CallType r3 = + AsLValue(Tuple(1, 0.75, CompressedTuple<int>(9), {})).get<3>().value(); + + EXPECT_EQ(r0, 1); + EXPECT_EQ(r1, 0.75); + EXPECT_EQ(r2, 9); + EXPECT_EQ(r3, CallType::kMutableRef); + + constexpr Tuple x(7, 1.25, CompressedTuple<int>(5), {}); constexpr int x0 = x.get<0>(); constexpr double x1 = x.get<1>(); constexpr int x2 = x.get<2>().get<0>(); @@ -358,7 +411,18 @@ TEST(CompressedTupleTest, Constexpr) { EXPECT_EQ(x2, 5); EXPECT_EQ(x3, CallType::kConstRef); -#if !defined(__GNUC__) || defined(__clang__) || __GNUC__ > 4 + constexpr int m0 = Tuple(5, 0.25, CompressedTuple<int>(3), {}).get<0>(); + constexpr double m1 = Tuple(5, 0.25, CompressedTuple<int>(3), {}).get<1>(); + constexpr int m2 = + Tuple(5, 0.25, CompressedTuple<int>(3), {}).get<2>().get<0>(); + constexpr CallType m3 = + Tuple(5, 0.25, CompressedTuple<int>(3), {}).get<3>().value(); + + EXPECT_EQ(m0, 5); + EXPECT_EQ(m1, 0.25); + EXPECT_EQ(m2, 3); + EXPECT_EQ(m3, CallType::kMutableMove); + constexpr CompressedTuple<Empty<0>, TrivialStruct, int> trivial = {}; constexpr CallType trivial0 = trivial.get<0>().value(); constexpr int trivial1 = trivial.get<1>().value(); @@ -367,7 +431,6 @@ TEST(CompressedTupleTest, Constexpr) { EXPECT_EQ(trivial0, CallType::kConstRef); EXPECT_EQ(trivial1, 0); EXPECT_EQ(trivial2, 0); -#endif constexpr CompressedTuple<Empty<0>, NonTrivialStruct, absl::optional<int>> non_trivial = {}; @@ -386,8 +449,8 @@ TEST(CompressedTupleTest, Constexpr) { #if defined(__clang__) // An apparent bug in earlier versions of gcc claims these are ambiguous. - constexpr int x2m = absl::move(x.get<2>()).get<0>(); - constexpr CallType x3m = absl::move(x).get<3>().value(); + constexpr int x2m = std::move(x.get<2>()).get<0>(); + constexpr CallType x3m = std::move(x).get<3>().value(); EXPECT_EQ(x2m, 5); EXPECT_EQ(x3m, CallType::kConstMove); #endif diff --git a/absl/container/internal/container_memory.h b/absl/container/internal/container_memory.h index 3262d4eb..ba8e08a2 100644 --- a/absl/container/internal/container_memory.h +++ b/absl/container/internal/container_memory.h @@ -68,6 +68,18 @@ void* Allocate(Alloc* alloc, size_t n) { return p; } +// Returns true if the destruction of the value with given Allocator will be +// trivial. +template <class Allocator, class ValueType> +constexpr auto IsDestructionTrivial() { + constexpr bool result = + std::is_trivially_destructible<ValueType>::value && + std::is_same<typename absl::allocator_traits< + Allocator>::template rebind_alloc<char>, + std::allocator<char>>::value; + return std::integral_constant<bool, result>(); +} + // The pointer must have been previously obtained by calling // Allocate<Alignment>(alloc, n). template <size_t Alignment, class Alloc> @@ -414,12 +426,13 @@ struct map_slot_policy { } template <class Allocator> - static void destroy(Allocator* alloc, slot_type* slot) { + static auto destroy(Allocator* alloc, slot_type* slot) { if (kMutableKeys::value) { absl::allocator_traits<Allocator>::destroy(*alloc, &slot->mutable_value); } else { absl::allocator_traits<Allocator>::destroy(*alloc, &slot->value); } + return IsDestructionTrivial<Allocator, value_type>(); } template <class Allocator> @@ -451,6 +464,26 @@ struct map_slot_policy { } }; +// Type erased function for computing hash of the slot. +using HashSlotFn = size_t (*)(const void* hash_fn, void* slot); + +// Type erased function to apply `Fn` to data inside of the `slot`. +// The data is expected to have type `T`. +template <class Fn, class T> +size_t TypeErasedApplyToSlotFn(const void* fn, void* slot) { + const auto* f = static_cast<const Fn*>(fn); + return (*f)(*static_cast<const T*>(slot)); +} + +// Type erased function to apply `Fn` to data inside of the `*slot_ptr`. +// The data is expected to have type `T`. +template <class Fn, class T> +size_t TypeErasedDerefAndApplyToSlotFn(const void* fn, void* slot_ptr) { + const auto* f = static_cast<const Fn*>(fn); + const T* slot = *static_cast<const T**>(slot_ptr); + return (*f)(*slot); +} + } // namespace container_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/container/internal/container_memory_test.cc b/absl/container/internal/container_memory_test.cc index 90d64bf5..7e4357d5 100644 --- a/absl/container/internal/container_memory_test.cc +++ b/absl/container/internal/container_memory_test.cc @@ -280,6 +280,38 @@ TEST(MapSlotPolicy, TransferReturnsTrue) { } } +TEST(MapSlotPolicy, DestroyReturnsTrue) { + { + using slot_policy = map_slot_policy<int, float>; + EXPECT_TRUE( + (std::is_same<decltype(slot_policy::destroy<std::allocator<char>>( + nullptr, nullptr)), + std::true_type>::value)); + } + { + EXPECT_FALSE(std::is_trivially_destructible<std::unique_ptr<int>>::value); + using slot_policy = map_slot_policy<int, std::unique_ptr<int>>; + EXPECT_TRUE( + (std::is_same<decltype(slot_policy::destroy<std::allocator<char>>( + nullptr, nullptr)), + std::false_type>::value)); + } +} + +TEST(ApplyTest, TypeErasedApplyToSlotFn) { + size_t x = 7; + auto fn = [](size_t v) { return v * 2; }; + EXPECT_EQ((TypeErasedApplyToSlotFn<decltype(fn), size_t>(&fn, &x)), 14); +} + +TEST(ApplyTest, TypeErasedDerefAndApplyToSlotFn) { + size_t x = 7; + auto fn = [](size_t v) { return v * 2; }; + size_t* x_ptr = &x; + EXPECT_EQ( + (TypeErasedDerefAndApplyToSlotFn<decltype(fn), size_t>(&fn, &x_ptr)), 14); +} + } // namespace } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/internal/hash_function_defaults.h b/absl/container/internal/hash_function_defaults.h index a3613b4b..0f07bcfe 100644 --- a/absl/container/internal/hash_function_defaults.h +++ b/absl/container/internal/hash_function_defaults.h @@ -45,14 +45,16 @@ #ifndef ABSL_CONTAINER_INTERNAL_HASH_FUNCTION_DEFAULTS_H_ #define ABSL_CONTAINER_INTERNAL_HASH_FUNCTION_DEFAULTS_H_ -#include <stdint.h> #include <cstddef> +#include <functional> #include <memory> #include <string> #include <type_traits> #include "absl/base/config.h" +#include "absl/container/internal/common.h" #include "absl/hash/hash.h" +#include "absl/meta/type_traits.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" @@ -188,6 +190,71 @@ struct HashEq<std::unique_ptr<T, D>> : HashEq<T*> {}; template <class T> struct HashEq<std::shared_ptr<T>> : HashEq<T*> {}; +template <typename T, typename E = void> +struct HasAbslContainerHash : std::false_type {}; + +template <typename T> +struct HasAbslContainerHash<T, absl::void_t<typename T::absl_container_hash>> + : std::true_type {}; + +template <typename T, typename E = void> +struct HasAbslContainerEq : std::false_type {}; + +template <typename T> +struct HasAbslContainerEq<T, absl::void_t<typename T::absl_container_eq>> + : std::true_type {}; + +template <typename T, typename E = void> +struct AbslContainerEq { + using type = std::equal_to<>; +}; + +template <typename T> +struct AbslContainerEq< + T, typename std::enable_if_t<HasAbslContainerEq<T>::value>> { + using type = typename T::absl_container_eq; +}; + +template <typename T, typename E = void> +struct AbslContainerHash { + using type = void; +}; + +template <typename T> +struct AbslContainerHash< + T, typename std::enable_if_t<HasAbslContainerHash<T>::value>> { + using type = typename T::absl_container_hash; +}; + +// HashEq specialization for user types that provide `absl_container_hash` and +// (optionally) `absl_container_eq`. This specialization allows user types to +// provide heterogeneous lookup without requiring to explicitly specify Hash/Eq +// type arguments in unordered Abseil containers. +// +// Both `absl_container_hash` and `absl_container_eq` should be transparent +// (have inner is_transparent type). While there is no technical reason to +// restrict to transparent-only types, there is also no feasible use case when +// it shouldn't be transparent - it is easier to relax the requirement later if +// such a case arises rather than restricting it. +// +// If type provides only `absl_container_hash` then `eq` part will be +// `std::equal_to<void>`. +// +// User types are not allowed to provide only a `Eq` part as there is no +// feasible use case for this behavior - if Hash should be a default one then Eq +// should be an equivalent to the `std::equal_to<T>`. +template <typename T> +struct HashEq<T, typename std::enable_if_t<HasAbslContainerHash<T>::value>> { + using Hash = typename AbslContainerHash<T>::type; + using Eq = typename AbslContainerEq<T>::type; + static_assert(IsTransparent<Hash>::value, + "absl_container_hash must be transparent. To achieve it add a " + "`using is_transparent = void;` clause to this type."); + static_assert(IsTransparent<Eq>::value, + "absl_container_eq must be transparent. To achieve it add a " + "`using is_transparent = void;` clause to this type."); +}; + // This header's visibility is restricted. If you need to access the default // hasher please use the container's ::hasher alias instead. // diff --git a/absl/container/internal/hash_function_defaults_test.cc b/absl/container/internal/hash_function_defaults_test.cc index c31af3be..912d1190 100644 --- a/absl/container/internal/hash_function_defaults_test.cc +++ b/absl/container/internal/hash_function_defaults_test.cc @@ -14,11 +14,15 @@ #include "absl/container/internal/hash_function_defaults.h" +#include <cstddef> #include <functional> #include <type_traits> #include <utility> #include "gtest/gtest.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/hash/hash.h" #include "absl/random/random.h" #include "absl/strings/cord.h" #include "absl/strings/cord_test_helpers.h" @@ -476,26 +480,157 @@ struct StringLikeTest : public ::testing::Test { hash_default_hash<typename T::first_type> hash; }; -TYPED_TEST_SUITE_P(StringLikeTest); +TYPED_TEST_SUITE(StringLikeTest, StringTypesCartesianProduct); -TYPED_TEST_P(StringLikeTest, Eq) { +TYPED_TEST(StringLikeTest, Eq) { EXPECT_TRUE(this->eq(this->a1, this->b1)); EXPECT_TRUE(this->eq(this->b1, this->a1)); } -TYPED_TEST_P(StringLikeTest, NotEq) { +TYPED_TEST(StringLikeTest, NotEq) { EXPECT_FALSE(this->eq(this->a1, this->b2)); EXPECT_FALSE(this->eq(this->b2, this->a1)); } -TYPED_TEST_P(StringLikeTest, HashEq) { +TYPED_TEST(StringLikeTest, HashEq) { EXPECT_EQ(this->hash(this->a1), this->hash(this->b1)); EXPECT_EQ(this->hash(this->a2), this->hash(this->b2)); // It would be a poor hash function which collides on these strings. EXPECT_NE(this->hash(this->a1), this->hash(this->b2)); } -TYPED_TEST_SUITE(StringLikeTest, StringTypesCartesianProduct); +struct TypeWithAbslContainerHash { + struct absl_container_hash { + using is_transparent = void; + + size_t operator()(const TypeWithAbslContainerHash& foo) const { + return absl::HashOf(foo.value); + } + + // Extra overload to test that heterogeneity works for this hasher. + size_t operator()(int value) const { return absl::HashOf(value); } + }; + + friend bool operator==(const TypeWithAbslContainerHash& lhs, + const TypeWithAbslContainerHash& rhs) { + return lhs.value == rhs.value; + } + + friend bool operator==(const TypeWithAbslContainerHash& lhs, int rhs) { + return lhs.value == rhs; + } + + int value; + int noise; +}; + +struct TypeWithAbslContainerHashAndEq { + struct absl_container_hash { + using is_transparent = void; + + size_t operator()(const TypeWithAbslContainerHashAndEq& foo) const { + return absl::HashOf(foo.value); + } + + // Extra overload to test that heterogeneity works for this hasher. + size_t operator()(int value) const { return absl::HashOf(value); } + }; + + struct absl_container_eq { + using is_transparent = void; + + bool operator()(const TypeWithAbslContainerHashAndEq& lhs, + const TypeWithAbslContainerHashAndEq& rhs) const { + return lhs.value == rhs.value; + } + + // Extra overload to test that heterogeneity works for this eq. + bool operator()(const TypeWithAbslContainerHashAndEq& lhs, int rhs) const { + return lhs.value == rhs; + } + }; + + template <typename T> + bool operator==(T&& other) const = delete; + + int value; + int noise; +}; + +using AbslContainerHashTypes = + Types<TypeWithAbslContainerHash, TypeWithAbslContainerHashAndEq>; + +template <typename T> +using AbslContainerHashTest = ::testing::Test; + +TYPED_TEST_SUITE(AbslContainerHashTest, AbslContainerHashTypes); + +TYPED_TEST(AbslContainerHashTest, HasherWorks) { + hash_default_hash<TypeParam> hasher; + + TypeParam foo1{/*value=*/1, /*noise=*/100}; + TypeParam foo1_copy{/*value=*/1, /*noise=*/20}; + TypeParam foo2{/*value=*/2, /*noise=*/100}; + + EXPECT_EQ(hasher(foo1), absl::HashOf(1)); + EXPECT_EQ(hasher(foo2), absl::HashOf(2)); + EXPECT_EQ(hasher(foo1), hasher(foo1_copy)); + + // Heterogeneity works. + EXPECT_EQ(hasher(foo1), hasher(1)); + EXPECT_EQ(hasher(foo2), hasher(2)); +} + +TYPED_TEST(AbslContainerHashTest, EqWorks) { + hash_default_eq<TypeParam> eq; + + TypeParam foo1{/*value=*/1, /*noise=*/100}; + TypeParam foo1_copy{/*value=*/1, /*noise=*/20}; + TypeParam foo2{/*value=*/2, /*noise=*/100}; + + EXPECT_TRUE(eq(foo1, foo1_copy)); + EXPECT_FALSE(eq(foo1, foo2)); + + // Heterogeneity works. + EXPECT_TRUE(eq(foo1, 1)); + EXPECT_FALSE(eq(foo1, 2)); +} + +TYPED_TEST(AbslContainerHashTest, HeterogeneityInMapWorks) { + absl::flat_hash_map<TypeParam, int> map; + + TypeParam foo1{/*value=*/1, /*noise=*/100}; + TypeParam foo1_copy{/*value=*/1, /*noise=*/20}; + TypeParam foo2{/*value=*/2, /*noise=*/100}; + TypeParam foo3{/*value=*/3, /*noise=*/100}; + + map[foo1] = 1; + map[foo2] = 2; + + EXPECT_TRUE(map.contains(foo1_copy)); + EXPECT_EQ(map.at(foo1_copy), 1); + EXPECT_TRUE(map.contains(1)); + EXPECT_EQ(map.at(1), 1); + EXPECT_TRUE(map.contains(2)); + EXPECT_EQ(map.at(2), 2); + EXPECT_FALSE(map.contains(foo3)); + EXPECT_FALSE(map.contains(3)); +} + +TYPED_TEST(AbslContainerHashTest, HeterogeneityInSetWorks) { + absl::flat_hash_set<TypeParam> set; + + TypeParam foo1{/*value=*/1, /*noise=*/100}; + TypeParam foo1_copy{/*value=*/1, /*noise=*/20}; + TypeParam foo2{/*value=*/2, /*noise=*/100}; + + set.insert(foo1); + + EXPECT_TRUE(set.contains(foo1_copy)); + EXPECT_TRUE(set.contains(1)); + EXPECT_FALSE(set.contains(foo2)); + EXPECT_FALSE(set.contains(2)); +} } // namespace } // namespace container_internal @@ -503,7 +638,7 @@ ABSL_NAMESPACE_END } // namespace absl enum Hash : size_t { - kStd = 0x1, // std::hash + kStd = 0x1, // std::hash #ifdef _MSC_VER kExtension = kStd, // In MSVC, std::hash == ::hash #else // _MSC_VER diff --git a/absl/container/internal/hash_policy_testing.h b/absl/container/internal/hash_policy_testing.h index 01c40d2e..66bb12ec 100644 --- a/absl/container/internal/hash_policy_testing.h +++ b/absl/container/internal/hash_policy_testing.h @@ -174,8 +174,7 @@ ABSL_NAMESPACE_END // From GCC-4.9 Changelog: (src: https://gcc.gnu.org/gcc-4.9/changes.html) // "the unordered associative containers in <unordered_map> and <unordered_set> // meet the allocator-aware container requirements;" -#if (defined(__GLIBCXX__) && __GLIBCXX__ <= 20140425 ) || \ -( __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 9 )) +#if defined(__GLIBCXX__) && __GLIBCXX__ <= 20140425 #define ABSL_UNORDERED_SUPPORTS_ALLOC_CTORS 0 #else #define ABSL_UNORDERED_SUPPORTS_ALLOC_CTORS 1 diff --git a/absl/container/internal/hash_policy_traits.h b/absl/container/internal/hash_policy_traits.h index 164ec123..ad835d6f 100644 --- a/absl/container/internal/hash_policy_traits.h +++ b/absl/container/internal/hash_policy_traits.h @@ -148,6 +148,56 @@ struct hash_policy_traits : common_policy_traits<Policy> { static auto value(T* elem) -> decltype(P::value(elem)) { return P::value(elem); } + + using HashSlotFn = size_t (*)(const void* hash_fn, void* slot); + + template <class Hash> + static constexpr HashSlotFn get_hash_slot_fn() { +// get_hash_slot_fn may return nullptr to signal that non type erased function +// should be used. GCC warns against comparing function address with nullptr. +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +// silent error: the address of * will never be NULL [-Werror=address] +#pragma GCC diagnostic ignored "-Waddress" +#endif + return Policy::template get_hash_slot_fn<Hash>() == nullptr + ? &hash_slot_fn_non_type_erased<Hash> + : Policy::template get_hash_slot_fn<Hash>(); +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif + } + + // Whether small object optimization is enabled. True by default. + static constexpr bool soo_enabled() { return soo_enabled_impl(Rank1{}); } + + private: + template <class Hash> + struct HashElement { + template <class K, class... Args> + size_t operator()(const K& key, Args&&...) const { + return h(key); + } + const Hash& h; + }; + + template <class Hash> + static size_t hash_slot_fn_non_type_erased(const void* hash_fn, void* slot) { + return Policy::apply(HashElement<Hash>{*static_cast<const Hash*>(hash_fn)}, + Policy::element(static_cast<slot_type*>(slot))); + } + + // Use go/ranked-overloads for dispatching. Rank1 is preferred. + struct Rank0 {}; + struct Rank1 : Rank0 {}; + + // Use auto -> decltype as an enabler. + template <class P = Policy> + static constexpr auto soo_enabled_impl(Rank1) -> decltype(P::soo_enabled()) { + return P::soo_enabled(); + } + + static constexpr bool soo_enabled_impl(Rank0) { return true; } }; } // namespace container_internal diff --git a/absl/container/internal/hash_policy_traits_test.cc b/absl/container/internal/hash_policy_traits_test.cc index 82d7cc3a..2d2c7c2c 100644 --- a/absl/container/internal/hash_policy_traits_test.cc +++ b/absl/container/internal/hash_policy_traits_test.cc @@ -14,12 +14,14 @@ #include "absl/container/internal/hash_policy_traits.h" +#include <cstddef> #include <functional> #include <memory> #include <new> #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "absl/container/internal/container_memory.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -42,6 +44,11 @@ struct PolicyWithoutOptionalOps { static int apply(int v) { return apply_impl(v); } static std::function<int(int)> apply_impl; static std::function<Slot&(Slot*)> value; + + template <class Hash> + static constexpr HashSlotFn get_hash_slot_fn() { + return nullptr; + } }; std::function<int(int)> PolicyWithoutOptionalOps::apply_impl; @@ -74,6 +81,63 @@ TEST_F(Test, value) { EXPECT_EQ(&b, &hash_policy_traits<PolicyWithoutOptionalOps>::value(&a)); } +struct Hash { + size_t operator()(Slot a) const { return static_cast<size_t>(a) * 5; } +}; + +struct PolicyNoHashFn { + using slot_type = Slot; + using key_type = Slot; + using init_type = Slot; + + static size_t* apply_called_count; + + static Slot& element(Slot* slot) { return *slot; } + template <typename Fn> + static size_t apply(const Fn& fn, int v) { + ++(*apply_called_count); + return fn(v); + } + + template <class Hash> + static constexpr HashSlotFn get_hash_slot_fn() { + return nullptr; + } +}; + +size_t* PolicyNoHashFn::apply_called_count; + +struct PolicyCustomHashFn : PolicyNoHashFn { + template <class Hash> + static constexpr HashSlotFn get_hash_slot_fn() { + return &TypeErasedApplyToSlotFn<Hash, int>; + } +}; + +TEST(HashTest, PolicyNoHashFn_get_hash_slot_fn) { + size_t apply_called_count = 0; + PolicyNoHashFn::apply_called_count = &apply_called_count; + + Hash hasher; + Slot value = 7; + auto* fn = hash_policy_traits<PolicyNoHashFn>::get_hash_slot_fn<Hash>(); + EXPECT_NE(fn, nullptr); + EXPECT_EQ(fn(&hasher, &value), hasher(value)); + EXPECT_EQ(apply_called_count, 1); +} + +TEST(HashTest, PolicyCustomHashFn_get_hash_slot_fn) { + size_t apply_called_count = 0; + PolicyNoHashFn::apply_called_count = &apply_called_count; + + Hash hasher; + Slot value = 7; + auto* fn = hash_policy_traits<PolicyCustomHashFn>::get_hash_slot_fn<Hash>(); + EXPECT_EQ(fn, PolicyCustomHashFn::get_hash_slot_fn<Hash>()); + EXPECT_EQ(fn(&hasher, &value), hasher(value)); + EXPECT_EQ(apply_called_count, 0); +} + } // namespace } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/internal/hashtablez_sampler.cc b/absl/container/internal/hashtablez_sampler.cc index 79a0973a..fd21d966 100644 --- a/absl/container/internal/hashtablez_sampler.cc +++ b/absl/container/internal/hashtablez_sampler.cc @@ -18,12 +18,18 @@ #include <atomic> #include <cassert> #include <cmath> +#include <cstddef> +#include <cstdint> #include <functional> #include <limits> #include "absl/base/attributes.h" #include "absl/base/config.h" +#include "absl/base/internal/per_thread_tls.h" #include "absl/base/internal/raw_logging.h" +#include "absl/base/macros.h" +#include "absl/base/no_destructor.h" +#include "absl/base/optimization.h" #include "absl/debugging/stacktrace.h" #include "absl/memory/memory.h" #include "absl/profiling/internal/exponential_biased.h" @@ -64,7 +70,7 @@ ABSL_PER_THREAD_TLS_KEYWORD SamplingState global_next_sample = {0, 0}; #endif // defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) HashtablezSampler& GlobalHashtablezSampler() { - static auto* sampler = new HashtablezSampler(); + static absl::NoDestructor<HashtablezSampler> sampler; return *sampler; } @@ -72,7 +78,10 @@ HashtablezInfo::HashtablezInfo() = default; HashtablezInfo::~HashtablezInfo() = default; void HashtablezInfo::PrepareForSampling(int64_t stride, - size_t inline_element_size_value) { + size_t inline_element_size_value, + size_t key_size_value, + size_t value_size_value, + uint16_t soo_capacity_value) { capacity.store(0, std::memory_order_relaxed); size.store(0, std::memory_order_relaxed); num_erases.store(0, std::memory_order_relaxed); @@ -92,6 +101,9 @@ void HashtablezInfo::PrepareForSampling(int64_t stride, depth = absl::GetStackTrace(stack, HashtablezInfo::kMaxStackDepth, /* skip_count= */ 0); inline_element_size = inline_element_size_value; + key_size = key_size_value; + value_size = value_size_value; + soo_capacity = soo_capacity_value; } static bool ShouldForceSampling() { @@ -115,12 +127,13 @@ static bool ShouldForceSampling() { } HashtablezInfo* SampleSlow(SamplingState& next_sample, - size_t inline_element_size) { + size_t inline_element_size, size_t key_size, + size_t value_size, uint16_t soo_capacity) { if (ABSL_PREDICT_FALSE(ShouldForceSampling())) { next_sample.next_sample = 1; const int64_t old_stride = exchange(next_sample.sample_stride, 1); - HashtablezInfo* result = - GlobalHashtablezSampler().Register(old_stride, inline_element_size); + HashtablezInfo* result = GlobalHashtablezSampler().Register( + old_stride, inline_element_size, key_size, value_size, soo_capacity); return result; } @@ -150,10 +163,12 @@ HashtablezInfo* SampleSlow(SamplingState& next_sample, // that case. if (first) { if (ABSL_PREDICT_TRUE(--next_sample.next_sample > 0)) return nullptr; - return SampleSlow(next_sample, inline_element_size); + return SampleSlow(next_sample, inline_element_size, key_size, value_size, + soo_capacity); } - return GlobalHashtablezSampler().Register(old_stride, inline_element_size); + return GlobalHashtablezSampler().Register(old_stride, inline_element_size, + key_size, value_size, soo_capacity); #endif } diff --git a/absl/container/internal/hashtablez_sampler.h b/absl/container/internal/hashtablez_sampler.h index e41ee2d7..d74acf8c 100644 --- a/absl/container/internal/hashtablez_sampler.h +++ b/absl/container/internal/hashtablez_sampler.h @@ -40,15 +40,20 @@ #define ABSL_CONTAINER_INTERNAL_HASHTABLEZ_SAMPLER_H_ #include <atomic> +#include <cstddef> +#include <cstdint> #include <functional> #include <memory> #include <vector> +#include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/internal/per_thread_tls.h" #include "absl/base/optimization.h" +#include "absl/base/thread_annotations.h" #include "absl/profiling/internal/sample_recorder.h" #include "absl/synchronization/mutex.h" +#include "absl/time/time.h" #include "absl/utility/utility.h" namespace absl { @@ -67,7 +72,9 @@ struct HashtablezInfo : public profiling_internal::Sample<HashtablezInfo> { // Puts the object into a clean state, fills in the logically `const` members, // blocking for any readers that are currently sampling the object. - void PrepareForSampling(int64_t stride, size_t inline_element_size_value) + void PrepareForSampling(int64_t stride, size_t inline_element_size_value, + size_t key_size, size_t value_size, + uint16_t soo_capacity_value) ABSL_EXCLUSIVE_LOCKS_REQUIRED(init_mu); // These fields are mutated by the various Record* APIs and need to be @@ -91,8 +98,15 @@ struct HashtablezInfo : public profiling_internal::Sample<HashtablezInfo> { static constexpr int kMaxStackDepth = 64; absl::Time create_time; int32_t depth; + // The SOO capacity for this table in elements (not bytes). Note that sampled + // tables are never SOO because we need to store the infoz handle on the heap. + // Tables that would be SOO if not sampled should have: soo_capacity > 0 && + // size <= soo_capacity && max_reserve <= soo_capacity. + uint16_t soo_capacity; void* stack[kMaxStackDepth]; - size_t inline_element_size; // How big is the slot? + size_t inline_element_size; // How big is the slot in bytes? + size_t key_size; // sizeof(key_type) + size_t value_size; // sizeof(value_type) }; void RecordRehashSlow(HashtablezInfo* info, size_t total_probe_length); @@ -117,7 +131,8 @@ struct SamplingState { }; HashtablezInfo* SampleSlow(SamplingState& next_sample, - size_t inline_element_size); + size_t inline_element_size, size_t key_size, + size_t value_size, uint16_t soo_capacity); void UnsampleSlow(HashtablezInfo* info); #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) @@ -204,16 +219,19 @@ class HashtablezInfoHandle { extern ABSL_PER_THREAD_TLS_KEYWORD SamplingState global_next_sample; #endif // defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) -// Returns an RAII sampling handle that manages registration and unregistation -// with the global sampler. +// Returns a sampling handle. inline HashtablezInfoHandle Sample( - size_t inline_element_size ABSL_ATTRIBUTE_UNUSED) { + ABSL_ATTRIBUTE_UNUSED size_t inline_element_size, + ABSL_ATTRIBUTE_UNUSED size_t key_size, + ABSL_ATTRIBUTE_UNUSED size_t value_size, + ABSL_ATTRIBUTE_UNUSED uint16_t soo_capacity) { #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) if (ABSL_PREDICT_TRUE(--global_next_sample.next_sample > 0)) { return HashtablezInfoHandle(nullptr); } - return HashtablezInfoHandle( - SampleSlow(global_next_sample, inline_element_size)); + return HashtablezInfoHandle(SampleSlow(global_next_sample, + inline_element_size, key_size, + value_size, soo_capacity)); #else return HashtablezInfoHandle(nullptr); #endif // !ABSL_PER_THREAD_TLS diff --git a/absl/container/internal/hashtablez_sampler_test.cc b/absl/container/internal/hashtablez_sampler_test.cc index 8ebb08da..24d3bc48 100644 --- a/absl/container/internal/hashtablez_sampler_test.cc +++ b/absl/container/internal/hashtablez_sampler_test.cc @@ -15,8 +15,12 @@ #include "absl/container/internal/hashtablez_sampler.h" #include <atomic> +#include <cassert> +#include <cstddef> +#include <cstdint> #include <limits> #include <random> +#include <vector> #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -67,7 +71,11 @@ std::vector<size_t> GetSizes(HashtablezSampler* s) { HashtablezInfo* Register(HashtablezSampler* s, size_t size) { const int64_t test_stride = 123; const size_t test_element_size = 17; - auto* info = s->Register(test_stride, test_element_size); + const size_t test_key_size = 3; + const size_t test_value_size = 5; + auto* info = + s->Register(test_stride, test_element_size, /*key_size=*/test_key_size, + /*value_size=*/test_value_size, /*soo_capacity=*/0); assert(info != nullptr); info->size.store(size); return info; @@ -77,9 +85,15 @@ TEST(HashtablezInfoTest, PrepareForSampling) { absl::Time test_start = absl::Now(); const int64_t test_stride = 123; const size_t test_element_size = 17; + const size_t test_key_size = 15; + const size_t test_value_size = 13; + HashtablezInfo info; absl::MutexLock l(&info.init_mu); - info.PrepareForSampling(test_stride, test_element_size); + info.PrepareForSampling(test_stride, test_element_size, + /*key_size=*/test_key_size, + /*value_size=*/test_value_size, + /*soo_capacity_value=*/1); EXPECT_EQ(info.capacity.load(), 0); EXPECT_EQ(info.size.load(), 0); @@ -94,6 +108,9 @@ TEST(HashtablezInfoTest, PrepareForSampling) { EXPECT_GE(info.create_time, test_start); EXPECT_EQ(info.weight, test_stride); EXPECT_EQ(info.inline_element_size, test_element_size); + EXPECT_EQ(info.key_size, test_key_size); + EXPECT_EQ(info.value_size, test_value_size); + EXPECT_EQ(info.soo_capacity, 1); info.capacity.store(1, std::memory_order_relaxed); info.size.store(1, std::memory_order_relaxed); @@ -106,7 +123,10 @@ TEST(HashtablezInfoTest, PrepareForSampling) { info.max_reserve.store(1, std::memory_order_relaxed); info.create_time = test_start - absl::Hours(20); - info.PrepareForSampling(test_stride * 2, test_element_size); + info.PrepareForSampling(test_stride * 2, test_element_size, + /*key_size=*/test_key_size, + /*value_size=*/test_value_size, + /*soo_capacity_value=*/0); EXPECT_EQ(info.capacity.load(), 0); EXPECT_EQ(info.size.load(), 0); EXPECT_EQ(info.num_erases.load(), 0); @@ -119,7 +139,10 @@ TEST(HashtablezInfoTest, PrepareForSampling) { EXPECT_EQ(info.max_reserve.load(), 0); EXPECT_EQ(info.weight, 2 * test_stride); EXPECT_EQ(info.inline_element_size, test_element_size); + EXPECT_EQ(info.key_size, test_key_size); + EXPECT_EQ(info.value_size, test_value_size); EXPECT_GE(info.create_time, test_start); + EXPECT_EQ(info.soo_capacity, 0); } TEST(HashtablezInfoTest, RecordStorageChanged) { @@ -127,7 +150,13 @@ TEST(HashtablezInfoTest, RecordStorageChanged) { absl::MutexLock l(&info.init_mu); const int64_t test_stride = 21; const size_t test_element_size = 19; - info.PrepareForSampling(test_stride, test_element_size); + const size_t test_key_size = 17; + const size_t test_value_size = 15; + + info.PrepareForSampling(test_stride, test_element_size, + /*key_size=*/test_key_size, + /*value_size=*/test_value_size, + /*soo_capacity_value=*/0); RecordStorageChangedSlow(&info, 17, 47); EXPECT_EQ(info.size.load(), 17); EXPECT_EQ(info.capacity.load(), 47); @@ -141,7 +170,13 @@ TEST(HashtablezInfoTest, RecordInsert) { absl::MutexLock l(&info.init_mu); const int64_t test_stride = 25; const size_t test_element_size = 23; - info.PrepareForSampling(test_stride, test_element_size); + const size_t test_key_size = 21; + const size_t test_value_size = 19; + + info.PrepareForSampling(test_stride, test_element_size, + /*key_size=*/test_key_size, + /*value_size=*/test_value_size, + /*soo_capacity_value=*/0); EXPECT_EQ(info.max_probe_length.load(), 0); RecordInsertSlow(&info, 0x0000FF00, 6 * kProbeLength); EXPECT_EQ(info.max_probe_length.load(), 6); @@ -163,9 +198,15 @@ TEST(HashtablezInfoTest, RecordInsert) { TEST(HashtablezInfoTest, RecordErase) { const int64_t test_stride = 31; const size_t test_element_size = 29; + const size_t test_key_size = 27; + const size_t test_value_size = 25; + HashtablezInfo info; absl::MutexLock l(&info.init_mu); - info.PrepareForSampling(test_stride, test_element_size); + info.PrepareForSampling(test_stride, test_element_size, + /*key_size=*/test_key_size, + /*value_size=*/test_value_size, + /*soo_capacity_value=*/1); EXPECT_EQ(info.num_erases.load(), 0); EXPECT_EQ(info.size.load(), 0); RecordInsertSlow(&info, 0x0000FF00, 6 * kProbeLength); @@ -174,14 +215,23 @@ TEST(HashtablezInfoTest, RecordErase) { EXPECT_EQ(info.size.load(), 0); EXPECT_EQ(info.num_erases.load(), 1); EXPECT_EQ(info.inline_element_size, test_element_size); + EXPECT_EQ(info.key_size, test_key_size); + EXPECT_EQ(info.value_size, test_value_size); + EXPECT_EQ(info.soo_capacity, 1); } TEST(HashtablezInfoTest, RecordRehash) { const int64_t test_stride = 33; const size_t test_element_size = 31; + const size_t test_key_size = 29; + const size_t test_value_size = 27; HashtablezInfo info; absl::MutexLock l(&info.init_mu); - info.PrepareForSampling(test_stride, test_element_size); + info.PrepareForSampling(test_stride, test_element_size, + /*key_size=*/test_key_size, + /*value_size=*/test_value_size, + + /*soo_capacity_value=*/0); RecordInsertSlow(&info, 0x1, 0); RecordInsertSlow(&info, 0x2, kProbeLength); RecordInsertSlow(&info, 0x4, kProbeLength); @@ -201,6 +251,9 @@ TEST(HashtablezInfoTest, RecordRehash) { EXPECT_EQ(info.num_erases.load(), 0); EXPECT_EQ(info.num_rehashes.load(), 1); EXPECT_EQ(info.inline_element_size, test_element_size); + EXPECT_EQ(info.key_size, test_key_size); + EXPECT_EQ(info.value_size, test_value_size); + EXPECT_EQ(info.soo_capacity, 0); } TEST(HashtablezInfoTest, RecordReservation) { @@ -208,7 +261,14 @@ TEST(HashtablezInfoTest, RecordReservation) { absl::MutexLock l(&info.init_mu); const int64_t test_stride = 35; const size_t test_element_size = 33; - info.PrepareForSampling(test_stride, test_element_size); + const size_t test_key_size = 31; + const size_t test_value_size = 29; + + info.PrepareForSampling(test_stride, test_element_size, + /*key_size=*/test_key_size, + /*value_size=*/test_value_size, + + /*soo_capacity_value=*/0); RecordReservationSlow(&info, 3); EXPECT_EQ(info.max_reserve.load(), 3); @@ -224,12 +284,19 @@ TEST(HashtablezInfoTest, RecordReservation) { #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) TEST(HashtablezSamplerTest, SmallSampleParameter) { const size_t test_element_size = 31; + const size_t test_key_size = 33; + const size_t test_value_size = 35; + SetHashtablezEnabled(true); SetHashtablezSampleParameter(100); for (int i = 0; i < 1000; ++i) { SamplingState next_sample = {0, 0}; - HashtablezInfo* sample = SampleSlow(next_sample, test_element_size); + HashtablezInfo* sample = + SampleSlow(next_sample, test_element_size, + /*key_size=*/test_key_size, /*value_size=*/test_value_size, + + /*soo_capacity=*/0); EXPECT_GT(next_sample.next_sample, 0); EXPECT_EQ(next_sample.next_sample, next_sample.sample_stride); EXPECT_NE(sample, nullptr); @@ -239,12 +306,17 @@ TEST(HashtablezSamplerTest, SmallSampleParameter) { TEST(HashtablezSamplerTest, LargeSampleParameter) { const size_t test_element_size = 31; + const size_t test_key_size = 33; + const size_t test_value_size = 35; SetHashtablezEnabled(true); SetHashtablezSampleParameter(std::numeric_limits<int32_t>::max()); for (int i = 0; i < 1000; ++i) { SamplingState next_sample = {0, 0}; - HashtablezInfo* sample = SampleSlow(next_sample, test_element_size); + HashtablezInfo* sample = + SampleSlow(next_sample, test_element_size, + /*key_size=*/test_key_size, /*value_size=*/test_value_size, + /*soo_capacity=*/0); EXPECT_GT(next_sample.next_sample, 0); EXPECT_EQ(next_sample.next_sample, next_sample.sample_stride); EXPECT_NE(sample, nullptr); @@ -254,13 +326,20 @@ TEST(HashtablezSamplerTest, LargeSampleParameter) { TEST(HashtablezSamplerTest, Sample) { const size_t test_element_size = 31; + const size_t test_key_size = 33; + const size_t test_value_size = 35; SetHashtablezEnabled(true); SetHashtablezSampleParameter(100); int64_t num_sampled = 0; int64_t total = 0; double sample_rate = 0.0; for (int i = 0; i < 1000000; ++i) { - HashtablezInfoHandle h = Sample(test_element_size); + HashtablezInfoHandle h = + Sample(test_element_size, + /*key_size=*/test_key_size, /*value_size=*/test_value_size, + + /*soo_capacity=*/0); + ++total; if (h.IsSampled()) { ++num_sampled; @@ -275,7 +354,12 @@ TEST(HashtablezSamplerTest, Handle) { auto& sampler = GlobalHashtablezSampler(); const int64_t test_stride = 41; const size_t test_element_size = 39; - HashtablezInfoHandle h(sampler.Register(test_stride, test_element_size)); + const size_t test_key_size = 37; + const size_t test_value_size = 35; + HashtablezInfoHandle h(sampler.Register(test_stride, test_element_size, + /*key_size=*/test_key_size, + /*value_size=*/test_value_size, + /*soo_capacity=*/0)); auto* info = HashtablezInfoHandlePeer::GetInfo(&h); info->hashes_bitwise_and.store(0x12345678, std::memory_order_relaxed); @@ -351,18 +435,28 @@ TEST(HashtablezSamplerTest, MultiThreaded) { for (int i = 0; i < 10; ++i) { const int64_t sampling_stride = 11 + i % 3; const size_t elt_size = 10 + i % 2; - pool.Schedule([&sampler, &stop, sampling_stride, elt_size]() { + const size_t key_size = 12 + i % 4; + const size_t value_size = 13 + i % 5; + pool.Schedule([&sampler, &stop, sampling_stride, elt_size, key_size, + value_size]() { std::random_device rd; std::mt19937 gen(rd()); std::vector<HashtablezInfo*> infoz; while (!stop.HasBeenNotified()) { if (infoz.empty()) { - infoz.push_back(sampler.Register(sampling_stride, elt_size)); + infoz.push_back(sampler.Register(sampling_stride, elt_size, + /*key_size=*/key_size, + /*value_size=*/value_size, + /*soo_capacity=*/0)); } switch (std::uniform_int_distribution<>(0, 2)(gen)) { case 0: { - infoz.push_back(sampler.Register(sampling_stride, elt_size)); + infoz.push_back(sampler.Register(sampling_stride, elt_size, + /*key_size=*/key_size, + /*value_size=*/value_size, + + /*soo_capacity=*/0)); break; } case 1: { diff --git a/absl/container/internal/inlined_vector.h b/absl/container/internal/inlined_vector.h index 0eb9c34d..2f24e461 100644 --- a/absl/container/internal/inlined_vector.h +++ b/absl/container/internal/inlined_vector.h @@ -27,6 +27,7 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" +#include "absl/base/internal/identity.h" #include "absl/base/macros.h" #include "absl/container/internal/compressed_tuple.h" #include "absl/memory/memory.h" @@ -82,16 +83,6 @@ using IsMoveAssignOk = std::is_move_assignable<ValueType<A>>; template <typename A> using IsSwapOk = absl::type_traits_internal::IsSwappable<ValueType<A>>; -template <typename T> -struct TypeIdentity { - using type = T; -}; - -// Used for function arguments in template functions to prevent ADL by forcing -// callers to explicitly specify the template parameter. -template <typename T> -using NoTypeDeduction = typename TypeIdentity<T>::type; - template <typename A, bool IsTriviallyDestructible = absl::is_trivially_destructible<ValueType<A>>::value> struct DestroyAdapter; @@ -139,7 +130,7 @@ struct MallocAdapter { }; template <typename A, typename ValueAdapter> -void ConstructElements(NoTypeDeduction<A>& allocator, +void ConstructElements(absl::internal::type_identity_t<A>& allocator, Pointer<A> construct_first, ValueAdapter& values, SizeType<A> construct_size) { for (SizeType<A> i = 0; i < construct_size; ++i) { @@ -322,14 +313,13 @@ class Storage { // The policy to be used specifically when swapping inlined elements. using SwapInlinedElementsPolicy = absl::conditional_t< - // Fast path: if the value type can be trivially move constructed/assigned - // and destroyed, and we know the allocator doesn't do anything fancy, - // then it's safe for us to simply swap the bytes in the inline storage. - // It's as if we had move-constructed a temporary vector, move-assigned - // one to the other, then move-assigned the first from the temporary. - absl::conjunction<absl::is_trivially_move_constructible<ValueType<A>>, - absl::is_trivially_move_assignable<ValueType<A>>, - absl::is_trivially_destructible<ValueType<A>>, + // Fast path: if the value type can be trivially relocated, and we + // know the allocator doesn't do anything fancy, then it's safe for us + // to simply swap the bytes in the inline storage. It's as if we had + // relocated the first vector's elements into temporary storage, + // relocated the second's elements into the (now-empty) first's, + // and then relocated from temporary storage into the second. + absl::conjunction<absl::is_trivially_relocatable<ValueType<A>>, std::is_same<A, std::allocator<ValueType<A>>>>::value, MemcpyPolicy, absl::conditional_t<IsSwapOk<A>::value, ElementwiseSwapPolicy, @@ -624,8 +614,8 @@ void Storage<T, N, A>::InitFrom(const Storage& other) { template <typename T, size_t N, typename A> template <typename ValueAdapter> -auto Storage<T, N, A>::Initialize(ValueAdapter values, SizeType<A> new_size) - -> void { +auto Storage<T, N, A>::Initialize(ValueAdapter values, + SizeType<A> new_size) -> void { // Only callable from constructors! ABSL_HARDENING_ASSERT(!GetIsAllocated()); ABSL_HARDENING_ASSERT(GetSize() == 0); @@ -656,8 +646,8 @@ auto Storage<T, N, A>::Initialize(ValueAdapter values, SizeType<A> new_size) template <typename T, size_t N, typename A> template <typename ValueAdapter> -auto Storage<T, N, A>::Assign(ValueAdapter values, SizeType<A> new_size) - -> void { +auto Storage<T, N, A>::Assign(ValueAdapter values, + SizeType<A> new_size) -> void { StorageView<A> storage_view = MakeStorageView(); AllocationTransaction<A> allocation_tx(GetAllocator()); @@ -699,8 +689,8 @@ auto Storage<T, N, A>::Assign(ValueAdapter values, SizeType<A> new_size) template <typename T, size_t N, typename A> template <typename ValueAdapter> -auto Storage<T, N, A>::Resize(ValueAdapter values, SizeType<A> new_size) - -> void { +auto Storage<T, N, A>::Resize(ValueAdapter values, + SizeType<A> new_size) -> void { StorageView<A> storage_view = MakeStorageView(); Pointer<A> const base = storage_view.data; const SizeType<A> size = storage_view.size; @@ -885,8 +875,8 @@ auto Storage<T, N, A>::EmplaceBackSlow(Args&&... args) -> Reference<A> { } template <typename T, size_t N, typename A> -auto Storage<T, N, A>::Erase(ConstIterator<A> from, ConstIterator<A> to) - -> Iterator<A> { +auto Storage<T, N, A>::Erase(ConstIterator<A> from, + ConstIterator<A> to) -> Iterator<A> { StorageView<A> storage_view = MakeStorageView(); auto erase_size = static_cast<SizeType<A>>(std::distance(from, to)); @@ -894,16 +884,30 @@ auto Storage<T, N, A>::Erase(ConstIterator<A> from, ConstIterator<A> to) std::distance(ConstIterator<A>(storage_view.data), from)); SizeType<A> erase_end_index = erase_index + erase_size; - IteratorValueAdapter<A, MoveIterator<A>> move_values( - MoveIterator<A>(storage_view.data + erase_end_index)); - - AssignElements<A>(storage_view.data + erase_index, move_values, - storage_view.size - erase_end_index); + // Fast path: if the value type is trivially relocatable and we know + // the allocator doesn't do anything fancy, then we know it is legal for us to + // simply destroy the elements in the "erasure window" (which cannot throw) + // and then memcpy downward to close the window. + if (absl::is_trivially_relocatable<ValueType<A>>::value && + std::is_nothrow_destructible<ValueType<A>>::value && + std::is_same<A, std::allocator<ValueType<A>>>::value) { + DestroyAdapter<A>::DestroyElements( + GetAllocator(), storage_view.data + erase_index, erase_size); + std::memmove( + reinterpret_cast<char*>(storage_view.data + erase_index), + reinterpret_cast<const char*>(storage_view.data + erase_end_index), + (storage_view.size - erase_end_index) * sizeof(ValueType<A>)); + } else { + IteratorValueAdapter<A, MoveIterator<A>> move_values( + MoveIterator<A>(storage_view.data + erase_end_index)); - DestroyAdapter<A>::DestroyElements( - GetAllocator(), storage_view.data + (storage_view.size - erase_size), - erase_size); + AssignElements<A>(storage_view.data + erase_index, move_values, + storage_view.size - erase_end_index); + DestroyAdapter<A>::DestroyElements( + GetAllocator(), storage_view.data + (storage_view.size - erase_size), + erase_size); + } SubtractSize(erase_size); return Iterator<A>(storage_view.data + erase_index); } diff --git a/absl/container/internal/layout.h b/absl/container/internal/layout.h index a4ba6101..384929af 100644 --- a/absl/container/internal/layout.h +++ b/absl/container/internal/layout.h @@ -81,9 +81,30 @@ // } // // The layout we used above combines fixed-size with dynamically-sized fields. -// This is quite common. Layout is optimized for this use case and generates -// optimal code. All computations that can be performed at compile time are -// indeed performed at compile time. +// This is quite common. Layout is optimized for this use case and attempts to +// generate optimal code. To help the compiler do that in more cases, you can +// specify the fixed sizes using `WithStaticSizes`. This ensures that all +// computations that can be performed at compile time are indeed performed at +// compile time. Note that sometimes the `template` keyword is needed. E.g.: +// +// using SL = L::template WithStaticSizes<1, 1>; +// +// void Use(unsigned char* p) { +// // First, extract N and M. +// // Using `prefix` we can access the first three arrays but not more. +// // +// // More details: The first element always has offset 0. `SL` +// // has offsets for the second and third array based on sizes of +// // the first and second array, specified via `WithStaticSizes`. +// constexpr auto prefix = SL::Partial(); +// size_t n = *prefix.Pointer<0>(p); +// size_t m = *prefix.Pointer<1>(p); +// +// // Now we can get a pointer to the final payload. +// const SL layout(n, m); +// double* a = layout.Pointer<double>(p); +// int* b = layout.Pointer<int>(p); +// } // // Efficiency tip: The order of fields matters. In `Layout<T1, ..., TN>` try to // ensure that `alignof(T1) >= ... >= alignof(TN)`. This way you'll have no @@ -107,7 +128,7 @@ // CompactString(const char* s = "") { // const size_t size = strlen(s); // // size_t[1] followed by char[size + 1]. -// const L layout(1, size + 1); +// const L layout(size + 1); // p_.reset(new unsigned char[layout.AllocSize()]); // // If running under ASAN, mark the padding bytes, if any, to catch // // memory errors. @@ -125,14 +146,13 @@ // // const char* c_str() const { // // Equivalent to reinterpret_cast<char*>(p.get() + sizeof(size_t)). -// // The argument in Partial(1) specifies that we have size_t[1] in front -// // of the characters. -// return L::Partial(1).Pointer<char>(p_.get()); +// return L::Partial().Pointer<char>(p_.get()); // } // // private: -// // Our heap allocation contains a size_t followed by an array of chars. -// using L = Layout<size_t, char>; +// // Our heap allocation contains a single size_t followed by an array of +// // chars. +// using L = Layout<size_t, char>::WithStaticSizes<1>; // std::unique_ptr<unsigned char[]> p_; // }; // @@ -146,11 +166,12 @@ // // The interface exported by this file consists of: // - class `Layout<>` and its public members. -// - The public members of class `internal_layout::LayoutImpl<>`. That class -// isn't intended to be used directly, and its name and template parameter -// list are internal implementation details, but the class itself provides -// most of the functionality in this file. See comments on its members for -// detailed documentation. +// - The public members of classes `internal_layout::LayoutWithStaticSizes<>` +// and `internal_layout::LayoutImpl<>`. Those classes aren't intended to be +// used directly, and their name and template parameter list are internal +// implementation details, but the classes themselves provide most of the +// functionality in this file. See comments on their members for detailed +// documentation. // // `Layout<T1,... Tn>::Partial(count1,..., countm)` (where `m` <= `n`) returns a // `LayoutImpl<>` object. `Layout<T1,..., Tn> layout(count1,..., countn)` @@ -164,13 +185,14 @@ #include <stddef.h> #include <stdint.h> -#include <ostream> +#include <array> #include <string> #include <tuple> #include <type_traits> #include <typeinfo> #include <utility> +#include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/debugging/internal/demangle.h" #include "absl/meta/type_traits.h" @@ -209,9 +231,6 @@ struct NotAligned<const Aligned<T, N>> { template <size_t> using IntToSize = size_t; -template <class> -using TypeToSize = size_t; - template <class T> struct Type : NotAligned<T> { using type = T; @@ -308,7 +327,8 @@ using IsLegalElementType = std::integral_constant< !std::is_volatile<typename Type<T>::type>::value && adl_barrier::IsPow2(AlignOf<T>::value)>; -template <class Elements, class SizeSeq, class OffsetSeq> +template <class Elements, class StaticSizeSeq, class RuntimeSizeSeq, + class SizeSeq, class OffsetSeq> class LayoutImpl; // Public base class of `Layout` and the result type of `Layout::Partial()`. @@ -316,31 +336,49 @@ class LayoutImpl; // `Elements...` contains all template arguments of `Layout` that created this // instance. // -// `SizeSeq...` is `[0, NumSizes)` where `NumSizes` is the number of arguments -// passed to `Layout::Partial()` or `Layout::Layout()`. +// `StaticSizeSeq...` is an index_sequence containing the sizes specified at +// compile-time. +// +// `RuntimeSizeSeq...` is `[0, NumRuntimeSizes)`, where `NumRuntimeSizes` is the +// number of arguments passed to `Layout::Partial()` or `Layout::Layout()`. +// +// `SizeSeq...` is `[0, NumSizes)` where `NumSizes` is `NumRuntimeSizes` plus +// the number of sizes in `StaticSizeSeq`. // // `OffsetSeq...` is `[0, NumOffsets)` where `NumOffsets` is // `Min(sizeof...(Elements), NumSizes + 1)` (the number of arrays for which we // can compute offsets). -template <class... Elements, size_t... SizeSeq, size_t... OffsetSeq> -class LayoutImpl<std::tuple<Elements...>, absl::index_sequence<SizeSeq...>, - absl::index_sequence<OffsetSeq...>> { +template <class... Elements, size_t... StaticSizeSeq, size_t... RuntimeSizeSeq, + size_t... SizeSeq, size_t... OffsetSeq> +class LayoutImpl< + std::tuple<Elements...>, absl::index_sequence<StaticSizeSeq...>, + absl::index_sequence<RuntimeSizeSeq...>, absl::index_sequence<SizeSeq...>, + absl::index_sequence<OffsetSeq...>> { private: static_assert(sizeof...(Elements) > 0, "At least one field is required"); static_assert(absl::conjunction<IsLegalElementType<Elements>...>::value, "Invalid element type (see IsLegalElementType)"); + static_assert(sizeof...(StaticSizeSeq) <= sizeof...(Elements), + "Too many static sizes specified"); enum { NumTypes = sizeof...(Elements), + NumStaticSizes = sizeof...(StaticSizeSeq), + NumRuntimeSizes = sizeof...(RuntimeSizeSeq), NumSizes = sizeof...(SizeSeq), NumOffsets = sizeof...(OffsetSeq), }; // These are guaranteed by `Layout`. + static_assert(NumStaticSizes + NumRuntimeSizes == NumSizes, "Internal error"); + static_assert(NumSizes <= NumTypes, "Internal error"); static_assert(NumOffsets == adl_barrier::Min(NumTypes, NumSizes + 1), "Internal error"); static_assert(NumTypes > 0, "Internal error"); + static constexpr std::array<size_t, sizeof...(StaticSizeSeq)> kStaticSizes = { + StaticSizeSeq...}; + // Returns the index of `T` in `Elements...`. Results in a compilation error // if `Elements...` doesn't contain exactly one instance of `T`. template <class T> @@ -363,7 +401,7 @@ class LayoutImpl<std::tuple<Elements...>, absl::index_sequence<SizeSeq...>, template <size_t N> using ElementType = typename std::tuple_element<N, ElementTypes>::type; - constexpr explicit LayoutImpl(IntToSize<SizeSeq>... sizes) + constexpr explicit LayoutImpl(IntToSize<RuntimeSizeSeq>... sizes) : size_{sizes...} {} // Alignment of the layout, equal to the strictest alignment of all elements. @@ -389,7 +427,7 @@ class LayoutImpl<std::tuple<Elements...>, absl::index_sequence<SizeSeq...>, constexpr size_t Offset() const { static_assert(N < NumOffsets, "Index out of bounds"); return adl_barrier::Align( - Offset<N - 1>() + SizeOf<ElementType<N - 1>>::value * size_[N - 1], + Offset<N - 1>() + SizeOf<ElementType<N - 1>>::value * Size<N - 1>(), ElementAlignment<N>::value); } @@ -411,8 +449,7 @@ class LayoutImpl<std::tuple<Elements...>, absl::index_sequence<SizeSeq...>, return {{Offset<OffsetSeq>()...}}; } - // The number of elements in the Nth array. This is the Nth argument of - // `Layout::Partial()` or `Layout::Layout()` (zero-based). + // The number of elements in the Nth array (zero-based). // // // int[3], 4 bytes of padding, double[4]. // Layout<int, double> x(3, 4); @@ -420,10 +457,15 @@ class LayoutImpl<std::tuple<Elements...>, absl::index_sequence<SizeSeq...>, // assert(x.Size<1>() == 4); // // Requires: `N < NumSizes`. - template <size_t N> + template <size_t N, EnableIf<(N < NumStaticSizes)> = 0> + constexpr size_t Size() const { + return kStaticSizes[N]; + } + + template <size_t N, EnableIf<(N >= NumStaticSizes)> = 0> constexpr size_t Size() const { static_assert(N < NumSizes, "Index out of bounds"); - return size_[N]; + return size_[N - NumStaticSizes]; } // The number of elements in the array with the specified element type. @@ -500,13 +542,8 @@ class LayoutImpl<std::tuple<Elements...>, absl::index_sequence<SizeSeq...>, // std::tie(ints, doubles) = x.Pointers(p); // // Requires: `p` is aligned to `Alignment()`. - // - // Note: We're not using ElementType alias here because it does not compile - // under MSVC. template <class Char> - std::tuple<CopyConst< - Char, typename std::tuple_element<OffsetSeq, ElementTypes>::type>*...> - Pointers(Char* p) const { + auto Pointers(Char* p) const { return std::tuple<CopyConst<Char, ElementType<OffsetSeq>>*...>( Pointer<OffsetSeq>(p)...); } @@ -559,15 +596,10 @@ class LayoutImpl<std::tuple<Elements...>, absl::index_sequence<SizeSeq...>, // // Requires: `p` is aligned to `Alignment()`. // - // Note: We're not using ElementType alias here because it does not compile - // under MSVC. + // Note: We mark the parameter as unused because GCC detects it is not used + // when `SizeSeq` is empty [-Werror=unused-but-set-parameter]. template <class Char> - std::tuple<SliceType<CopyConst< - Char, typename std::tuple_element<SizeSeq, ElementTypes>::type>>...> - Slices(Char* p) const { - // Workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63875 (fixed - // in 6.1). - (void)p; + auto Slices(ABSL_ATTRIBUTE_UNUSED Char* p) const { return std::tuple<SliceType<CopyConst<Char, ElementType<SizeSeq>>>...>( Slice<SizeSeq>(p)...); } @@ -582,7 +614,7 @@ class LayoutImpl<std::tuple<Elements...>, absl::index_sequence<SizeSeq...>, constexpr size_t AllocSize() const { static_assert(NumTypes == NumSizes, "You must specify sizes of all fields"); return Offset<NumTypes - 1>() + - SizeOf<ElementType<NumTypes - 1>>::value * size_[NumTypes - 1]; + SizeOf<ElementType<NumTypes - 1>>::value * Size<NumTypes - 1>(); } // If built with --config=asan, poisons padding bytes (if any) in the @@ -606,7 +638,7 @@ class LayoutImpl<std::tuple<Elements...>, absl::index_sequence<SizeSeq...>, // The `if` is an optimization. It doesn't affect the observable behaviour. if (ElementAlignment<N - 1>::value % ElementAlignment<N>::value) { size_t start = - Offset<N - 1>() + SizeOf<ElementType<N - 1>>::value * size_[N - 1]; + Offset<N - 1>() + SizeOf<ElementType<N - 1>>::value * Size<N - 1>(); ASAN_POISON_MEMORY_REGION(p + start, Offset<N>() - start); } #endif @@ -635,47 +667,66 @@ class LayoutImpl<std::tuple<Elements...>, absl::index_sequence<SizeSeq...>, adl_barrier::TypeName<ElementType<OffsetSeq>>()...}; std::string res = absl::StrCat("@0", types[0], "(", sizes[0], ")"); for (size_t i = 0; i != NumOffsets - 1; ++i) { - absl::StrAppend(&res, "[", size_[i], "]; @", offsets[i + 1], types[i + 1], - "(", sizes[i + 1], ")"); + absl::StrAppend(&res, "[", DebugSize(i), "]; @", offsets[i + 1], + types[i + 1], "(", sizes[i + 1], ")"); } // NumSizes is a constant that may be zero. Some compilers cannot see that // inside the if statement "size_[NumSizes - 1]" must be valid. int last = static_cast<int>(NumSizes) - 1; if (NumTypes == NumSizes && last >= 0) { - absl::StrAppend(&res, "[", size_[last], "]"); + absl::StrAppend(&res, "[", DebugSize(static_cast<size_t>(last)), "]"); } return res; } private: + size_t DebugSize(size_t n) const { + if (n < NumStaticSizes) { + return kStaticSizes[n]; + } else { + return size_[n - NumStaticSizes]; + } + } + // Arguments of `Layout::Partial()` or `Layout::Layout()`. - size_t size_[NumSizes > 0 ? NumSizes : 1]; + size_t size_[NumRuntimeSizes > 0 ? NumRuntimeSizes : 1]; }; -template <size_t NumSizes, class... Ts> -using LayoutType = LayoutImpl< - std::tuple<Ts...>, absl::make_index_sequence<NumSizes>, - absl::make_index_sequence<adl_barrier::Min(sizeof...(Ts), NumSizes + 1)>>; +// Defining a constexpr static class member variable is redundant and deprecated +// in C++17, but required in C++14. +template <class... Elements, size_t... StaticSizeSeq, size_t... RuntimeSizeSeq, + size_t... SizeSeq, size_t... OffsetSeq> +constexpr std::array<size_t, sizeof...(StaticSizeSeq)> LayoutImpl< + std::tuple<Elements...>, absl::index_sequence<StaticSizeSeq...>, + absl::index_sequence<RuntimeSizeSeq...>, absl::index_sequence<SizeSeq...>, + absl::index_sequence<OffsetSeq...>>::kStaticSizes; -} // namespace internal_layout +template <class StaticSizeSeq, size_t NumRuntimeSizes, class... Ts> +using LayoutType = LayoutImpl< + std::tuple<Ts...>, StaticSizeSeq, + absl::make_index_sequence<NumRuntimeSizes>, + absl::make_index_sequence<NumRuntimeSizes + StaticSizeSeq::size()>, + absl::make_index_sequence<adl_barrier::Min( + sizeof...(Ts), NumRuntimeSizes + StaticSizeSeq::size() + 1)>>; + +template <class StaticSizeSeq, class... Ts> +class LayoutWithStaticSizes + : public LayoutType<StaticSizeSeq, + sizeof...(Ts) - adl_barrier::Min(sizeof...(Ts), + StaticSizeSeq::size()), + Ts...> { + private: + using Super = + LayoutType<StaticSizeSeq, + sizeof...(Ts) - + adl_barrier::Min(sizeof...(Ts), StaticSizeSeq::size()), + Ts...>; -// Descriptor of arrays of various types and sizes laid out in memory one after -// another. See the top of the file for documentation. -// -// Check out the public API of internal_layout::LayoutImpl above. The type is -// internal to the library but its methods are public, and they are inherited -// by `Layout`. -template <class... Ts> -class Layout : public internal_layout::LayoutType<sizeof...(Ts), Ts...> { public: - static_assert(sizeof...(Ts) > 0, "At least one field is required"); - static_assert( - absl::conjunction<internal_layout::IsLegalElementType<Ts>...>::value, - "Invalid element type (see IsLegalElementType)"); - // The result type of `Partial()` with `NumSizes` arguments. template <size_t NumSizes> - using PartialType = internal_layout::LayoutType<NumSizes, Ts...>; + using PartialType = + internal_layout::LayoutType<StaticSizeSeq, NumSizes, Ts...>; // `Layout` knows the element types of the arrays we want to lay out in // memory but not the number of elements in each array. @@ -701,14 +752,18 @@ class Layout : public internal_layout::LayoutType<sizeof...(Ts), Ts...> { // Note: The sizes of the arrays must be specified in number of elements, // not in bytes. // - // Requires: `sizeof...(Sizes) <= sizeof...(Ts)`. + // Requires: `sizeof...(Sizes) + NumStaticSizes <= sizeof...(Ts)`. // Requires: all arguments are convertible to `size_t`. template <class... Sizes> static constexpr PartialType<sizeof...(Sizes)> Partial(Sizes&&... sizes) { - static_assert(sizeof...(Sizes) <= sizeof...(Ts), ""); - return PartialType<sizeof...(Sizes)>(absl::forward<Sizes>(sizes)...); + static_assert(sizeof...(Sizes) + StaticSizeSeq::size() <= sizeof...(Ts), + ""); + return PartialType<sizeof...(Sizes)>( + static_cast<size_t>(std::forward<Sizes>(sizes))...); } + // Inherit LayoutType's constructor. + // // Creates a layout with the sizes of all arrays specified. If you know // only the sizes of the first N arrays (where N can be zero), you can use // `Partial()` defined above. The constructor is essentially equivalent to @@ -717,8 +772,69 @@ class Layout : public internal_layout::LayoutType<sizeof...(Ts), Ts...> { // // Note: The sizes of the arrays must be specified in number of elements, // not in bytes. - constexpr explicit Layout(internal_layout::TypeToSize<Ts>... sizes) - : internal_layout::LayoutType<sizeof...(Ts), Ts...>(sizes...) {} + // + // Implementation note: we do this via a `using` declaration instead of + // defining our own explicit constructor because the signature of LayoutType's + // constructor depends on RuntimeSizeSeq, which we don't have access to here. + // If we defined our own constructor here, it would have to use a parameter + // pack and then cast the arguments to size_t when calling the superclass + // constructor, similar to what Partial() does. But that would suffer from the + // same problem that Partial() has, which is that the parameter types are + // inferred from the arguments, which may be signed types, which must then be + // cast to size_t. This can lead to negative values being silently (i.e. with + // no compiler warnings) cast to an unsigned type. Having a constructor with + // size_t parameters helps the compiler generate better warnings about + // potential bad casts, while avoiding false warnings when positive literal + // arguments are used. If an argument is a positive literal integer (e.g. + // `1`), the compiler will understand that it can be safely converted to + // size_t, and hence not generate a warning. But if a negative literal (e.g. + // `-1`) or a variable with signed type is used, then it can generate a + // warning about a potentially unsafe implicit cast. It would be great if we + // could do this for Partial() too, but unfortunately as of C++23 there seems + // to be no way to define a function with a variable number of parameters of a + // certain type, a.k.a. homogeneous function parameter packs. So we're forced + // to choose between explicitly casting the arguments to size_t, which + // suppresses all warnings, even potentially valid ones, or implicitly casting + // them to size_t, which generates bogus warnings whenever literal arguments + // are used, even if they're positive. + using Super::Super; +}; + +} // namespace internal_layout + +// Descriptor of arrays of various types and sizes laid out in memory one after +// another. See the top of the file for documentation. +// +// Check out the public API of internal_layout::LayoutWithStaticSizes and +// internal_layout::LayoutImpl above. Those types are internal to the library +// but their methods are public, and they are inherited by `Layout`. +template <class... Ts> +class Layout : public internal_layout::LayoutWithStaticSizes< + absl::make_index_sequence<0>, Ts...> { + private: + using Super = + internal_layout::LayoutWithStaticSizes<absl::make_index_sequence<0>, + Ts...>; + + public: + // If you know the sizes of some or all of the arrays at compile time, you can + // use `WithStaticSizes` or `WithStaticSizeSequence` to create a `Layout` type + // with those sizes baked in. This can help the compiler generate optimal code + // for calculating array offsets and AllocSize(). + // + // Like `Partial()`, the N sizes you specify are for the first N arrays, and + // they specify the number of elements in each array, not the number of bytes. + template <class StaticSizeSeq> + using WithStaticSizeSequence = + internal_layout::LayoutWithStaticSizes<StaticSizeSeq, Ts...>; + + template <size_t... StaticSizes> + using WithStaticSizes = + WithStaticSizeSequence<std::index_sequence<StaticSizes...>>; + + // Inherit LayoutWithStaticSizes's constructor, which requires you to specify + // all the array sizes. + using Super::Super; }; } // namespace container_internal diff --git a/absl/container/internal/layout_benchmark.cc b/absl/container/internal/layout_benchmark.cc index 3af35e33..d6f26697 100644 --- a/absl/container/internal/layout_benchmark.cc +++ b/absl/container/internal/layout_benchmark.cc @@ -15,6 +15,9 @@ // Every benchmark should have the same performance as the corresponding // headroom benchmark. +#include <cstddef> +#include <cstdint> + #include "absl/base/internal/raw_logging.h" #include "absl/container/internal/layout.h" #include "benchmark/benchmark.h" @@ -28,6 +31,8 @@ using ::benchmark::DoNotOptimize; using Int128 = int64_t[2]; +constexpr size_t MyAlign(size_t n, size_t m) { return (n + m - 1) & ~(m - 1); } + // This benchmark provides the upper bound on performance for BM_OffsetConstant. template <size_t Offset, class... Ts> void BM_OffsetConstantHeadroom(benchmark::State& state) { @@ -37,6 +42,15 @@ void BM_OffsetConstantHeadroom(benchmark::State& state) { } template <size_t Offset, class... Ts> +void BM_OffsetConstantStatic(benchmark::State& state) { + using L = typename Layout<Ts...>::template WithStaticSizes<3, 5, 7>; + ABSL_RAW_CHECK(L::Partial().template Offset<3>() == Offset, "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(L::Partial().template Offset<3>()); + } +} + +template <size_t Offset, class... Ts> void BM_OffsetConstant(benchmark::State& state) { using L = Layout<Ts...>; ABSL_RAW_CHECK(L::Partial(3, 5, 7).template Offset<3>() == Offset, @@ -46,14 +60,74 @@ void BM_OffsetConstant(benchmark::State& state) { } } +template <size_t Offset, class... Ts> +void BM_OffsetConstantIndirect(benchmark::State& state) { + using L = Layout<Ts...>; + auto p = L::Partial(3, 5, 7); + ABSL_RAW_CHECK(p.template Offset<3>() == Offset, "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(p); + DoNotOptimize(p.template Offset<3>()); + } +} + +template <class... Ts> +size_t PartialOffset(size_t k); + +template <> +size_t PartialOffset<int8_t, int16_t, int32_t, Int128>(size_t k) { + constexpr size_t o = MyAlign(MyAlign(3 * 1, 2) + 5 * 2, 4); + return MyAlign(o + k * 4, 8); +} + +template <> +size_t PartialOffset<Int128, int32_t, int16_t, int8_t>(size_t k) { + // No alignment is necessary. + return 3 * 16 + 5 * 4 + k * 2; +} + +// This benchmark provides the upper bound on performance for BM_OffsetVariable. +template <size_t Offset, class... Ts> +void BM_OffsetPartialHeadroom(benchmark::State& state) { + size_t k = 7; + ABSL_RAW_CHECK(PartialOffset<Ts...>(k) == Offset, "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(k); + DoNotOptimize(PartialOffset<Ts...>(k)); + } +} + +template <size_t Offset, class... Ts> +void BM_OffsetPartialStatic(benchmark::State& state) { + using L = typename Layout<Ts...>::template WithStaticSizes<3, 5>; + size_t k = 7; + ABSL_RAW_CHECK(L::Partial(k).template Offset<3>() == Offset, + "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(k); + DoNotOptimize(L::Partial(k).template Offset<3>()); + } +} + +template <size_t Offset, class... Ts> +void BM_OffsetPartial(benchmark::State& state) { + using L = Layout<Ts...>; + size_t k = 7; + ABSL_RAW_CHECK(L::Partial(3, 5, k).template Offset<3>() == Offset, + "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(k); + DoNotOptimize(L::Partial(3, 5, k).template Offset<3>()); + } +} + template <class... Ts> size_t VariableOffset(size_t n, size_t m, size_t k); template <> size_t VariableOffset<int8_t, int16_t, int32_t, Int128>(size_t n, size_t m, size_t k) { - auto Align = [](size_t n, size_t m) { return (n + m - 1) & ~(m - 1); }; - return Align(Align(Align(n * 1, 2) + m * 2, 4) + k * 4, 8); + return MyAlign(MyAlign(MyAlign(n * 1, 2) + m * 2, 4) + k * 4, 8); } template <> @@ -94,6 +168,75 @@ void BM_OffsetVariable(benchmark::State& state) { } } +template <class... Ts> +size_t AllocSize(size_t x); + +template <> +size_t AllocSize<int8_t, int16_t, int32_t, Int128>(size_t x) { + constexpr size_t o = + Layout<int8_t, int16_t, int32_t, Int128>::Partial(3, 5, 7) + .template Offset<Int128>(); + return o + sizeof(Int128) * x; +} + +template <> +size_t AllocSize<Int128, int32_t, int16_t, int8_t>(size_t x) { + constexpr size_t o = + Layout<Int128, int32_t, int16_t, int8_t>::Partial(3, 5, 7) + .template Offset<int8_t>(); + return o + sizeof(int8_t) * x; +} + +// This benchmark provides the upper bound on performance for BM_AllocSize +template <size_t Size, class... Ts> +void BM_AllocSizeHeadroom(benchmark::State& state) { + size_t x = 9; + ABSL_RAW_CHECK(AllocSize<Ts...>(x) == Size, "Invalid size"); + for (auto _ : state) { + DoNotOptimize(x); + DoNotOptimize(AllocSize<Ts...>(x)); + } +} + +template <size_t Size, class... Ts> +void BM_AllocSizeStatic(benchmark::State& state) { + using L = typename Layout<Ts...>::template WithStaticSizes<3, 5, 7>; + size_t x = 9; + ABSL_RAW_CHECK(L(x).AllocSize() == Size, "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(x); + DoNotOptimize(L(x).AllocSize()); + } +} + +template <size_t Size, class... Ts> +void BM_AllocSize(benchmark::State& state) { + using L = Layout<Ts...>; + size_t n = 3; + size_t m = 5; + size_t k = 7; + size_t x = 9; + ABSL_RAW_CHECK(L(n, m, k, x).AllocSize() == Size, "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(n); + DoNotOptimize(m); + DoNotOptimize(k); + DoNotOptimize(x); + DoNotOptimize(L(n, m, k, x).AllocSize()); + } +} + +template <size_t Size, class... Ts> +void BM_AllocSizeIndirect(benchmark::State& state) { + using L = Layout<Ts...>; + auto l = L(3, 5, 7, 9); + ABSL_RAW_CHECK(l.AllocSize() == Size, "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(l); + DoNotOptimize(l.AllocSize()); + } +} + // Run all benchmarks in two modes: // // Layout with padding: int8_t[3], int16_t[5], int32_t[7], Int128[?]. @@ -106,16 +249,46 @@ void BM_OffsetVariable(benchmark::State& state) { OFFSET_BENCHMARK(BM_OffsetConstantHeadroom, 48, int8_t, int16_t, int32_t, Int128); +OFFSET_BENCHMARK(BM_OffsetConstantStatic, 48, int8_t, int16_t, int32_t, Int128); OFFSET_BENCHMARK(BM_OffsetConstant, 48, int8_t, int16_t, int32_t, Int128); +OFFSET_BENCHMARK(BM_OffsetConstantIndirect, 48, int8_t, int16_t, int32_t, + Int128); + OFFSET_BENCHMARK(BM_OffsetConstantHeadroom, 82, Int128, int32_t, int16_t, int8_t); +OFFSET_BENCHMARK(BM_OffsetConstantStatic, 82, Int128, int32_t, int16_t, int8_t); OFFSET_BENCHMARK(BM_OffsetConstant, 82, Int128, int32_t, int16_t, int8_t); +OFFSET_BENCHMARK(BM_OffsetConstantIndirect, 82, Int128, int32_t, int16_t, + int8_t); + +OFFSET_BENCHMARK(BM_OffsetPartialHeadroom, 48, int8_t, int16_t, int32_t, + Int128); +OFFSET_BENCHMARK(BM_OffsetPartialStatic, 48, int8_t, int16_t, int32_t, Int128); +OFFSET_BENCHMARK(BM_OffsetPartial, 48, int8_t, int16_t, int32_t, Int128); + +OFFSET_BENCHMARK(BM_OffsetPartialHeadroom, 82, Int128, int32_t, int16_t, + int8_t); +OFFSET_BENCHMARK(BM_OffsetPartialStatic, 82, Int128, int32_t, int16_t, int8_t); +OFFSET_BENCHMARK(BM_OffsetPartial, 82, Int128, int32_t, int16_t, int8_t); + OFFSET_BENCHMARK(BM_OffsetVariableHeadroom, 48, int8_t, int16_t, int32_t, Int128); OFFSET_BENCHMARK(BM_OffsetVariable, 48, int8_t, int16_t, int32_t, Int128); + OFFSET_BENCHMARK(BM_OffsetVariableHeadroom, 82, Int128, int32_t, int16_t, int8_t); OFFSET_BENCHMARK(BM_OffsetVariable, 82, Int128, int32_t, int16_t, int8_t); + +OFFSET_BENCHMARK(BM_AllocSizeHeadroom, 192, int8_t, int16_t, int32_t, Int128); +OFFSET_BENCHMARK(BM_AllocSizeStatic, 192, int8_t, int16_t, int32_t, Int128); +OFFSET_BENCHMARK(BM_AllocSize, 192, int8_t, int16_t, int32_t, Int128); +OFFSET_BENCHMARK(BM_AllocSizeIndirect, 192, int8_t, int16_t, int32_t, Int128); + +OFFSET_BENCHMARK(BM_AllocSizeHeadroom, 91, Int128, int32_t, int16_t, int8_t); +OFFSET_BENCHMARK(BM_AllocSizeStatic, 91, Int128, int32_t, int16_t, int8_t); +OFFSET_BENCHMARK(BM_AllocSize, 91, Int128, int32_t, int16_t, int8_t); +OFFSET_BENCHMARK(BM_AllocSizeIndirect, 91, Int128, int32_t, int16_t, int8_t); + } // namespace } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/internal/layout_test.cc b/absl/container/internal/layout_test.cc index ae55cf7e..47fc9f33 100644 --- a/absl/container/internal/layout_test.cc +++ b/absl/container/internal/layout_test.cc @@ -68,9 +68,7 @@ struct alignas(8) Int128 { // int64_t is *not* 8-byte aligned on all platforms! struct alignas(8) Int64 { int64_t a; - friend bool operator==(Int64 lhs, Int64 rhs) { - return lhs.a == rhs.a; - } + friend bool operator==(Int64 lhs, Int64 rhs) { return lhs.a == rhs.a; } }; // Properties of types that this test relies on. @@ -271,6 +269,35 @@ TEST(Layout, Offsets) { } } +TEST(Layout, StaticOffsets) { + using L = Layout<int8_t, int32_t, Int128>; + { + using SL = L::WithStaticSizes<>; + EXPECT_THAT(SL::Partial().Offsets(), ElementsAre(0)); + EXPECT_THAT(SL::Partial(5).Offsets(), ElementsAre(0, 8)); + EXPECT_THAT(SL::Partial(5, 3, 1).Offsets(), ElementsAre(0, 8, 24)); + EXPECT_THAT(SL(5, 3, 1).Offsets(), ElementsAre(0, 8, 24)); + } + { + using SL = L::WithStaticSizes<5>; + EXPECT_THAT(SL::Partial().Offsets(), ElementsAre(0, 8)); + EXPECT_THAT(SL::Partial(3).Offsets(), ElementsAre(0, 8, 24)); + EXPECT_THAT(SL::Partial(3, 1).Offsets(), ElementsAre(0, 8, 24)); + EXPECT_THAT(SL(3, 1).Offsets(), ElementsAre(0, 8, 24)); + } + { + using SL = L::WithStaticSizes<5, 3>; + EXPECT_THAT(SL::Partial().Offsets(), ElementsAre(0, 8, 24)); + EXPECT_THAT(SL::Partial(1).Offsets(), ElementsAre(0, 8, 24)); + EXPECT_THAT(SL(1).Offsets(), ElementsAre(0, 8, 24)); + } + { + using SL = L::WithStaticSizes<5, 3, 1>; + EXPECT_THAT(SL::Partial().Offsets(), ElementsAre(0, 8, 24)); + EXPECT_THAT(SL().Offsets(), ElementsAre(0, 8, 24)); + } +} + TEST(Layout, AllocSize) { { using L = Layout<int32_t>; @@ -295,6 +322,30 @@ TEST(Layout, AllocSize) { } } +TEST(Layout, StaticAllocSize) { + using L = Layout<int8_t, int32_t, Int128>; + { + using SL = L::WithStaticSizes<>; + EXPECT_EQ(136, SL::Partial(3, 5, 7).AllocSize()); + EXPECT_EQ(136, SL(3, 5, 7).AllocSize()); + } + { + using SL = L::WithStaticSizes<3>; + EXPECT_EQ(136, SL::Partial(5, 7).AllocSize()); + EXPECT_EQ(136, SL(5, 7).AllocSize()); + } + { + using SL = L::WithStaticSizes<3, 5>; + EXPECT_EQ(136, SL::Partial(7).AllocSize()); + EXPECT_EQ(136, SL(7).AllocSize()); + } + { + using SL = L::WithStaticSizes<3, 5, 7>; + EXPECT_EQ(136, SL::Partial().AllocSize()); + EXPECT_EQ(136, SL().AllocSize()); + } +} + TEST(Layout, SizeByIndex) { { using L = Layout<int32_t>; @@ -370,6 +421,27 @@ TEST(Layout, Sizes) { } } +TEST(Layout, StaticSize) { + using L = Layout<int8_t, int32_t, Int128>; + { + using SL = L::WithStaticSizes<>; + EXPECT_THAT(SL::Partial().Sizes(), ElementsAre()); + EXPECT_THAT(SL::Partial(3).Size<0>(), 3); + EXPECT_THAT(SL::Partial(3).Size<int8_t>(), 3); + EXPECT_THAT(SL::Partial(3).Sizes(), ElementsAre(3)); + EXPECT_THAT(SL::Partial(3, 5, 7).Size<0>(), 3); + EXPECT_THAT(SL::Partial(3, 5, 7).Size<int8_t>(), 3); + EXPECT_THAT(SL::Partial(3, 5, 7).Size<2>(), 7); + EXPECT_THAT(SL::Partial(3, 5, 7).Size<Int128>(), 7); + EXPECT_THAT(SL::Partial(3, 5, 7).Sizes(), ElementsAre(3, 5, 7)); + EXPECT_THAT(SL(3, 5, 7).Size<0>(), 3); + EXPECT_THAT(SL(3, 5, 7).Size<int8_t>(), 3); + EXPECT_THAT(SL(3, 5, 7).Size<2>(), 7); + EXPECT_THAT(SL(3, 5, 7).Size<Int128>(), 7); + EXPECT_THAT(SL(3, 5, 7).Sizes(), ElementsAre(3, 5, 7)); + } +} + TEST(Layout, PointerByIndex) { alignas(max_align_t) const unsigned char p[100] = {0}; { @@ -720,6 +792,61 @@ TEST(Layout, MutablePointers) { } } +TEST(Layout, StaticPointers) { + alignas(max_align_t) const unsigned char p[100] = {0}; + using L = Layout<int8_t, int8_t, Int128>; + { + const auto x = L::WithStaticSizes<>::Partial(); + EXPECT_EQ(std::make_tuple(x.Pointer<0>(p)), + Type<std::tuple<const int8_t*>>(x.Pointers(p))); + } + { + const auto x = L::WithStaticSizes<>::Partial(1); + EXPECT_EQ(std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p)), + (Type<std::tuple<const int8_t*, const int8_t*>>(x.Pointers(p)))); + } + { + const auto x = L::WithStaticSizes<1>::Partial(); + EXPECT_EQ(std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p)), + (Type<std::tuple<const int8_t*, const int8_t*>>(x.Pointers(p)))); + } + { + const auto x = L::WithStaticSizes<>::Partial(1, 2, 3); + EXPECT_EQ( + std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p), x.Pointer<2>(p)), + (Type<std::tuple<const int8_t*, const int8_t*, const Int128*>>( + x.Pointers(p)))); + } + { + const auto x = L::WithStaticSizes<1>::Partial(2, 3); + EXPECT_EQ( + std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p), x.Pointer<2>(p)), + (Type<std::tuple<const int8_t*, const int8_t*, const Int128*>>( + x.Pointers(p)))); + } + { + const auto x = L::WithStaticSizes<1, 2>::Partial(3); + EXPECT_EQ( + std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p), x.Pointer<2>(p)), + (Type<std::tuple<const int8_t*, const int8_t*, const Int128*>>( + x.Pointers(p)))); + } + { + const auto x = L::WithStaticSizes<1, 2, 3>::Partial(); + EXPECT_EQ( + std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p), x.Pointer<2>(p)), + (Type<std::tuple<const int8_t*, const int8_t*, const Int128*>>( + x.Pointers(p)))); + } + { + const L::WithStaticSizes<1, 2, 3> x; + EXPECT_EQ( + std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p), x.Pointer<2>(p)), + (Type<std::tuple<const int8_t*, const int8_t*, const Int128*>>( + x.Pointers(p)))); + } +} + TEST(Layout, SliceByIndexSize) { alignas(max_align_t) const unsigned char p[100] = {0}; { @@ -769,7 +896,6 @@ TEST(Layout, SliceByTypeSize) { EXPECT_EQ(7, L(3, 5, 7).Slice<Int128>(p).size()); } } - TEST(Layout, MutableSliceByIndexSize) { alignas(max_align_t) unsigned char p[100] = {0}; { @@ -820,6 +946,39 @@ TEST(Layout, MutableSliceByTypeSize) { } } +TEST(Layout, StaticSliceSize) { + alignas(max_align_t) const unsigned char cp[100] = {0}; + alignas(max_align_t) unsigned char p[100] = {0}; + using L = Layout<int8_t, int32_t, Int128>; + using SL = L::WithStaticSizes<3, 5>; + + EXPECT_EQ(3, SL::Partial().Slice<0>(cp).size()); + EXPECT_EQ(3, SL::Partial().Slice<int8_t>(cp).size()); + EXPECT_EQ(3, SL::Partial(7).Slice<0>(cp).size()); + EXPECT_EQ(3, SL::Partial(7).Slice<int8_t>(cp).size()); + + EXPECT_EQ(5, SL::Partial().Slice<1>(cp).size()); + EXPECT_EQ(5, SL::Partial().Slice<int32_t>(cp).size()); + EXPECT_EQ(5, SL::Partial(7).Slice<1>(cp).size()); + EXPECT_EQ(5, SL::Partial(7).Slice<int32_t>(cp).size()); + + EXPECT_EQ(7, SL::Partial(7).Slice<2>(cp).size()); + EXPECT_EQ(7, SL::Partial(7).Slice<Int128>(cp).size()); + + EXPECT_EQ(3, SL::Partial().Slice<0>(p).size()); + EXPECT_EQ(3, SL::Partial().Slice<int8_t>(p).size()); + EXPECT_EQ(3, SL::Partial(7).Slice<0>(p).size()); + EXPECT_EQ(3, SL::Partial(7).Slice<int8_t>(p).size()); + + EXPECT_EQ(5, SL::Partial().Slice<1>(p).size()); + EXPECT_EQ(5, SL::Partial().Slice<int32_t>(p).size()); + EXPECT_EQ(5, SL::Partial(7).Slice<1>(p).size()); + EXPECT_EQ(5, SL::Partial(7).Slice<int32_t>(p).size()); + + EXPECT_EQ(7, SL::Partial(7).Slice<2>(p).size()); + EXPECT_EQ(7, SL::Partial(7).Slice<Int128>(p).size()); +} + TEST(Layout, SliceByIndexData) { alignas(max_align_t) const unsigned char p[100] = {0}; { @@ -1230,6 +1389,39 @@ TEST(Layout, MutableSliceByTypeData) { } } +TEST(Layout, StaticSliceData) { + alignas(max_align_t) const unsigned char cp[100] = {0}; + alignas(max_align_t) unsigned char p[100] = {0}; + using L = Layout<int8_t, int32_t, Int128>; + using SL = L::WithStaticSizes<3, 5>; + + EXPECT_EQ(0, Distance(cp, SL::Partial().Slice<0>(cp).data())); + EXPECT_EQ(0, Distance(cp, SL::Partial().Slice<int8_t>(cp).data())); + EXPECT_EQ(0, Distance(cp, SL::Partial(7).Slice<0>(cp).data())); + EXPECT_EQ(0, Distance(cp, SL::Partial(7).Slice<int8_t>(cp).data())); + + EXPECT_EQ(4, Distance(cp, SL::Partial().Slice<1>(cp).data())); + EXPECT_EQ(4, Distance(cp, SL::Partial().Slice<int32_t>(cp).data())); + EXPECT_EQ(4, Distance(cp, SL::Partial(7).Slice<1>(cp).data())); + EXPECT_EQ(4, Distance(cp, SL::Partial(7).Slice<int32_t>(cp).data())); + + EXPECT_EQ(24, Distance(cp, SL::Partial(7).Slice<2>(cp).data())); + EXPECT_EQ(24, Distance(cp, SL::Partial(7).Slice<Int128>(cp).data())); + + EXPECT_EQ(0, Distance(p, SL::Partial().Slice<0>(p).data())); + EXPECT_EQ(0, Distance(p, SL::Partial().Slice<int8_t>(p).data())); + EXPECT_EQ(0, Distance(p, SL::Partial(7).Slice<0>(p).data())); + EXPECT_EQ(0, Distance(p, SL::Partial(7).Slice<int8_t>(p).data())); + + EXPECT_EQ(4, Distance(p, SL::Partial().Slice<1>(p).data())); + EXPECT_EQ(4, Distance(p, SL::Partial().Slice<int32_t>(p).data())); + EXPECT_EQ(4, Distance(p, SL::Partial(7).Slice<1>(p).data())); + EXPECT_EQ(4, Distance(p, SL::Partial(7).Slice<int32_t>(p).data())); + + EXPECT_EQ(24, Distance(p, SL::Partial(7).Slice<2>(p).data())); + EXPECT_EQ(24, Distance(p, SL::Partial(7).Slice<Int128>(p).data())); +} + MATCHER_P(IsSameSlice, slice, "") { return arg.size() == slice.size() && arg.data() == slice.data(); } @@ -1339,6 +1531,43 @@ TEST(Layout, MutableSlices) { } } +TEST(Layout, StaticSlices) { + alignas(max_align_t) const unsigned char cp[100] = {0}; + alignas(max_align_t) unsigned char p[100] = {0}; + using SL = Layout<int8_t, int8_t, Int128>::WithStaticSizes<1, 2>; + { + const auto x = SL::Partial(); + EXPECT_THAT( + (Type<std::tuple<Span<const int8_t>, Span<const int8_t>>>( + x.Slices(cp))), + Tuple(IsSameSlice(x.Slice<0>(cp)), IsSameSlice(x.Slice<1>(cp)))); + EXPECT_THAT((Type<std::tuple<Span<int8_t>, Span<int8_t>>>(x.Slices(p))), + Tuple(IsSameSlice(x.Slice<0>(p)), IsSameSlice(x.Slice<1>(p)))); + } + { + const auto x = SL::Partial(3); + EXPECT_THAT((Type<std::tuple<Span<const int8_t>, Span<const int8_t>, + Span<const Int128>>>(x.Slices(cp))), + Tuple(IsSameSlice(x.Slice<0>(cp)), IsSameSlice(x.Slice<1>(cp)), + IsSameSlice(x.Slice<2>(cp)))); + EXPECT_THAT((Type<std::tuple<Span<int8_t>, Span<int8_t>, Span<Int128>>>( + x.Slices(p))), + Tuple(IsSameSlice(x.Slice<0>(p)), IsSameSlice(x.Slice<1>(p)), + IsSameSlice(x.Slice<2>(p)))); + } + { + const SL x(3); + EXPECT_THAT((Type<std::tuple<Span<const int8_t>, Span<const int8_t>, + Span<const Int128>>>(x.Slices(cp))), + Tuple(IsSameSlice(x.Slice<0>(cp)), IsSameSlice(x.Slice<1>(cp)), + IsSameSlice(x.Slice<2>(cp)))); + EXPECT_THAT((Type<std::tuple<Span<int8_t>, Span<int8_t>, Span<Int128>>>( + x.Slices(p))), + Tuple(IsSameSlice(x.Slice<0>(p)), IsSameSlice(x.Slice<1>(p)), + IsSameSlice(x.Slice<2>(p)))); + } +} + TEST(Layout, UnalignedTypes) { constexpr Layout<unsigned char, unsigned char, unsigned char> x(1, 2, 3); alignas(max_align_t) unsigned char p[x.AllocSize() + 1]; @@ -1377,6 +1606,36 @@ TEST(Layout, Alignment) { static_assert(Layout<int32_t, Int64, int8_t>::Alignment() == 8, ""); static_assert(Layout<Int64, int8_t, int32_t>::Alignment() == 8, ""); static_assert(Layout<Int64, int32_t, int8_t>::Alignment() == 8, ""); + static_assert(Layout<Int64, int32_t, int8_t>::Alignment() == 8, ""); + static_assert( + Layout<Aligned<int8_t, 64>>::WithStaticSizes<>::Alignment() == 64, ""); + static_assert( + Layout<Aligned<int8_t, 64>>::WithStaticSizes<2>::Alignment() == 64, ""); +} + +TEST(Layout, StaticAlignment) { + static_assert(Layout<int8_t>::WithStaticSizes<>::Alignment() == 1, ""); + static_assert(Layout<int8_t>::WithStaticSizes<0>::Alignment() == 1, ""); + static_assert(Layout<int8_t>::WithStaticSizes<7>::Alignment() == 1, ""); + static_assert(Layout<int32_t>::WithStaticSizes<>::Alignment() == 4, ""); + static_assert(Layout<int32_t>::WithStaticSizes<0>::Alignment() == 4, ""); + static_assert(Layout<int32_t>::WithStaticSizes<3>::Alignment() == 4, ""); + static_assert( + Layout<Aligned<int8_t, 64>>::WithStaticSizes<>::Alignment() == 64, ""); + static_assert( + Layout<Aligned<int8_t, 64>>::WithStaticSizes<0>::Alignment() == 64, ""); + static_assert( + Layout<Aligned<int8_t, 64>>::WithStaticSizes<2>::Alignment() == 64, ""); + static_assert( + Layout<int32_t, Int64, int8_t>::WithStaticSizes<>::Alignment() == 8, ""); + static_assert( + Layout<int32_t, Int64, int8_t>::WithStaticSizes<0, 0, 0>::Alignment() == + 8, + ""); + static_assert( + Layout<int32_t, Int64, int8_t>::WithStaticSizes<1, 1, 1>::Alignment() == + 8, + ""); } TEST(Layout, ConstexprPartial) { @@ -1384,6 +1643,15 @@ TEST(Layout, ConstexprPartial) { constexpr Layout<unsigned char, Aligned<unsigned char, 2 * M>> x(1, 3); static_assert(x.Partial(1).template Offset<1>() == 2 * M, ""); } + +TEST(Layout, StaticConstexpr) { + constexpr size_t M = alignof(max_align_t); + using L = Layout<unsigned char, Aligned<unsigned char, 2 * M>>; + using SL = L::WithStaticSizes<1, 3>; + constexpr SL x; + static_assert(x.Offset<1>() == 2 * M, ""); +} + // [from, to) struct Region { size_t from; @@ -1458,6 +1726,41 @@ TEST(Layout, PoisonPadding) { } } +TEST(Layout, StaticPoisonPadding) { + using L = Layout<int8_t, Int64, int32_t, Int128>; + using SL = L::WithStaticSizes<1, 2>; + + constexpr size_t n = L::Partial(1, 2, 3, 4).AllocSize(); + { + constexpr auto x = SL::Partial(); + alignas(max_align_t) const unsigned char c[n] = {}; + x.PoisonPadding(c); + EXPECT_EQ(x.Slices(c), x.Slices(c)); + ExpectPoisoned(c, {{1, 8}}); + } + { + constexpr auto x = SL::Partial(3); + alignas(max_align_t) const unsigned char c[n] = {}; + x.PoisonPadding(c); + EXPECT_EQ(x.Slices(c), x.Slices(c)); + ExpectPoisoned(c, {{1, 8}, {36, 40}}); + } + { + constexpr auto x = SL::Partial(3, 4); + alignas(max_align_t) const unsigned char c[n] = {}; + x.PoisonPadding(c); + EXPECT_EQ(x.Slices(c), x.Slices(c)); + ExpectPoisoned(c, {{1, 8}, {36, 40}}); + } + { + constexpr SL x(3, 4); + alignas(max_align_t) const unsigned char c[n] = {}; + x.PoisonPadding(c); + EXPECT_EQ(x.Slices(c), x.Slices(c)); + ExpectPoisoned(c, {{1, 8}, {36, 40}}); + } +} + TEST(Layout, DebugString) { { constexpr auto x = Layout<int8_t, int32_t, int8_t, Int128>::Partial(); @@ -1500,6 +1803,62 @@ TEST(Layout, DebugString) { } } +TEST(Layout, StaticDebugString) { + { + constexpr auto x = + Layout<int8_t, int32_t, int8_t, Int128>::WithStaticSizes<>::Partial(); + EXPECT_EQ("@0<signed char>(1)", x.DebugString()); + } + { + constexpr auto x = + Layout<int8_t, int32_t, int8_t, Int128>::WithStaticSizes<>::Partial(1); + EXPECT_EQ("@0<signed char>(1)[1]; @4<int>(4)", x.DebugString()); + } + { + constexpr auto x = + Layout<int8_t, int32_t, int8_t, Int128>::WithStaticSizes<1>::Partial(); + EXPECT_EQ("@0<signed char>(1)[1]; @4<int>(4)", x.DebugString()); + } + { + constexpr auto x = + Layout<int8_t, int32_t, int8_t, Int128>::WithStaticSizes<>::Partial(1, + 2); + EXPECT_EQ("@0<signed char>(1)[1]; @4<int>(4)[2]; @12<signed char>(1)", + x.DebugString()); + } + { + constexpr auto x = + Layout<int8_t, int32_t, int8_t, Int128>::WithStaticSizes<1>::Partial(2); + EXPECT_EQ("@0<signed char>(1)[1]; @4<int>(4)[2]; @12<signed char>(1)", + x.DebugString()); + } + { + constexpr auto x = Layout<int8_t, int32_t, int8_t, + Int128>::WithStaticSizes<1, 2>::Partial(); + EXPECT_EQ("@0<signed char>(1)[1]; @4<int>(4)[2]; @12<signed char>(1)", + x.DebugString()); + } + { + constexpr auto x = Layout<int8_t, int32_t, int8_t, + Int128>::WithStaticSizes<1, 2, 3, 4>::Partial(); + EXPECT_EQ( + "@0<signed char>(1)[1]; @4<int>(4)[2]; @12<signed char>(1)[3]; " + "@16" + + Int128::Name() + "(16)[4]", + x.DebugString()); + } + { + constexpr Layout<int8_t, int32_t, int8_t, Int128>::WithStaticSizes<1, 2, 3, + 4> + x; + EXPECT_EQ( + "@0<signed char>(1)[1]; @4<int>(4)[2]; @12<signed char>(1)[3]; " + "@16" + + Int128::Name() + "(16)[4]", + x.DebugString()); + } +} + TEST(Layout, CharTypes) { constexpr Layout<int32_t> x(1); alignas(max_align_t) char c[x.AllocSize()] = {}; @@ -1638,6 +1997,35 @@ TEST(CompactString, Works) { EXPECT_STREQ("hello", s.c_str()); } +// Same as the previous CompactString example, except we set the first array +// size to 1 statically, since we know it is always 1. This allows us to compute +// the offset of the character array at compile time. +class StaticCompactString { + public: + StaticCompactString(const char* s = "") { // NOLINT + const size_t size = strlen(s); + const SL layout(size + 1); + p_.reset(new unsigned char[layout.AllocSize()]); + layout.PoisonPadding(p_.get()); + *layout.Pointer<size_t>(p_.get()) = size; + memcpy(layout.Pointer<char>(p_.get()), s, size + 1); + } + + size_t size() const { return *SL::Partial().Pointer<size_t>(p_.get()); } + + const char* c_str() const { return SL::Partial().Pointer<char>(p_.get()); } + + private: + using SL = Layout<size_t, char>::WithStaticSizes<1>; + std::unique_ptr<unsigned char[]> p_; +}; + +TEST(StaticCompactString, Works) { + StaticCompactString s = "hello"; + EXPECT_EQ(5, s.size()); + EXPECT_STREQ("hello", s.c_str()); +} + } // namespace example } // namespace diff --git a/absl/container/internal/raw_hash_map.h b/absl/container/internal/raw_hash_map.h index 97182bc7..464bf23b 100644 --- a/absl/container/internal/raw_hash_map.h +++ b/absl/container/internal/raw_hash_map.h @@ -198,22 +198,24 @@ class raw_hash_map : public raw_hash_set<Policy, Hash, Eq, Alloc> { std::pair<iterator, bool> insert_or_assign_impl(K&& k, V&& v) ABSL_ATTRIBUTE_LIFETIME_BOUND { auto res = this->find_or_prepare_insert(k); - if (res.second) + if (res.second) { this->emplace_at(res.first, std::forward<K>(k), std::forward<V>(v)); - else - Policy::value(&*this->iterator_at(res.first)) = std::forward<V>(v); - return {this->iterator_at(res.first), res.second}; + } else { + Policy::value(&*res.first) = std::forward<V>(v); + } + return res; } template <class K = key_type, class... Args> std::pair<iterator, bool> try_emplace_impl(K&& k, Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND { auto res = this->find_or_prepare_insert(k); - if (res.second) + if (res.second) { this->emplace_at(res.first, std::piecewise_construct, std::forward_as_tuple(std::forward<K>(k)), std::forward_as_tuple(std::forward<Args>(args)...)); - return {this->iterator_at(res.first), res.second}; + } + return res; } }; diff --git a/absl/container/internal/raw_hash_set.cc b/absl/container/internal/raw_hash_set.cc index 9f8ea519..1cae0381 100644 --- a/absl/container/internal/raw_hash_set.cc +++ b/absl/container/internal/raw_hash_set.cc @@ -23,19 +23,24 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/dynamic_annotations.h" +#include "absl/base/internal/endian.h" +#include "absl/base/optimization.h" #include "absl/container/internal/container_memory.h" +#include "absl/container/internal/hashtablez_sampler.h" #include "absl/hash/hash.h" namespace absl { ABSL_NAMESPACE_BEGIN namespace container_internal { -// We have space for `growth_left` before a single block of control bytes. A +// Represents a control byte corresponding to a full slot with arbitrary hash. +constexpr ctrl_t ZeroCtrlT() { return static_cast<ctrl_t>(0); } + +// We have space for `growth_info` before a single block of control bytes. A // single block of empty control bytes for tables without any slots allocated. // This enables removing a branch in the hot path of find(). In order to ensure // that the control bytes are aligned to 16, we have 16 bytes before the control -// bytes even though growth_left only needs 8. -constexpr ctrl_t ZeroCtrlT() { return static_cast<ctrl_t>(0); } +// bytes even though growth_info only needs 8. alignas(16) ABSL_CONST_INIT ABSL_DLL const ctrl_t kEmptyGroup[32] = { ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), @@ -46,6 +51,18 @@ alignas(16) ABSL_CONST_INIT ABSL_DLL const ctrl_t kEmptyGroup[32] = { ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty}; +// We need one full byte followed by a sentinel byte for iterator::operator++ to +// work. We have a full group after kSentinel to be safe (in case operator++ is +// changed to read a full group). +ABSL_CONST_INIT ABSL_DLL const ctrl_t kSooControl[17] = { + ZeroCtrlT(), ctrl_t::kSentinel, ZeroCtrlT(), ctrl_t::kEmpty, + ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, + ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, + ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, + ctrl_t::kEmpty}; +static_assert(NumControlBytes(SooCapacity()) <= 17, + "kSooControl capacity too small"); + #ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL constexpr size_t Group::kWidth; #endif @@ -104,10 +121,25 @@ bool CommonFieldsGenerationInfoEnabled::should_rehash_for_bug_detection_on_move( return ShouldRehashForBugDetection(ctrl, capacity); } -bool ShouldInsertBackwards(size_t hash, const ctrl_t* ctrl) { +bool ShouldInsertBackwardsForDebug(size_t capacity, size_t hash, + const ctrl_t* ctrl) { // To avoid problems with weak hashes and single bit tests, we use % 13. // TODO(kfm,sbenza): revisit after we do unconditional mixing - return (H1(hash, ctrl) ^ RandomSeed()) % 13 > 6; + return !is_small(capacity) && (H1(hash, ctrl) ^ RandomSeed()) % 13 > 6; +} + +size_t PrepareInsertAfterSoo(size_t hash, size_t slot_size, + CommonFields& common) { + assert(common.capacity() == NextCapacity(SooCapacity())); + // After resize from capacity 1 to 3, we always have exactly the slot with + // index 1 occupied, so we need to insert either at index 0 or index 2. + assert(HashSetResizeHelper::SooSlotIndex() == 1); + PrepareInsertCommon(common); + const size_t offset = H1(hash, common.control()) & 2; + common.growth_info().OverwriteEmptyAsFull(); + SetCtrlInSingleGroupTable(common, offset, H2(hash), slot_size); + common.infoz().RecordInsert(hash, /*distance_from_desired=*/0); + return offset; } void ConvertDeletedToEmptyAndFullToDeleted(ctrl_t* ctrl, size_t capacity) { @@ -128,6 +160,8 @@ FindInfo find_first_non_full_outofline(const CommonFields& common, return find_first_non_full(common, hash); } +namespace { + // Returns the address of the slot just after slot assuming each slot has the // specified size. static inline void* NextSlot(void* slot, size_t slot_size) { @@ -140,8 +174,22 @@ static inline void* PrevSlot(void* slot, size_t slot_size) { return reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(slot) - slot_size); } +// Finds guaranteed to exists empty slot from the given position. +// NOTE: this function is almost never triggered inside of the +// DropDeletesWithoutResize, so we keep it simple. +// The table is rather sparse, so empty slot will be found very quickly. +size_t FindEmptySlot(size_t start, size_t end, const ctrl_t* ctrl) { + for (size_t i = start; i < end; ++i) { + if (IsEmpty(ctrl[i])) { + return i; + } + } + assert(false && "no empty slot"); + return ~size_t{}; +} + void DropDeletesWithoutResize(CommonFields& common, - const PolicyFunctions& policy, void* tmp_space) { + const PolicyFunctions& policy) { void* set = &common; void* slot_array = common.slot_array(); const size_t capacity = common.capacity(); @@ -165,17 +213,28 @@ void DropDeletesWithoutResize(CommonFields& common, // repeat procedure for current slot with moved from element (target) ctrl_t* ctrl = common.control(); ConvertDeletedToEmptyAndFullToDeleted(ctrl, capacity); + const void* hash_fn = policy.hash_fn(common); auto hasher = policy.hash_slot; auto transfer = policy.transfer; const size_t slot_size = policy.slot_size; size_t total_probe_length = 0; void* slot_ptr = SlotAddress(slot_array, 0, slot_size); + + // The index of an empty slot that can be used as temporary memory for + // the swap operation. + constexpr size_t kUnknownId = ~size_t{}; + size_t tmp_space_id = kUnknownId; + for (size_t i = 0; i != capacity; ++i, slot_ptr = NextSlot(slot_ptr, slot_size)) { assert(slot_ptr == SlotAddress(slot_array, i, slot_size)); + if (IsEmpty(ctrl[i])) { + tmp_space_id = i; + continue; + } if (!IsDeleted(ctrl[i])) continue; - const size_t hash = (*hasher)(set, slot_ptr); + const size_t hash = (*hasher)(hash_fn, slot_ptr); const FindInfo target = find_first_non_full(common, hash); const size_t new_i = target.offset; total_probe_length += target.probe_length; @@ -202,16 +261,26 @@ void DropDeletesWithoutResize(CommonFields& common, SetCtrl(common, new_i, H2(hash), slot_size); (*transfer)(set, new_slot_ptr, slot_ptr); SetCtrl(common, i, ctrl_t::kEmpty, slot_size); + // Initialize or change empty space id. + tmp_space_id = i; } else { assert(IsDeleted(ctrl[new_i])); SetCtrl(common, new_i, H2(hash), slot_size); // Until we are done rehashing, DELETED marks previously FULL slots. + if (tmp_space_id == kUnknownId) { + tmp_space_id = FindEmptySlot(i + 1, capacity, ctrl); + } + void* tmp_space = SlotAddress(slot_array, tmp_space_id, slot_size); + SanitizerUnpoisonMemoryRegion(tmp_space, slot_size); + // Swap i and new_i elements. (*transfer)(set, tmp_space, new_slot_ptr); (*transfer)(set, new_slot_ptr, slot_ptr); (*transfer)(set, slot_ptr, tmp_space); + SanitizerPoisonMemoryRegion(tmp_space, slot_size); + // repeat the processing of the ith slot --i; slot_ptr = PrevSlot(slot_ptr, slot_size); @@ -238,6 +307,8 @@ static bool WasNeverFull(CommonFields& c, size_t index) { Group::kWidth; } +} // namespace + void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size) { assert(IsFull(c.control()[index]) && "erasing a dangling iterator"); c.decrement_size(); @@ -245,17 +316,19 @@ void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size) { if (WasNeverFull(c, index)) { SetCtrl(c, index, ctrl_t::kEmpty, slot_size); - c.set_growth_left(c.growth_left() + 1); + c.growth_info().OverwriteFullAsEmpty(); return; } + c.growth_info().OverwriteFullAsDeleted(); SetCtrl(c, index, ctrl_t::kDeleted, slot_size); } void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy, - bool reuse) { + bool reuse, bool soo_enabled) { c.set_size(0); if (reuse) { + assert(!soo_enabled || c.capacity() > SooCapacity()); ResetCtrl(c, policy.slot_size); ResetGrowthLeft(c); c.infoz().RecordStorageChanged(0, c.capacity()); @@ -263,118 +336,308 @@ void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy, // We need to record infoz before calling dealloc, which will unregister // infoz. c.infoz().RecordClearedReservation(); - c.infoz().RecordStorageChanged(0, 0); + c.infoz().RecordStorageChanged(0, soo_enabled ? SooCapacity() : 0); (*policy.dealloc)(c, policy); - c.set_control(EmptyGroup()); - c.set_generation_ptr(EmptyGeneration()); - c.set_slots(nullptr); - c.set_capacity(0); + c = soo_enabled ? CommonFields{soo_tag_t{}} : CommonFields{}; } } void HashSetResizeHelper::GrowIntoSingleGroupShuffleControlBytes( - ctrl_t* new_ctrl, size_t new_capacity) const { + ctrl_t* __restrict new_ctrl, size_t new_capacity) const { assert(is_single_group(new_capacity)); constexpr size_t kHalfWidth = Group::kWidth / 2; + constexpr size_t kQuarterWidth = Group::kWidth / 4; assert(old_capacity_ < kHalfWidth); + static_assert(sizeof(uint64_t) >= kHalfWidth, + "Group size is too large. The ctrl bytes for half a group must " + "fit into a uint64_t for this implementation."); + static_assert(sizeof(uint64_t) <= Group::kWidth, + "Group size is too small. The ctrl bytes for a group must " + "cover a uint64_t for this implementation."); const size_t half_old_capacity = old_capacity_ / 2; // NOTE: operations are done with compile time known size = kHalfWidth. // Compiler optimizes that into single ASM operation. - // Copy second half of bytes to the beginning. - // We potentially copy more bytes in order to have compile time known size. - // Mirrored bytes from the old_ctrl_ will also be copied. - // In case of old_capacity_ == 3, we will copy 1st element twice. + // Load the bytes from half_old_capacity + 1. This contains the last half of + // old_ctrl bytes, followed by the sentinel byte, and then the first half of + // the cloned bytes. This effectively shuffles the control bytes. + uint64_t copied_bytes = 0; + copied_bytes = + absl::little_endian::Load64(old_ctrl() + half_old_capacity + 1); + + // We change the sentinel byte to kEmpty before storing to both the start of + // the new_ctrl, and past the end of the new_ctrl later for the new cloned + // bytes. Note that this is faster than setting the sentinel byte to kEmpty + // after the copy directly in new_ctrl because we are limited on store + // bandwidth. + constexpr uint64_t kEmptyXorSentinel = + static_cast<uint8_t>(ctrl_t::kEmpty) ^ + static_cast<uint8_t>(ctrl_t::kSentinel); + const uint64_t mask_convert_old_sentinel_to_empty = + kEmptyXorSentinel << (half_old_capacity * 8); + copied_bytes ^= mask_convert_old_sentinel_to_empty; + + // Copy second half of bytes to the beginning. This correctly sets the bytes + // [0, old_capacity]. We potentially copy more bytes in order to have compile + // time known size. Mirrored bytes from the old_ctrl() will also be copied. In + // case of old_capacity_ == 3, we will copy 1st element twice. // Examples: + // (old capacity = 1) // old_ctrl = 0S0EEEEEEE... - // new_ctrl = S0EEEEEEEE... + // new_ctrl = E0EEEEEE??... // - // old_ctrl = 01S01EEEEE... - // new_ctrl = 1S01EEEEEE... + // (old capacity = 3) + // old_ctrl = 012S012EEEEE... + // new_ctrl = 12E012EE????... // + // (old capacity = 7) // old_ctrl = 0123456S0123456EE... - // new_ctrl = 456S0123?????????... - std::memcpy(new_ctrl, old_ctrl_ + half_old_capacity + 1, kHalfWidth); - // Clean up copied kSentinel from old_ctrl. - new_ctrl[half_old_capacity] = ctrl_t::kEmpty; - - // Clean up damaged or uninitialized bytes. - - // Clean bytes after the intended size of the copy. - // Example: - // new_ctrl = 1E01EEEEEEE???? - // *new_ctrl= 1E0EEEEEEEE???? - // position / - std::memset(new_ctrl + old_capacity_ + 1, static_cast<int8_t>(ctrl_t::kEmpty), - kHalfWidth); - // Clean non-mirrored bytes that are not initialized. - // For small old_capacity that may be inside of mirrored bytes zone. + // new_ctrl = 456E0123?????????... + absl::little_endian::Store64(new_ctrl, copied_bytes); + + // Set the space [old_capacity + 1, new_capacity] to empty as these bytes will + // not be written again. This is safe because + // NumControlBytes = new_capacity + kWidth and new_capacity >= + // old_capacity+1. // Examples: - // new_ctrl = 1E0EEEEEEEE??????????.... - // *new_ctrl= 1E0EEEEEEEEEEEEE?????.... - // position / + // (old_capacity = 3, new_capacity = 15) + // new_ctrl = 12E012EE?????????????...?? + // *new_ctrl = 12E0EEEEEEEEEEEEEEEE?...?? + // position / S // - // new_ctrl = 456E0123???????????... - // *new_ctrl= 456E0123EEEEEEEE???... - // position / - std::memset(new_ctrl + kHalfWidth, static_cast<int8_t>(ctrl_t::kEmpty), - kHalfWidth); - // Clean last mirrored bytes that are not initialized - // and will not be overwritten by mirroring. + // (old_capacity = 7, new_capacity = 15) + // new_ctrl = 456E0123?????????????????...?? + // *new_ctrl = 456E0123EEEEEEEEEEEEEEEE?...?? + // position / S + std::memset(new_ctrl + old_capacity_ + 1, static_cast<int8_t>(ctrl_t::kEmpty), + Group::kWidth); + + // Set the last kHalfWidth bytes to empty, to ensure the bytes all the way to + // the end are initialized. // Examples: - // new_ctrl = 1E0EEEEEEEEEEEEE???????? - // *new_ctrl= 1E0EEEEEEEEEEEEEEEEEEEEE - // position S / + // new_ctrl = 12E0EEEEEEEEEEEEEEEE?...??????? + // *new_ctrl = 12E0EEEEEEEEEEEEEEEE???EEEEEEEE + // position S / // - // new_ctrl = 456E0123EEEEEEEE??????????????? - // *new_ctrl= 456E0123EEEEEEEE???????EEEEEEEE - // position S / - std::memset(new_ctrl + new_capacity + kHalfWidth, + // new_ctrl = 456E0123EEEEEEEEEEEEEEEE??????? + // *new_ctrl = 456E0123EEEEEEEEEEEEEEEEEEEEEEE + // position S / + std::memset(new_ctrl + NumControlBytes(new_capacity) - kHalfWidth, static_cast<int8_t>(ctrl_t::kEmpty), kHalfWidth); - // Create mirrored bytes. old_capacity_ < kHalfWidth - // Example: - // new_ctrl = 456E0123EEEEEEEE???????EEEEEEEE - // *new_ctrl= 456E0123EEEEEEEE456E0123EEEEEEE - // position S/ - ctrl_t g[kHalfWidth]; - std::memcpy(g, new_ctrl, kHalfWidth); - std::memcpy(new_ctrl + new_capacity + 1, g, kHalfWidth); + // Copy the first bytes to the end (starting at new_capacity +1) to set the + // cloned bytes. Note that we use the already copied bytes from old_ctrl here + // rather than copying from new_ctrl to avoid a Read-after-Write hazard, since + // new_ctrl was just written to. The first old_capacity-1 bytes are set + // correctly. Then there may be up to old_capacity bytes that need to be + // overwritten, and any remaining bytes will be correctly set to empty. This + // sets [new_capacity + 1, new_capacity +1 + old_capacity] correctly. + // Examples: + // new_ctrl = 12E0EEEEEEEEEEEEEEEE?...??????? + // *new_ctrl = 12E0EEEEEEEEEEEE12E012EEEEEEEEE + // position S/ + // + // new_ctrl = 456E0123EEEEEEEE?...???EEEEEEEE + // *new_ctrl = 456E0123EEEEEEEE456E0123EEEEEEE + // position S/ + absl::little_endian::Store64(new_ctrl + new_capacity + 1, copied_bytes); + + // Set The remaining bytes at the end past the cloned bytes to empty. The + // incorrectly set bytes are [new_capacity + old_capacity + 2, + // min(new_capacity + 1 + kHalfWidth, new_capacity + old_capacity + 2 + + // half_old_capacity)]. Taking the difference, we need to set min(kHalfWidth - + // (old_capacity + 1), half_old_capacity)]. Since old_capacity < kHalfWidth, + // half_old_capacity < kQuarterWidth, so we set kQuarterWidth beginning at + // new_capacity + old_capacity + 2 to kEmpty. + // Examples: + // new_ctrl = 12E0EEEEEEEEEEEE12E012EEEEEEEEE + // *new_ctrl = 12E0EEEEEEEEEEEE12E0EEEEEEEEEEE + // position S / + // + // new_ctrl = 456E0123EEEEEEEE456E0123EEEEEEE + // *new_ctrl = 456E0123EEEEEEEE456E0123EEEEEEE (no change) + // position S / + std::memset(new_ctrl + new_capacity + old_capacity_ + 2, + static_cast<int8_t>(ctrl_t::kEmpty), kQuarterWidth); + + // Finally, we set the new sentinel byte. + new_ctrl[new_capacity] = ctrl_t::kSentinel; +} - // Finally set sentinel to its place. +void HashSetResizeHelper::InitControlBytesAfterSoo(ctrl_t* new_ctrl, ctrl_t h2, + size_t new_capacity) { + assert(is_single_group(new_capacity)); + std::memset(new_ctrl, static_cast<int8_t>(ctrl_t::kEmpty), + NumControlBytes(new_capacity)); + assert(HashSetResizeHelper::SooSlotIndex() == 1); + // This allows us to avoid branching on had_soo_slot_. + assert(had_soo_slot_ || h2 == ctrl_t::kEmpty); + new_ctrl[1] = new_ctrl[new_capacity + 2] = h2; new_ctrl[new_capacity] = ctrl_t::kSentinel; } void HashSetResizeHelper::GrowIntoSingleGroupShuffleTransferableSlots( - void* old_slots, void* new_slots, size_t slot_size) const { + void* new_slots, size_t slot_size) const { assert(old_capacity_ > 0); const size_t half_old_capacity = old_capacity_ / 2; - SanitizerUnpoisonMemoryRegion(old_slots, slot_size * old_capacity_); + SanitizerUnpoisonMemoryRegion(old_slots(), slot_size * old_capacity_); std::memcpy(new_slots, - SlotAddress(old_slots, half_old_capacity + 1, slot_size), + SlotAddress(old_slots(), half_old_capacity + 1, slot_size), slot_size * half_old_capacity); std::memcpy(SlotAddress(new_slots, half_old_capacity + 1, slot_size), - old_slots, slot_size * (half_old_capacity + 1)); + old_slots(), slot_size * (half_old_capacity + 1)); } void HashSetResizeHelper::GrowSizeIntoSingleGroupTransferable( - CommonFields& c, void* old_slots, size_t slot_size) { + CommonFields& c, size_t slot_size) { assert(old_capacity_ < Group::kWidth / 2); assert(is_single_group(c.capacity())); assert(IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity())); GrowIntoSingleGroupShuffleControlBytes(c.control(), c.capacity()); - GrowIntoSingleGroupShuffleTransferableSlots(old_slots, c.slot_array(), - slot_size); + GrowIntoSingleGroupShuffleTransferableSlots(c.slot_array(), slot_size); // We poison since GrowIntoSingleGroupShuffleTransferableSlots // may leave empty slots unpoisoned. PoisonSingleGroupEmptySlots(c, slot_size); } +void HashSetResizeHelper::TransferSlotAfterSoo(CommonFields& c, + size_t slot_size) { + assert(was_soo_); + assert(had_soo_slot_); + assert(is_single_group(c.capacity())); + std::memcpy(SlotAddress(c.slot_array(), SooSlotIndex(), slot_size), + old_soo_data(), slot_size); + PoisonSingleGroupEmptySlots(c, slot_size); +} + +namespace { + +// Called whenever the table needs to vacate empty slots either by removing +// tombstones via rehash or growth. +ABSL_ATTRIBUTE_NOINLINE +FindInfo FindInsertPositionWithGrowthOrRehash(CommonFields& common, size_t hash, + const PolicyFunctions& policy) { + const size_t cap = common.capacity(); + if (cap > Group::kWidth && + // Do these calculations in 64-bit to avoid overflow. + common.size() * uint64_t{32} <= cap * uint64_t{25}) { + // Squash DELETED without growing if there is enough capacity. + // + // Rehash in place if the current size is <= 25/32 of capacity. + // Rationale for such a high factor: 1) DropDeletesWithoutResize() is + // faster than resize, and 2) it takes quite a bit of work to add + // tombstones. In the worst case, seems to take approximately 4 + // insert/erase pairs to create a single tombstone and so if we are + // rehashing because of tombstones, we can afford to rehash-in-place as + // long as we are reclaiming at least 1/8 the capacity without doing more + // than 2X the work. (Where "work" is defined to be size() for rehashing + // or rehashing in place, and 1 for an insert or erase.) But rehashing in + // place is faster per operation than inserting or even doubling the size + // of the table, so we actually afford to reclaim even less space from a + // resize-in-place. The decision is to rehash in place if we can reclaim + // at about 1/8th of the usable capacity (specifically 3/28 of the + // capacity) which means that the total cost of rehashing will be a small + // fraction of the total work. + // + // Here is output of an experiment using the BM_CacheInSteadyState + // benchmark running the old case (where we rehash-in-place only if we can + // reclaim at least 7/16*capacity) vs. this code (which rehashes in place + // if we can recover 3/32*capacity). + // + // Note that although in the worst-case number of rehashes jumped up from + // 15 to 190, but the number of operations per second is almost the same. + // + // Abridged output of running BM_CacheInSteadyState benchmark from + // raw_hash_set_benchmark. N is the number of insert/erase operations. + // + // | OLD (recover >= 7/16 | NEW (recover >= 3/32) + // size | N/s LoadFactor NRehashes | N/s LoadFactor NRehashes + // 448 | 145284 0.44 18 | 140118 0.44 19 + // 493 | 152546 0.24 11 | 151417 0.48 28 + // 538 | 151439 0.26 11 | 151152 0.53 38 + // 583 | 151765 0.28 11 | 150572 0.57 50 + // 628 | 150241 0.31 11 | 150853 0.61 66 + // 672 | 149602 0.33 12 | 150110 0.66 90 + // 717 | 149998 0.35 12 | 149531 0.70 129 + // 762 | 149836 0.37 13 | 148559 0.74 190 + // 807 | 149736 0.39 14 | 151107 0.39 14 + // 852 | 150204 0.42 15 | 151019 0.42 15 + DropDeletesWithoutResize(common, policy); + } else { + // Otherwise grow the container. + policy.resize(common, NextCapacity(cap), HashtablezInfoHandle{}); + } + // This function is typically called with tables containing deleted slots. + // The table will be big and `FindFirstNonFullAfterResize` will always + // fallback to `find_first_non_full`. So using `find_first_non_full` directly. + return find_first_non_full(common, hash); +} + +} // namespace + +const void* GetHashRefForEmptyHasher(const CommonFields& common) { + // Empty base optimization typically make the empty base class address to be + // the same as the first address of the derived class object. + // But we generally assume that for empty hasher we can return any valid + // pointer. + return &common; +} + +size_t PrepareInsertNonSoo(CommonFields& common, size_t hash, FindInfo target, + const PolicyFunctions& policy) { + // When there are no deleted slots in the table + // and growth_left is positive, we can insert at the first + // empty slot in the probe sequence (target). + const bool use_target_hint = + // Optimization is disabled when generations are enabled. + // We have to rehash even sparse tables randomly in such mode. + !SwisstableGenerationsEnabled() && + common.growth_info().HasNoDeletedAndGrowthLeft(); + if (ABSL_PREDICT_FALSE(!use_target_hint)) { + // Notes about optimized mode when generations are disabled: + // We do not enter this branch if table has no deleted slots + // and growth_left is positive. + // We enter this branch in the following cases listed in decreasing + // frequency: + // 1. Table without deleted slots (>95% cases) that needs to be resized. + // 2. Table with deleted slots that has space for the inserting element. + // 3. Table with deleted slots that needs to be rehashed or resized. + if (ABSL_PREDICT_TRUE(common.growth_info().HasNoGrowthLeftAndNoDeleted())) { + const size_t old_capacity = common.capacity(); + policy.resize(common, NextCapacity(old_capacity), HashtablezInfoHandle{}); + target = HashSetResizeHelper::FindFirstNonFullAfterResize( + common, old_capacity, hash); + } else { + // Note: the table may have no deleted slots here when generations + // are enabled. + const bool rehash_for_bug_detection = + common.should_rehash_for_bug_detection_on_insert(); + if (rehash_for_bug_detection) { + // Move to a different heap allocation in order to detect bugs. + const size_t cap = common.capacity(); + policy.resize(common, + common.growth_left() > 0 ? cap : NextCapacity(cap), + HashtablezInfoHandle{}); + } + if (ABSL_PREDICT_TRUE(common.growth_left() > 0)) { + target = find_first_non_full(common, hash); + } else { + target = FindInsertPositionWithGrowthOrRehash(common, hash, policy); + } + } + } + PrepareInsertCommon(common); + common.growth_info().OverwriteControlAsFull(common.control()[target.offset]); + SetCtrl(common, target.offset, H2(hash), policy.slot_size); + common.infoz().RecordInsert(hash, target.probe_length); + return target.offset; +} + } // namespace container_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 3518bc34..d4fe8f5c 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -80,7 +80,7 @@ // slot_type slots[capacity]; // }; // -// The length of this array is computed by `AllocSize()` below. +// The length of this array is computed by `RawHashSetLayout::alloc_size` below. // // Control bytes (`ctrl_t`) are bytes (collected into groups of a // platform-specific size) that define the state of the corresponding slot in @@ -100,6 +100,13 @@ // Storing control bytes in a separate array also has beneficial cache effects, // since more logical slots will fit into a cache line. // +// # Small Object Optimization (SOO) +// +// When the size/alignment of the value_type and the capacity of the table are +// small, we enable small object optimization and store the values inline in +// the raw_hash_set object. This optimization allows us to avoid +// allocation/deallocation as well as cache/dTLB misses. +// // # Hashing // // We compute two separate hashes, `H1` and `H2`, from the hash of an object. @@ -233,9 +240,10 @@ namespace container_internal { #ifdef ABSL_SWISSTABLE_ENABLE_GENERATIONS #error ABSL_SWISSTABLE_ENABLE_GENERATIONS cannot be directly set -#elif defined(ABSL_HAVE_ADDRESS_SANITIZER) || \ - defined(ABSL_HAVE_HWADDRESS_SANITIZER) || \ - defined(ABSL_HAVE_MEMORY_SANITIZER) +#elif (defined(ABSL_HAVE_ADDRESS_SANITIZER) || \ + defined(ABSL_HAVE_HWADDRESS_SANITIZER) || \ + defined(ABSL_HAVE_MEMORY_SANITIZER)) && \ + !defined(NDEBUG_SANITIZER) // If defined, performance is important. // When compiled in sanitizer mode, we add generation integers to the backing // array and iterators. In the backing array, we store the generation between // the control bytes and the slots. When iterators are dereferenced, we assert @@ -374,6 +382,9 @@ uint32_t TrailingZeros(T x) { return static_cast<uint32_t>(countr_zero(x)); } +// 8 bytes bitmask with most significant bit set for every byte. +constexpr uint64_t kMsbs8Bytes = 0x8080808080808080ULL; + // An abstract bitmask, such as that emitted by a SIMD instruction. // // Specifically, this type implements a simple bitset whose representation is @@ -423,27 +434,35 @@ class NonIterableBitMask { // an ordinary 16-bit bitset occupying the low 16 bits of `mask`. When // `SignificantBits` is 8 and `Shift` is 3, abstract bits are represented as // the bytes `0x00` and `0x80`, and it occupies all 64 bits of the bitmask. +// If NullifyBitsOnIteration is true (only allowed for Shift == 3), +// non zero abstract bit is allowed to have additional bits +// (e.g., `0xff`, `0x83` and `0x9c` are ok, but `0x6f` is not). // // For example: // for (int i : BitMask<uint32_t, 16>(0b101)) -> yields 0, 2 // for (int i : BitMask<uint64_t, 8, 3>(0x0000000080800000)) -> yields 2, 3 -template <class T, int SignificantBits, int Shift = 0> +template <class T, int SignificantBits, int Shift = 0, + bool NullifyBitsOnIteration = false> class BitMask : public NonIterableBitMask<T, SignificantBits, Shift> { using Base = NonIterableBitMask<T, SignificantBits, Shift>; static_assert(std::is_unsigned<T>::value, ""); static_assert(Shift == 0 || Shift == 3, ""); + static_assert(!NullifyBitsOnIteration || Shift == 3, ""); public: - explicit BitMask(T mask) : Base(mask) {} + explicit BitMask(T mask) : Base(mask) { + if (Shift == 3 && !NullifyBitsOnIteration) { + assert(this->mask_ == (this->mask_ & kMsbs8Bytes)); + } + } // BitMask is an iterator over the indices of its abstract bits. using value_type = int; using iterator = BitMask; using const_iterator = BitMask; BitMask& operator++() { - if (Shift == 3) { - constexpr uint64_t msbs = 0x8080808080808080ULL; - this->mask_ &= msbs; + if (Shift == 3 && NullifyBitsOnIteration) { + this->mask_ &= kMsbs8Bytes; } this->mask_ &= (this->mask_ - 1); return *this; @@ -520,10 +539,24 @@ ABSL_DLL extern const ctrl_t kEmptyGroup[32]; // Returns a pointer to a control byte group that can be used by empty tables. inline ctrl_t* EmptyGroup() { // Const must be cast away here; no uses of this function will actually write - // to it, because it is only used for empty tables. + // to it because it is only used for empty tables. return const_cast<ctrl_t*>(kEmptyGroup + 16); } +// For use in SOO iterators. +// TODO(b/289225379): we could potentially get rid of this by adding an is_soo +// bit in iterators. This would add branches but reduce cache misses. +ABSL_DLL extern const ctrl_t kSooControl[17]; + +// Returns a pointer to a full byte followed by a sentinel byte. +inline ctrl_t* SooControl() { + // Const must be cast away here; no uses of this function will actually write + // to it because it is only used for SOO iterators. + return const_cast<ctrl_t*>(kSooControl); +} +// Whether ctrl is from the SooControl array. +inline bool IsSooControl(const ctrl_t* ctrl) { return ctrl == SooControl(); } + // Returns a pointer to a generation to use for an empty hashtable. GenerationType* EmptyGeneration(); @@ -535,7 +568,37 @@ inline bool IsEmptyGeneration(const GenerationType* generation) { // Mixes a randomly generated per-process seed with `hash` and `ctrl` to // randomize insertion order within groups. -bool ShouldInsertBackwards(size_t hash, const ctrl_t* ctrl); +bool ShouldInsertBackwardsForDebug(size_t capacity, size_t hash, + const ctrl_t* ctrl); + +ABSL_ATTRIBUTE_ALWAYS_INLINE inline bool ShouldInsertBackwards( + ABSL_ATTRIBUTE_UNUSED size_t capacity, ABSL_ATTRIBUTE_UNUSED size_t hash, + ABSL_ATTRIBUTE_UNUSED const ctrl_t* ctrl) { +#if defined(NDEBUG) + return false; +#else + return ShouldInsertBackwardsForDebug(capacity, hash, ctrl); +#endif +} + +// Returns insert position for the given mask. +// We want to add entropy even when ASLR is not enabled. +// In debug build we will randomly insert in either the front or back of +// the group. +// TODO(kfm,sbenza): revisit after we do unconditional mixing +template <class Mask> +ABSL_ATTRIBUTE_ALWAYS_INLINE inline auto GetInsertionOffset( + Mask mask, ABSL_ATTRIBUTE_UNUSED size_t capacity, + ABSL_ATTRIBUTE_UNUSED size_t hash, + ABSL_ATTRIBUTE_UNUSED const ctrl_t* ctrl) { +#if defined(NDEBUG) + return mask.LowestBitSet(); +#else + return ShouldInsertBackwardsForDebug(capacity, hash, ctrl) + ? mask.HighestBitSet() + : mask.LowestBitSet(); +#endif +} // Returns a per-table, hash salt, which changes on resize. This gets mixed into // H1 to randomize iteration order per-table. @@ -560,7 +623,12 @@ inline h2_t H2(size_t hash) { return hash & 0x7F; } // Helpers for checking the state of a control byte. inline bool IsEmpty(ctrl_t c) { return c == ctrl_t::kEmpty; } -inline bool IsFull(ctrl_t c) { return c >= static_cast<ctrl_t>(0); } +inline bool IsFull(ctrl_t c) { + // Cast `c` to the underlying type instead of casting `0` to `ctrl_t` as `0` + // is not a value in the enum. Both ways are equivalent, but this way makes + // linters happier. + return static_cast<std::underlying_type_t<ctrl_t>>(c) >= 0; +} inline bool IsDeleted(ctrl_t c) { return c == ctrl_t::kDeleted; } inline bool IsEmptyOrDeleted(ctrl_t c) { return c < ctrl_t::kSentinel; } @@ -646,6 +714,14 @@ struct GroupSse2Impl { static_cast<uint16_t>(_mm_movemask_epi8(ctrl) ^ 0xffff)); } + // Returns a bitmask representing the positions of non full slots. + // Note: this includes: kEmpty, kDeleted, kSentinel. + // It is useful in contexts when kSentinel is not present. + auto MaskNonFull() const { + return BitMask<uint16_t, kWidth>( + static_cast<uint16_t>(_mm_movemask_epi8(ctrl))); + } + // Returns a bitmask representing the positions of empty or deleted slots. NonIterableBitMask<uint16_t, kWidth> MaskEmptyOrDeleted() const { auto special = _mm_set1_epi8(static_cast<char>(ctrl_t::kSentinel)); @@ -685,10 +761,11 @@ struct GroupAArch64Impl { ctrl = vld1_u8(reinterpret_cast<const uint8_t*>(pos)); } - BitMask<uint64_t, kWidth, 3> Match(h2_t hash) const { + auto Match(h2_t hash) const { uint8x8_t dup = vdup_n_u8(hash); auto mask = vceq_u8(ctrl, dup); - return BitMask<uint64_t, kWidth, 3>( + return BitMask<uint64_t, kWidth, /*Shift=*/3, + /*NullifyBitsOnIteration=*/true>( vget_lane_u64(vreinterpret_u64_u8(mask), 0)); } @@ -704,12 +781,25 @@ struct GroupAArch64Impl { // Returns a bitmask representing the positions of full slots. // Note: for `is_small()` tables group may contain the "same" slot twice: // original and mirrored. - BitMask<uint64_t, kWidth, 3> MaskFull() const { + auto MaskFull() const { uint64_t mask = vget_lane_u64( vreinterpret_u64_u8(vcge_s8(vreinterpret_s8_u8(ctrl), vdup_n_s8(static_cast<int8_t>(0)))), 0); - return BitMask<uint64_t, kWidth, 3>(mask); + return BitMask<uint64_t, kWidth, /*Shift=*/3, + /*NullifyBitsOnIteration=*/true>(mask); + } + + // Returns a bitmask representing the positions of non full slots. + // Note: this includes: kEmpty, kDeleted, kSentinel. + // It is useful in contexts when kSentinel is not present. + auto MaskNonFull() const { + uint64_t mask = vget_lane_u64( + vreinterpret_u64_u8(vclt_s8(vreinterpret_s8_u8(ctrl), + vdup_n_s8(static_cast<int8_t>(0)))), + 0); + return BitMask<uint64_t, kWidth, /*Shift=*/3, + /*NullifyBitsOnIteration=*/true>(mask); } NonIterableBitMask<uint64_t, kWidth, 3> MaskEmptyOrDeleted() const { @@ -736,11 +826,10 @@ struct GroupAArch64Impl { void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(ctrl), 0); - constexpr uint64_t msbs = 0x8080808080808080ULL; constexpr uint64_t slsbs = 0x0202020202020202ULL; constexpr uint64_t midbs = 0x7e7e7e7e7e7e7e7eULL; auto x = slsbs & (mask >> 6); - auto res = (x + midbs) | msbs; + auto res = (x + midbs) | kMsbs8Bytes; little_endian::Store64(dst, res); } @@ -768,30 +857,33 @@ struct GroupPortableImpl { // v = 0x1716151413121110 // hash = 0x12 // retval = (v - lsbs) & ~v & msbs = 0x0000000080800000 - constexpr uint64_t msbs = 0x8080808080808080ULL; constexpr uint64_t lsbs = 0x0101010101010101ULL; auto x = ctrl ^ (lsbs * hash); - return BitMask<uint64_t, kWidth, 3>((x - lsbs) & ~x & msbs); + return BitMask<uint64_t, kWidth, 3>((x - lsbs) & ~x & kMsbs8Bytes); } NonIterableBitMask<uint64_t, kWidth, 3> MaskEmpty() const { - constexpr uint64_t msbs = 0x8080808080808080ULL; return NonIterableBitMask<uint64_t, kWidth, 3>((ctrl & ~(ctrl << 6)) & - msbs); + kMsbs8Bytes); } // Returns a bitmask representing the positions of full slots. // Note: for `is_small()` tables group may contain the "same" slot twice: // original and mirrored. BitMask<uint64_t, kWidth, 3> MaskFull() const { - constexpr uint64_t msbs = 0x8080808080808080ULL; - return BitMask<uint64_t, kWidth, 3>((ctrl ^ msbs) & msbs); + return BitMask<uint64_t, kWidth, 3>((ctrl ^ kMsbs8Bytes) & kMsbs8Bytes); + } + + // Returns a bitmask representing the positions of non full slots. + // Note: this includes: kEmpty, kDeleted, kSentinel. + // It is useful in contexts when kSentinel is not present. + auto MaskNonFull() const { + return BitMask<uint64_t, kWidth, 3>(ctrl & kMsbs8Bytes); } NonIterableBitMask<uint64_t, kWidth, 3> MaskEmptyOrDeleted() const { - constexpr uint64_t msbs = 0x8080808080808080ULL; return NonIterableBitMask<uint64_t, kWidth, 3>((ctrl & ~(ctrl << 7)) & - msbs); + kMsbs8Bytes); } uint32_t CountLeadingEmptyOrDeleted() const { @@ -803,9 +895,8 @@ struct GroupPortableImpl { } void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { - constexpr uint64_t msbs = 0x8080808080808080ULL; constexpr uint64_t lsbs = 0x0101010101010101ULL; - auto x = ctrl & msbs; + auto x = ctrl & kMsbs8Bytes; auto res = (~x + (x >> 7)) & ~lsbs; little_endian::Store64(dst, res); } @@ -815,21 +906,21 @@ struct GroupPortableImpl { #ifdef ABSL_INTERNAL_HAVE_SSE2 using Group = GroupSse2Impl; -using GroupEmptyOrDeleted = GroupSse2Impl; +using GroupFullEmptyOrDeleted = GroupSse2Impl; #elif defined(ABSL_INTERNAL_HAVE_ARM_NEON) && defined(ABSL_IS_LITTLE_ENDIAN) using Group = GroupAArch64Impl; // For Aarch64, we use the portable implementation for counting and masking -// empty or deleted group elements. This is to avoid the latency of moving +// full, empty or deleted group elements. This is to avoid the latency of moving // between data GPRs and Neon registers when it does not provide a benefit. // Using Neon is profitable when we call Match(), but is not when we don't, -// which is the case when we do *EmptyOrDeleted operations. It is difficult to -// make a similar approach beneficial on other architectures such as x86 since -// they have much lower GPR <-> vector register transfer latency and 16-wide -// Groups. -using GroupEmptyOrDeleted = GroupPortableImpl; +// which is the case when we do *EmptyOrDeleted and MaskFull operations. +// It is difficult to make a similar approach beneficial on other architectures +// such as x86 since they have much lower GPR <-> vector register transfer +// latency and 16-wide Groups. +using GroupFullEmptyOrDeleted = GroupPortableImpl; #else using Group = GroupPortableImpl; -using GroupEmptyOrDeleted = GroupPortableImpl; +using GroupFullEmptyOrDeleted = GroupPortableImpl; #endif // When there is an insertion with no reserved growth, we rehash with @@ -978,17 +1069,96 @@ using CommonFieldsGenerationInfo = CommonFieldsGenerationInfoDisabled; using HashSetIteratorGenerationInfo = HashSetIteratorGenerationInfoDisabled; #endif +// Stored the information regarding number of slots we can still fill +// without needing to rehash. +// +// We want to ensure sufficient number of empty slots in the table in order +// to keep probe sequences relatively short. Empty slot in the probe group +// is required to stop probing. +// +// Tombstones (kDeleted slots) are not included in the growth capacity, +// because we'd like to rehash when the table is filled with tombstones and/or +// full slots. +// +// GrowthInfo also stores a bit that encodes whether table may have any +// deleted slots. +// Most of the tables (>95%) have no deleted slots, so some functions can +// be more efficient with this information. +// +// Callers can also force a rehash via the standard `rehash(0)`, +// which will recompute this value as a side-effect. +// +// See also `CapacityToGrowth()`. +class GrowthInfo { + public: + // Leaves data member uninitialized. + GrowthInfo() = default; + + // Initializes the GrowthInfo assuming we can grow `growth_left` elements + // and there are no kDeleted slots in the table. + void InitGrowthLeftNoDeleted(size_t growth_left) { + growth_left_info_ = growth_left; + } + + // Overwrites single full slot with an empty slot. + void OverwriteFullAsEmpty() { ++growth_left_info_; } + + // Overwrites single empty slot with a full slot. + void OverwriteEmptyAsFull() { + assert(GetGrowthLeft() > 0); + --growth_left_info_; + } + + // Overwrites several empty slots with full slots. + void OverwriteManyEmptyAsFull(size_t cnt) { + assert(GetGrowthLeft() >= cnt); + growth_left_info_ -= cnt; + } + + // Overwrites specified control element with full slot. + void OverwriteControlAsFull(ctrl_t ctrl) { + assert(GetGrowthLeft() >= static_cast<size_t>(IsEmpty(ctrl))); + growth_left_info_ -= static_cast<size_t>(IsEmpty(ctrl)); + } + + // Overwrites single full slot with a deleted slot. + void OverwriteFullAsDeleted() { growth_left_info_ |= kDeletedBit; } + + // Returns true if table satisfies two properties: + // 1. Guaranteed to have no kDeleted slots. + // 2. There is a place for at least one element to grow. + bool HasNoDeletedAndGrowthLeft() const { + return static_cast<std::make_signed_t<size_t>>(growth_left_info_) > 0; + } + + // Returns true if the table satisfies two properties: + // 1. Guaranteed to have no kDeleted slots. + // 2. There is no growth left. + bool HasNoGrowthLeftAndNoDeleted() const { return growth_left_info_ == 0; } + + // Returns true if table guaranteed to have no k + bool HasNoDeleted() const { + return static_cast<std::make_signed_t<size_t>>(growth_left_info_) >= 0; + } + + // Returns the number of elements left to grow. + size_t GetGrowthLeft() const { return growth_left_info_ & kGrowthLeftMask; } + + private: + static constexpr size_t kGrowthLeftMask = ((~size_t{}) >> 1); + static constexpr size_t kDeletedBit = ~kGrowthLeftMask; + // Topmost bit signal whenever there are deleted slots. + size_t growth_left_info_; +}; + +static_assert(sizeof(GrowthInfo) == sizeof(size_t), ""); +static_assert(alignof(GrowthInfo) == alignof(size_t), ""); + // Returns whether `n` is a valid capacity (i.e., number of slots). // // A valid capacity is a non-zero integer `2^m - 1`. inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; } -// Computes the offset from the start of the backing allocation of control. -// infoz and growth_left are stored at the beginning of the backing array. -inline size_t ControlOffset(bool has_infoz) { - return (has_infoz ? sizeof(HashtablezInfoHandle) : 0) + sizeof(size_t); -} - // Returns the number of "cloned control bytes". // // This is the number of control bytes that are present both at the beginning @@ -996,36 +1166,157 @@ inline size_t ControlOffset(bool has_infoz) { // `Group::kWidth`-width probe window starting from any control byte. constexpr size_t NumClonedBytes() { return Group::kWidth - 1; } -// Given the capacity of a table, computes the offset (from the start of the -// backing allocation) of the generation counter (if it exists). -inline size_t GenerationOffset(size_t capacity, bool has_infoz) { - assert(IsValidCapacity(capacity)); - const size_t num_control_bytes = capacity + 1 + NumClonedBytes(); - return ControlOffset(has_infoz) + num_control_bytes; +// Returns the number of control bytes including cloned. +constexpr size_t NumControlBytes(size_t capacity) { + return capacity + 1 + NumClonedBytes(); } -// Given the capacity of a table, computes the offset (from the start of the -// backing allocation) at which the slots begin. -inline size_t SlotOffset(size_t capacity, size_t slot_align, bool has_infoz) { - assert(IsValidCapacity(capacity)); - return (GenerationOffset(capacity, has_infoz) + NumGenerationBytes() + - slot_align - 1) & - (~slot_align + 1); +// Computes the offset from the start of the backing allocation of control. +// infoz and growth_info are stored at the beginning of the backing array. +inline static size_t ControlOffset(bool has_infoz) { + return (has_infoz ? sizeof(HashtablezInfoHandle) : 0) + sizeof(GrowthInfo); } -// Given the capacity of a table, computes the total size of the backing -// array. -inline size_t AllocSize(size_t capacity, size_t slot_size, size_t slot_align, - bool has_infoz) { - return SlotOffset(capacity, slot_align, has_infoz) + capacity * slot_size; -} +// Helper class for computing offsets and allocation size of hash set fields. +class RawHashSetLayout { + public: + explicit RawHashSetLayout(size_t capacity, size_t slot_align, bool has_infoz) + : capacity_(capacity), + control_offset_(ControlOffset(has_infoz)), + generation_offset_(control_offset_ + NumControlBytes(capacity)), + slot_offset_( + (generation_offset_ + NumGenerationBytes() + slot_align - 1) & + (~slot_align + 1)) { + assert(IsValidCapacity(capacity)); + } + + // Returns the capacity of a table. + size_t capacity() const { return capacity_; } + + // Returns precomputed offset from the start of the backing allocation of + // control. + size_t control_offset() const { return control_offset_; } + + // Given the capacity of a table, computes the offset (from the start of the + // backing allocation) of the generation counter (if it exists). + size_t generation_offset() const { return generation_offset_; } + + // Given the capacity of a table, computes the offset (from the start of the + // backing allocation) at which the slots begin. + size_t slot_offset() const { return slot_offset_; } + + // Given the capacity of a table, computes the total size of the backing + // array. + size_t alloc_size(size_t slot_size) const { + return slot_offset_ + capacity_ * slot_size; + } + + private: + size_t capacity_; + size_t control_offset_; + size_t generation_offset_; + size_t slot_offset_; +}; + +struct HashtableFreeFunctionsAccess; + +// We only allow a maximum of 1 SOO element, which makes the implementation +// much simpler. Complications with multiple SOO elements include: +// - Satisfying the guarantee that erasing one element doesn't invalidate +// iterators to other elements means we would probably need actual SOO +// control bytes. +// - In order to prevent user code from depending on iteration order for small +// tables, we would need to randomize the iteration order somehow. +constexpr size_t SooCapacity() { return 1; } +// Sentinel type to indicate SOO CommonFields construction. +struct soo_tag_t {}; +// Sentinel type to indicate SOO CommonFields construction with full size. +struct full_soo_tag_t {}; + +// Suppress erroneous uninitialized memory errors on GCC. For example, GCC +// thinks that the call to slot_array() in find_or_prepare_insert() is reading +// uninitialized memory, but slot_array is only called there when the table is +// non-empty and this memory is initialized when the table is non-empty. +#if !defined(__clang__) && defined(__GNUC__) +#define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED(x) \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wmaybe-uninitialized\"") \ + _Pragma("GCC diagnostic ignored \"-Wuninitialized\"") x; \ + _Pragma("GCC diagnostic pop") +#define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(x) \ + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED(return x) +#else +#define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED(x) x +#define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(x) return x +#endif + +// This allows us to work around an uninitialized memory warning when +// constructing begin() iterators in empty hashtables. +union MaybeInitializedPtr { + void* get() const { ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(p); } + void set(void* ptr) { p = ptr; } + + void* p; +}; + +struct HeapPtrs { + HeapPtrs() = default; + explicit HeapPtrs(ctrl_t* c) : control(c) {} + + // The control bytes (and, also, a pointer near to the base of the backing + // array). + // + // This contains `capacity + 1 + NumClonedBytes()` entries, even + // when the table is empty (hence EmptyGroup). + // + // Note that growth_info is stored immediately before this pointer. + // May be uninitialized for SOO tables. + ctrl_t* control; + + // The beginning of the slots, located at `SlotOffset()` bytes after + // `control`. May be uninitialized for empty tables. + // Note: we can't use `slots` because Qt defines "slots" as a macro. + MaybeInitializedPtr slot_array; +}; + +// Manages the backing array pointers or the SOO slot. When raw_hash_set::is_soo +// is true, the SOO slot is stored in `soo_data`. Otherwise, we use `heap`. +union HeapOrSoo { + HeapOrSoo() = default; + explicit HeapOrSoo(ctrl_t* c) : heap(c) {} + + ctrl_t*& control() { + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap.control); + } + ctrl_t* control() const { + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap.control); + } + MaybeInitializedPtr& slot_array() { + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap.slot_array); + } + MaybeInitializedPtr slot_array() const { + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap.slot_array); + } + void* get_soo_data() { + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(soo_data); + } + const void* get_soo_data() const { + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(soo_data); + } + + HeapPtrs heap; + unsigned char soo_data[sizeof(HeapPtrs)]; +}; // CommonFields hold the fields in raw_hash_set that do not depend // on template parameters. This allows us to conveniently pass all // of this state to helper functions as a single argument. class CommonFields : public CommonFieldsGenerationInfo { public: - CommonFields() = default; + CommonFields() : capacity_(0), size_(0), heap_or_soo_(EmptyGroup()) {} + explicit CommonFields(soo_tag_t) : capacity_(SooCapacity()), size_(0) {} + explicit CommonFields(full_soo_tag_t) + : capacity_(SooCapacity()), size_(size_t{1} << HasInfozShift()) {} // Not copyable CommonFields(const CommonFields&) = delete; @@ -1035,23 +1326,44 @@ class CommonFields : public CommonFieldsGenerationInfo { CommonFields(CommonFields&& that) = default; CommonFields& operator=(CommonFields&&) = default; - ctrl_t* control() const { return control_; } - void set_control(ctrl_t* c) { control_ = c; } + template <bool kSooEnabled> + static CommonFields CreateDefault() { + return kSooEnabled ? CommonFields{soo_tag_t{}} : CommonFields{}; + } + + // The inline data for SOO is written on top of control_/slots_. + const void* soo_data() const { return heap_or_soo_.get_soo_data(); } + void* soo_data() { return heap_or_soo_.get_soo_data(); } + + HeapOrSoo heap_or_soo() const { return heap_or_soo_; } + const HeapOrSoo& heap_or_soo_ref() const { return heap_or_soo_; } + + ctrl_t* control() const { return heap_or_soo_.control(); } + void set_control(ctrl_t* c) { heap_or_soo_.control() = c; } void* backing_array_start() const { - // growth_left (and maybe infoz) is stored before control bytes. + // growth_info (and maybe infoz) is stored before control bytes. assert(reinterpret_cast<uintptr_t>(control()) % alignof(size_t) == 0); return control() - ControlOffset(has_infoz()); } // Note: we can't use slots() because Qt defines "slots" as a macro. - void* slot_array() const { return slots_; } - void set_slots(void* s) { slots_ = s; } + void* slot_array() const { return heap_or_soo_.slot_array().get(); } + MaybeInitializedPtr slots_union() const { return heap_or_soo_.slot_array(); } + void set_slots(void* s) { heap_or_soo_.slot_array().set(s); } // The number of filled slots. size_t size() const { return size_ >> HasInfozShift(); } void set_size(size_t s) { size_ = (s << HasInfozShift()) | (size_ & HasInfozMask()); } + void set_empty_soo() { + AssertInSooMode(); + size_ = 0; + } + void set_full_soo() { + AssertInSooMode(); + size_ = size_t{1} << HasInfozShift(); + } void increment_size() { assert(size() < capacity()); size_ += size_t{1} << HasInfozShift(); @@ -1070,15 +1382,17 @@ class CommonFields : public CommonFieldsGenerationInfo { // The number of slots we can still fill without needing to rehash. // This is stored in the heap allocation before the control bytes. - size_t growth_left() const { - const size_t* gl_ptr = reinterpret_cast<size_t*>(control()) - 1; - assert(reinterpret_cast<uintptr_t>(gl_ptr) % alignof(size_t) == 0); + // TODO(b/289225379): experiment with moving growth_info back inline to + // increase room for SOO. + size_t growth_left() const { return growth_info().GetGrowthLeft(); } + + GrowthInfo& growth_info() { + auto* gl_ptr = reinterpret_cast<GrowthInfo*>(control()) - 1; + assert(reinterpret_cast<uintptr_t>(gl_ptr) % alignof(GrowthInfo) == 0); return *gl_ptr; } - void set_growth_left(size_t gl) { - size_t* gl_ptr = reinterpret_cast<size_t*>(control()) - 1; - assert(reinterpret_cast<uintptr_t>(gl_ptr) % alignof(size_t) == 0); - *gl_ptr = gl; + GrowthInfo growth_info() const { + return const_cast<CommonFields*>(this)->growth_info(); } bool has_infoz() const { @@ -1103,12 +1417,8 @@ class CommonFields : public CommonFieldsGenerationInfo { should_rehash_for_bug_detection_on_insert(control(), capacity()); } bool should_rehash_for_bug_detection_on_move() const { - return CommonFieldsGenerationInfo:: - should_rehash_for_bug_detection_on_move(control(), capacity()); - } - void maybe_increment_generation_on_move() { - if (capacity() == 0) return; - increment_generation(); + return CommonFieldsGenerationInfo::should_rehash_for_bug_detection_on_move( + control(), capacity()); } void reset_reserved_growth(size_t reservation) { CommonFieldsGenerationInfo::reset_reserved_growth(reservation, size()); @@ -1116,7 +1426,16 @@ class CommonFields : public CommonFieldsGenerationInfo { // The size of the backing array allocation. size_t alloc_size(size_t slot_size, size_t slot_align) const { - return AllocSize(capacity(), slot_size, slot_align, has_infoz()); + return RawHashSetLayout(capacity(), slot_align, has_infoz()) + .alloc_size(slot_size); + } + + // Move fields other than heap_or_soo_. + void move_non_heap_or_soo_fields(CommonFields& that) { + static_cast<CommonFieldsGenerationInfo&>(*this) = + std::move(static_cast<CommonFieldsGenerationInfo&>(that)); + capacity_ = that.capacity_; + size_ = that.size_; } // Returns the number of control bytes set to kDeleted. For testing only. @@ -1132,21 +1451,12 @@ class CommonFields : public CommonFieldsGenerationInfo { return (size_t{1} << HasInfozShift()) - 1; } - // TODO(b/182800944): Investigate removing some of these fields: - // - control/slots can be derived from each other - - // The control bytes (and, also, a pointer near to the base of the backing - // array). - // - // This contains `capacity + 1 + NumClonedBytes()` entries, even - // when the table is empty (hence EmptyGroup). - // - // Note that growth_left is stored immediately before this pointer. - ctrl_t* control_ = EmptyGroup(); - - // The beginning of the slots, located at `SlotOffset()` bytes after - // `control`. May be null for empty tables. - void* slots_ = nullptr; + // We can't assert that SOO is enabled because we don't have SooEnabled(), but + // we assert what we can. + void AssertInSooMode() const { + assert(capacity() == SooCapacity()); + assert(!has_infoz()); + } // The number of slots in the backing array. This is always 2^N-1 for an // integer N. NOTE: we tried experimenting with compressing the capacity and @@ -1154,10 +1464,16 @@ class CommonFields : public CommonFieldsGenerationInfo { // power (N in 2^N-1), and (b) storing 2^N as the most significant bit of // size_ and storing size in the low bits. Both of these experiments were // regressions, presumably because we need capacity to do find operations. - size_t capacity_ = 0; + size_t capacity_; // The size and also has one bit that stores whether we have infoz. - size_t size_ = 0; + // TODO(b/289225379): we could put size_ into HeapOrSoo and make capacity_ + // encode the size in SOO case. We would be making size()/capacity() more + // expensive in order to have more SOO space. + size_t size_; + + // Either the control/slots pointers or the SOO slot. + HeapOrSoo heap_or_soo_; }; template <class Policy, class Hash, class Eq, class Alloc> @@ -1320,6 +1636,10 @@ inline bool AreItersFromSameContainer(const ctrl_t* ctrl_a, const void* const& slot_b) { // If either control byte is null, then we can't tell. if (ctrl_a == nullptr || ctrl_b == nullptr) return true; + const bool a_is_soo = IsSooControl(ctrl_a); + if (a_is_soo != IsSooControl(ctrl_b)) return false; + if (a_is_soo) return slot_a == slot_b; + const void* low_slot = slot_a; const void* hi_slot = slot_b; if (ctrl_a > ctrl_b) { @@ -1343,41 +1663,45 @@ inline void AssertSameContainer(const ctrl_t* ctrl_a, const ctrl_t* ctrl_b, // - use `ABSL_PREDICT_FALSE()` to provide a compiler hint for code layout // - use `ABSL_RAW_LOG()` with a format string to reduce code size and improve // the chances that the hot paths will be inlined. + + // fail_if(is_invalid, message) crashes when is_invalid is true and provides + // an error message based on `message`. + const auto fail_if = [](bool is_invalid, const char* message) { + if (ABSL_PREDICT_FALSE(is_invalid)) { + ABSL_RAW_LOG(FATAL, "Invalid iterator comparison. %s", message); + } + }; + const bool a_is_default = ctrl_a == EmptyGroup(); const bool b_is_default = ctrl_b == EmptyGroup(); - if (ABSL_PREDICT_FALSE(a_is_default != b_is_default)) { - ABSL_RAW_LOG( - FATAL, - "Invalid iterator comparison. Comparing default-constructed iterator " - "with non-default-constructed iterator."); - } if (a_is_default && b_is_default) return; + fail_if(a_is_default != b_is_default, + "Comparing default-constructed hashtable iterator with a " + "non-default-constructed hashtable iterator."); if (SwisstableGenerationsEnabled()) { if (ABSL_PREDICT_TRUE(generation_ptr_a == generation_ptr_b)) return; + // Users don't need to know whether the tables are SOO so don't mention SOO + // in the debug message. + const bool a_is_soo = IsSooControl(ctrl_a); + const bool b_is_soo = IsSooControl(ctrl_b); + fail_if(a_is_soo != b_is_soo || (a_is_soo && b_is_soo), + "Comparing iterators from different hashtables."); + const bool a_is_empty = IsEmptyGeneration(generation_ptr_a); const bool b_is_empty = IsEmptyGeneration(generation_ptr_b); - if (a_is_empty != b_is_empty) { - ABSL_RAW_LOG(FATAL, - "Invalid iterator comparison. Comparing iterator from a " - "non-empty hashtable with an iterator from an empty " - "hashtable."); - } - if (a_is_empty && b_is_empty) { - ABSL_RAW_LOG(FATAL, - "Invalid iterator comparison. Comparing iterators from " - "different empty hashtables."); - } + fail_if(a_is_empty != b_is_empty, + "Comparing an iterator from an empty hashtable with an iterator " + "from a non-empty hashtable."); + fail_if(a_is_empty && b_is_empty, + "Comparing iterators from different empty hashtables."); + const bool a_is_end = ctrl_a == nullptr; const bool b_is_end = ctrl_b == nullptr; - if (a_is_end || b_is_end) { - ABSL_RAW_LOG(FATAL, - "Invalid iterator comparison. Comparing iterator with an " - "end() iterator from a different hashtable."); - } - ABSL_RAW_LOG(FATAL, - "Invalid iterator comparison. Comparing non-end() iterators " - "from different hashtables."); + fail_if(a_is_end || b_is_end, + "Comparing iterator with an end() iterator from a different " + "hashtable."); + fail_if(true, "Comparing non-end() iterators from different hashtables."); } else { ABSL_HARDENING_ASSERT( AreItersFromSameContainer(ctrl_a, ctrl_b, slot_a, slot_b) && @@ -1432,20 +1756,17 @@ template <typename = void> inline FindInfo find_first_non_full(const CommonFields& common, size_t hash) { auto seq = probe(common, hash); const ctrl_t* ctrl = common.control(); + if (IsEmptyOrDeleted(ctrl[seq.offset()]) && + !ShouldInsertBackwards(common.capacity(), hash, ctrl)) { + return {seq.offset(), /*probe_length=*/0}; + } while (true) { - GroupEmptyOrDeleted g{ctrl + seq.offset()}; + GroupFullEmptyOrDeleted g{ctrl + seq.offset()}; auto mask = g.MaskEmptyOrDeleted(); if (mask) { -#if !defined(NDEBUG) - // We want to add entropy even when ASLR is not enabled. - // In debug build we will randomly insert in either the front or back of - // the group. - // TODO(kfm,sbenza): revisit after we do unconditional mixing - if (!is_small(common.capacity()) && ShouldInsertBackwards(hash, ctrl)) { - return {seq.offset(mask.HighestBitSet()), seq.index()}; - } -#endif - return {seq.offset(mask.LowestBitSet()), seq.index()}; + return { + seq.offset(GetInsertionOffset(mask, common.capacity(), hash, ctrl)), + seq.index()}; } seq.next(); assert(seq.index() <= common.capacity() && "full table!"); @@ -1462,7 +1783,8 @@ extern template FindInfo find_first_non_full(const CommonFields&, size_t); FindInfo find_first_non_full_outofline(const CommonFields&, size_t); inline void ResetGrowthLeft(CommonFields& common) { - common.set_growth_left(CapacityToGrowth(common.capacity()) - common.size()); + common.growth_info().InitGrowthLeftNoDeleted( + CapacityToGrowth(common.capacity()) - common.size()); } // Sets `ctrl` to `{kEmpty, kSentinel, ..., kEmpty}`, marking the entire @@ -1476,43 +1798,140 @@ inline void ResetCtrl(CommonFields& common, size_t slot_size) { SanitizerPoisonMemoryRegion(common.slot_array(), slot_size * capacity); } -// Sets `ctrl[i]` to `h`. -// -// Unlike setting it directly, this function will perform bounds checks and -// mirror the value to the cloned tail if necessary. -inline void SetCtrl(const CommonFields& common, size_t i, ctrl_t h, - size_t slot_size) { - const size_t capacity = common.capacity(); - assert(i < capacity); - - auto* slot_i = static_cast<const char*>(common.slot_array()) + i * slot_size; +// Sets sanitizer poisoning for slot corresponding to control byte being set. +inline void DoSanitizeOnSetCtrl(const CommonFields& c, size_t i, ctrl_t h, + size_t slot_size) { + assert(i < c.capacity()); + auto* slot_i = static_cast<const char*>(c.slot_array()) + i * slot_size; if (IsFull(h)) { SanitizerUnpoisonMemoryRegion(slot_i, slot_size); } else { SanitizerPoisonMemoryRegion(slot_i, slot_size); } +} - ctrl_t* ctrl = common.control(); +// Sets `ctrl[i]` to `h`. +// +// Unlike setting it directly, this function will perform bounds checks and +// mirror the value to the cloned tail if necessary. +inline void SetCtrl(const CommonFields& c, size_t i, ctrl_t h, + size_t slot_size) { + DoSanitizeOnSetCtrl(c, i, h, slot_size); + ctrl_t* ctrl = c.control(); ctrl[i] = h; - ctrl[((i - NumClonedBytes()) & capacity) + (NumClonedBytes() & capacity)] = h; + ctrl[((i - NumClonedBytes()) & c.capacity()) + + (NumClonedBytes() & c.capacity())] = h; +} +// Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`. +inline void SetCtrl(const CommonFields& c, size_t i, h2_t h, size_t slot_size) { + SetCtrl(c, i, static_cast<ctrl_t>(h), slot_size); } +// Like SetCtrl, but in a single group table, we can save some operations when +// setting the cloned control byte. +inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, ctrl_t h, + size_t slot_size) { + assert(is_single_group(c.capacity())); + DoSanitizeOnSetCtrl(c, i, h, slot_size); + ctrl_t* ctrl = c.control(); + ctrl[i] = h; + ctrl[i + c.capacity() + 1] = h; +} // Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`. -inline void SetCtrl(const CommonFields& common, size_t i, h2_t h, - size_t slot_size) { - SetCtrl(common, i, static_cast<ctrl_t>(h), slot_size); +inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, h2_t h, + size_t slot_size) { + SetCtrlInSingleGroupTable(c, i, static_cast<ctrl_t>(h), slot_size); } -// growth_left (which is a size_t) is stored with the backing array. +// growth_info (which is a size_t) is stored with the backing array. constexpr size_t BackingArrayAlignment(size_t align_of_slot) { - return (std::max)(align_of_slot, alignof(size_t)); + return (std::max)(align_of_slot, alignof(GrowthInfo)); } // Returns the address of the ith slot in slots where each slot occupies // slot_size. inline void* SlotAddress(void* slot_array, size_t slot, size_t slot_size) { - return reinterpret_cast<void*>(reinterpret_cast<char*>(slot_array) + - (slot * slot_size)); + return static_cast<void*>(static_cast<char*>(slot_array) + + (slot * slot_size)); +} + +// Iterates over all full slots and calls `cb(const ctrl_t*, SlotType*)`. +// No insertion to the table allowed during Callback call. +// Erasure is allowed only for the element passed to the callback. +template <class SlotType, class Callback> +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots( + const CommonFields& c, SlotType* slot, Callback cb) { + const size_t cap = c.capacity(); + const ctrl_t* ctrl = c.control(); + if (is_small(cap)) { + // Mirrored/cloned control bytes in small table are also located in the + // first group (starting from position 0). We are taking group from position + // `capacity` in order to avoid duplicates. + + // Small tables capacity fits into portable group, where + // GroupPortableImpl::MaskFull is more efficient for the + // capacity <= GroupPortableImpl::kWidth. + assert(cap <= GroupPortableImpl::kWidth && + "unexpectedly large small capacity"); + static_assert(Group::kWidth >= GroupPortableImpl::kWidth, + "unexpected group width"); + // Group starts from kSentinel slot, so indices in the mask will + // be increased by 1. + const auto mask = GroupPortableImpl(ctrl + cap).MaskFull(); + --ctrl; + --slot; + for (uint32_t i : mask) { + cb(ctrl + i, slot + i); + } + return; + } + size_t remaining = c.size(); + ABSL_ATTRIBUTE_UNUSED const size_t original_size_for_assert = remaining; + while (remaining != 0) { + for (uint32_t i : GroupFullEmptyOrDeleted(ctrl).MaskFull()) { + assert(IsFull(ctrl[i]) && "hash table was modified unexpectedly"); + cb(ctrl + i, slot + i); + --remaining; + } + ctrl += Group::kWidth; + slot += Group::kWidth; + assert((remaining == 0 || *(ctrl - 1) != ctrl_t::kSentinel) && + "hash table was modified unexpectedly"); + } + // NOTE: erasure of the current element is allowed in callback for + // absl::erase_if specialization. So we use `>=`. + assert(original_size_for_assert >= c.size() && + "hash table was modified unexpectedly"); +} + +template <typename CharAlloc> +constexpr bool ShouldSampleHashtablezInfo() { + // Folks with custom allocators often make unwarranted assumptions about the + // behavior of their classes vis-a-vis trivial destructability and what + // calls they will or won't make. Avoid sampling for people with custom + // allocators to get us out of this mess. This is not a hard guarantee but + // a workaround while we plan the exact guarantee we want to provide. + return std::is_same<CharAlloc, std::allocator<char>>::value; +} + +template <bool kSooEnabled> +HashtablezInfoHandle SampleHashtablezInfo(size_t sizeof_slot, size_t sizeof_key, + size_t sizeof_value, + size_t old_capacity, bool was_soo, + HashtablezInfoHandle forced_infoz, + CommonFields& c) { + if (forced_infoz.IsSampled()) return forced_infoz; + // In SOO, we sample on the first insertion so if this is an empty SOO case + // (e.g. when reserve is called), then we still need to sample. + if (kSooEnabled && was_soo && c.size() == 0) { + return Sample(sizeof_slot, sizeof_key, sizeof_value, SooCapacity()); + } + // For non-SOO cases, we sample whenever the capacity is increasing from zero + // to non-zero. + if (!kSooEnabled && old_capacity == 0) { + return Sample(sizeof_slot, sizeof_key, sizeof_value, 0); + } + return c.infoz(); } // Helper class to perform resize of the hash set. @@ -1521,17 +1940,21 @@ inline void* SlotAddress(void* slot_array, size_t slot, size_t slot_size) { // See GrowIntoSingleGroupShuffleControlBytes for details. class HashSetResizeHelper { public: - explicit HashSetResizeHelper(CommonFields& c) - : old_ctrl_(c.control()), - old_capacity_(c.capacity()), - had_infoz_(c.has_infoz()) {} - - // Optimized for small groups version of `find_first_non_full` applicable - // only right after calling `raw_hash_set::resize`. + explicit HashSetResizeHelper(CommonFields& c, bool was_soo, bool had_soo_slot, + HashtablezInfoHandle forced_infoz) + : old_capacity_(c.capacity()), + had_infoz_(c.has_infoz()), + was_soo_(was_soo), + had_soo_slot_(had_soo_slot), + forced_infoz_(forced_infoz) {} + + // Optimized for small groups version of `find_first_non_full`. + // Beneficial only right after calling `raw_hash_set::resize`. + // It is safe to call in case capacity is big or was not changed, but there + // will be no performance benefit. // It has implicit assumption that `resize` will call // `GrowSizeIntoSingleGroup*` in case `IsGrowingIntoSingleGroupApplicable`. - // Falls back to `find_first_non_full` in case of big groups, so it is - // safe to use after `rehash_and_grow_if_necessary`. + // Falls back to `find_first_non_full` in case of big groups. static FindInfo FindFirstNonFullAfterResize(const CommonFields& c, size_t old_capacity, size_t hash) { @@ -1553,14 +1976,30 @@ class HashSetResizeHelper { return FindInfo{offset, 0}; } - ctrl_t* old_ctrl() const { return old_ctrl_; } + HeapOrSoo& old_heap_or_soo() { return old_heap_or_soo_; } + void* old_soo_data() { return old_heap_or_soo_.get_soo_data(); } + ctrl_t* old_ctrl() const { + assert(!was_soo_); + return old_heap_or_soo_.control(); + } + void* old_slots() const { + assert(!was_soo_); + return old_heap_or_soo_.slot_array().get(); + } size_t old_capacity() const { return old_capacity_; } + // Returns the index of the SOO slot when growing from SOO to non-SOO in a + // single group. See also InitControlBytesAfterSoo(). It's important to use + // index 1 so that when resizing from capacity 1 to 3, we can still have + // random iteration order between the first two inserted elements. + // I.e. it allows inserting the second element at either index 0 or 2. + static size_t SooSlotIndex() { return 1; } + // Allocates a backing array for the hashtable. // Reads `capacity` and updates all other fields based on the result of // the allocation. // - // It also may do the folowing actions: + // It also may do the following actions: // 1. initialize control bytes // 2. initialize slots // 3. deallocate old slots. @@ -1590,45 +2029,45 @@ class HashSetResizeHelper { // // Returns IsGrowingIntoSingleGroupApplicable result to avoid recomputation. template <typename Alloc, size_t SizeOfSlot, bool TransferUsesMemcpy, - size_t AlignOfSlot> - ABSL_ATTRIBUTE_NOINLINE bool InitializeSlots(CommonFields& c, void* old_slots, - Alloc alloc) { + bool SooEnabled, size_t AlignOfSlot> + ABSL_ATTRIBUTE_NOINLINE bool InitializeSlots(CommonFields& c, Alloc alloc, + ctrl_t soo_slot_h2, + size_t key_size, + size_t value_size) { assert(c.capacity()); - // Folks with custom allocators often make unwarranted assumptions about the - // behavior of their classes vis-a-vis trivial destructability and what - // calls they will or won't make. Avoid sampling for people with custom - // allocators to get us out of this mess. This is not a hard guarantee but - // a workaround while we plan the exact guarantee we want to provide. - const size_t sample_size = - (std::is_same<Alloc, std::allocator<char>>::value && - c.slot_array() == nullptr) - ? SizeOfSlot - : 0; HashtablezInfoHandle infoz = - sample_size > 0 ? Sample(sample_size) : c.infoz(); + ShouldSampleHashtablezInfo<Alloc>() + ? SampleHashtablezInfo<SooEnabled>(SizeOfSlot, key_size, value_size, + old_capacity_, was_soo_, + forced_infoz_, c) + : HashtablezInfoHandle{}; const bool has_infoz = infoz.IsSampled(); - const size_t cap = c.capacity(); - const size_t alloc_size = - AllocSize(cap, SizeOfSlot, AlignOfSlot, has_infoz); - char* mem = static_cast<char*>( - Allocate<BackingArrayAlignment(AlignOfSlot)>(&alloc, alloc_size)); + RawHashSetLayout layout(c.capacity(), AlignOfSlot, has_infoz); + char* mem = static_cast<char*>(Allocate<BackingArrayAlignment(AlignOfSlot)>( + &alloc, layout.alloc_size(SizeOfSlot))); const GenerationType old_generation = c.generation(); - c.set_generation_ptr(reinterpret_cast<GenerationType*>( - mem + GenerationOffset(cap, has_infoz))); + c.set_generation_ptr( + reinterpret_cast<GenerationType*>(mem + layout.generation_offset())); c.set_generation(NextGeneration(old_generation)); - c.set_control(reinterpret_cast<ctrl_t*>(mem + ControlOffset(has_infoz))); - c.set_slots(mem + SlotOffset(cap, AlignOfSlot, has_infoz)); + c.set_control(reinterpret_cast<ctrl_t*>(mem + layout.control_offset())); + c.set_slots(mem + layout.slot_offset()); ResetGrowthLeft(c); const bool grow_single_group = - IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity()); - if (old_capacity_ != 0 && grow_single_group) { + IsGrowingIntoSingleGroupApplicable(old_capacity_, layout.capacity()); + if (SooEnabled && was_soo_ && grow_single_group) { + InitControlBytesAfterSoo(c.control(), soo_slot_h2, layout.capacity()); + if (TransferUsesMemcpy && had_soo_slot_) { + TransferSlotAfterSoo(c, SizeOfSlot); + } + // SooEnabled implies that old_capacity_ != 0. + } else if ((SooEnabled || old_capacity_ != 0) && grow_single_group) { if (TransferUsesMemcpy) { - GrowSizeIntoSingleGroupTransferable(c, old_slots, SizeOfSlot); - DeallocateOld<AlignOfSlot>(alloc, SizeOfSlot, old_slots); + GrowSizeIntoSingleGroupTransferable(c, SizeOfSlot); + DeallocateOld<AlignOfSlot>(alloc, SizeOfSlot); } else { - GrowIntoSingleGroupShuffleControlBytes(c.control(), c.capacity()); + GrowIntoSingleGroupShuffleControlBytes(c.control(), layout.capacity()); } } else { ResetCtrl(c, SizeOfSlot); @@ -1636,8 +2075,8 @@ class HashSetResizeHelper { c.set_has_infoz(has_infoz); if (has_infoz) { - infoz.RecordStorageChanged(c.size(), cap); - if (grow_single_group || old_capacity_ == 0) { + infoz.RecordStorageChanged(c.size(), layout.capacity()); + if ((SooEnabled && was_soo_) || grow_single_group || old_capacity_ == 0) { infoz.RecordRehash(0); } c.set_infoz(infoz); @@ -1651,21 +2090,22 @@ class HashSetResizeHelper { // PRECONDITIONS: // 1. GrowIntoSingleGroupShuffleControlBytes was already called. template <class PolicyTraits, class Alloc> - void GrowSizeIntoSingleGroup(CommonFields& c, Alloc& alloc_ref, - typename PolicyTraits::slot_type* old_slots) { + void GrowSizeIntoSingleGroup(CommonFields& c, Alloc& alloc_ref) { assert(old_capacity_ < Group::kWidth / 2); assert(IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity())); using slot_type = typename PolicyTraits::slot_type; assert(is_single_group(c.capacity())); - auto* new_slots = reinterpret_cast<slot_type*>(c.slot_array()); + auto* new_slots = static_cast<slot_type*>(c.slot_array()); + auto* old_slots_ptr = static_cast<slot_type*>(old_slots()); size_t shuffle_bit = old_capacity_ / 2 + 1; for (size_t i = 0; i < old_capacity_; ++i) { - if (IsFull(old_ctrl_[i])) { + if (IsFull(old_ctrl()[i])) { size_t new_i = i ^ shuffle_bit; SanitizerUnpoisonMemoryRegion(new_slots + new_i, sizeof(slot_type)); - PolicyTraits::transfer(&alloc_ref, new_slots + new_i, old_slots + i); + PolicyTraits::transfer(&alloc_ref, new_slots + new_i, + old_slots_ptr + i); } } PoisonSingleGroupEmptySlots(c, sizeof(slot_type)); @@ -1673,11 +2113,12 @@ class HashSetResizeHelper { // Deallocates old backing array. template <size_t AlignOfSlot, class CharAlloc> - void DeallocateOld(CharAlloc alloc_ref, size_t slot_size, void* old_slots) { - SanitizerUnpoisonMemoryRegion(old_slots, slot_size * old_capacity_); + void DeallocateOld(CharAlloc alloc_ref, size_t slot_size) { + SanitizerUnpoisonMemoryRegion(old_slots(), slot_size * old_capacity_); + auto layout = RawHashSetLayout(old_capacity_, AlignOfSlot, had_infoz_); Deallocate<BackingArrayAlignment(AlignOfSlot)>( - &alloc_ref, old_ctrl_ - ControlOffset(had_infoz_), - AllocSize(old_capacity_, slot_size, AlignOfSlot, had_infoz_)); + &alloc_ref, old_ctrl() - layout.control_offset(), + layout.alloc_size(slot_size)); } private: @@ -1692,8 +2133,12 @@ class HashSetResizeHelper { // Relocates control bytes and slots into new single group for // transferable objects. // Must be called only if IsGrowingIntoSingleGroupApplicable returned true. - void GrowSizeIntoSingleGroupTransferable(CommonFields& c, void* old_slots, - size_t slot_size); + void GrowSizeIntoSingleGroupTransferable(CommonFields& c, size_t slot_size); + + // If there was an SOO slot and slots are transferable, transfers the SOO slot + // into the new heap allocation. Must be called only if + // IsGrowingIntoSingleGroupApplicable returned true. + void TransferSlotAfterSoo(CommonFields& c, size_t slot_size); // Shuffle control bits deterministically to the next capacity. // Returns offset for newly added element with given hash. @@ -1726,6 +2171,13 @@ class HashSetResizeHelper { void GrowIntoSingleGroupShuffleControlBytes(ctrl_t* new_ctrl, size_t new_capacity) const; + // If the table was SOO, initializes new control bytes. `h2` is the control + // byte corresponding to the full slot. Must be called only if + // IsGrowingIntoSingleGroupApplicable returned true. + // Requires: `had_soo_slot_ || h2 == ctrl_t::kEmpty`. + void InitControlBytesAfterSoo(ctrl_t* new_ctrl, ctrl_t h2, + size_t new_capacity); + // Shuffle trivially transferable slots in the way consistent with // GrowIntoSingleGroupShuffleControlBytes. // @@ -1739,8 +2191,7 @@ class HashSetResizeHelper { // 1. new_slots are transferred from old_slots_ consistent with // GrowIntoSingleGroupShuffleControlBytes. // 2. Empty new_slots are *not* poisoned. - void GrowIntoSingleGroupShuffleTransferableSlots(void* old_slots, - void* new_slots, + void GrowIntoSingleGroupShuffleTransferableSlots(void* new_slots, size_t slot_size) const; // Poison empty slots that were transferred using the deterministic algorithm @@ -1760,11 +2211,24 @@ class HashSetResizeHelper { } } - ctrl_t* old_ctrl_; + HeapOrSoo old_heap_or_soo_; size_t old_capacity_; bool had_infoz_; + bool was_soo_; + bool had_soo_slot_; + // Either null infoz or a pre-sampled forced infoz for SOO tables. + HashtablezInfoHandle forced_infoz_; }; +inline void PrepareInsertCommon(CommonFields& common) { + common.increment_size(); + common.maybe_increment_generation_on_insert(); +} + +// Like prepare_insert, but for the case of inserting into a full SOO table. +size_t PrepareInsertAfterSoo(size_t hash, size_t slot_size, + CommonFields& common); + // PolicyFunctions bundles together some information for a particular // raw_hash_set<T, ...> instantiation. This information is passed to // type-erased functions that want to do small amounts of type-specific @@ -1772,21 +2236,29 @@ class HashSetResizeHelper { struct PolicyFunctions { size_t slot_size; + // Returns the pointer to the hash function stored in the set. + const void* (*hash_fn)(const CommonFields& common); + // Returns the hash of the pointed-to slot. - size_t (*hash_slot)(void* set, void* slot); + size_t (*hash_slot)(const void* hash_fn, void* slot); - // Transfer the contents of src_slot to dst_slot. + // Transfers the contents of src_slot to dst_slot. void (*transfer)(void* set, void* dst_slot, void* src_slot); - // Deallocate the backing store from common. + // Deallocates the backing store from common. void (*dealloc)(CommonFields& common, const PolicyFunctions& policy); + + // Resizes set to the new capacity. + // Arguments are used as in raw_hash_set::resize_impl. + void (*resize)(CommonFields& common, size_t new_capacity, + HashtablezInfoHandle forced_infoz); }; // ClearBackingArray clears the backing array, either modifying it in place, // or creating a new one based on the value of "reuse". // REQUIRES: c.capacity > 0 void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy, - bool reuse); + bool reuse, bool soo_enabled); // Type-erased version of raw_hash_set::erase_meta_only. void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size); @@ -1817,9 +2289,26 @@ ABSL_ATTRIBUTE_NOINLINE void TransferRelocatable(void*, void* dst, void* src) { memcpy(dst, src, SizeOfSlot); } -// Type-erased version of raw_hash_set::drop_deletes_without_resize. -void DropDeletesWithoutResize(CommonFields& common, - const PolicyFunctions& policy, void* tmp_space); +// Type erased raw_hash_set::get_hash_ref_fn for the empty hash function case. +const void* GetHashRefForEmptyHasher(const CommonFields& common); + +// Given the hash of a value not currently in the table and the first empty +// slot in the probe sequence, finds a viable slot index to insert it at. +// +// In case there's no space left, the table can be resized or rehashed +// (for tables with deleted slots, see FindInsertPositionWithGrowthOrRehash). +// +// In the case of absence of deleted slots and positive growth_left, the element +// can be inserted in the provided `target` position. +// +// When the table has deleted slots (according to GrowthInfo), the target +// position will be searched one more time using `find_first_non_full`. +// +// REQUIRES: Table is not SOO. +// REQUIRES: At least one non-full slot available. +// REQUIRES: `target` is a valid empty position to insert. +size_t PrepareInsertNonSoo(CommonFields& common, size_t hash, FindInfo target, + const PolicyFunctions& policy); // A SwissTable. // @@ -1875,6 +2364,26 @@ class raw_hash_set { using key_arg = typename KeyArgImpl::template type<K, key_type>; private: + // TODO(b/289225379): we could add extra SOO space inside raw_hash_set + // after CommonFields to allow inlining larger slot_types (e.g. std::string), + // but it's a bit complicated if we want to support incomplete mapped_type in + // flat_hash_map. We could potentially do this for flat_hash_set and for an + // allowlist of `mapped_type`s of flat_hash_map that includes e.g. arithmetic + // types, strings, cords, and pairs/tuples of allowlisted types. + constexpr static bool SooEnabled() { + return PolicyTraits::soo_enabled() && + sizeof(slot_type) <= sizeof(HeapOrSoo) && + alignof(slot_type) <= alignof(HeapOrSoo); + } + + // Whether `size` fits in the SOO capacity of this table. + bool fits_in_soo(size_t size) const { + return SooEnabled() && size <= SooCapacity(); + } + // Whether this table is in SOO mode or non-SOO mode. + bool is_soo() const { return fits_in_soo(capacity()); } + bool is_full_soo() const { return is_soo() && !empty(); } + // Give an early error when key_type is not hashable/eq. auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); @@ -1928,6 +2437,7 @@ class raw_hash_set { class iterator : private HashSetIteratorGenerationInfo { friend class raw_hash_set; + friend struct HashtableFreeFunctionsAccess; public: using iterator_category = std::forward_iterator_tag; @@ -1958,6 +2468,7 @@ class raw_hash_set { ++ctrl_; ++slot_; skip_empty_or_deleted(); + if (ABSL_PREDICT_FALSE(*ctrl_ == ctrl_t::kSentinel)) ctrl_ = nullptr; return *this; } // PRECONDITION: not an end() iterator. @@ -1988,22 +2499,31 @@ class raw_hash_set { // not equal to any end iterator. ABSL_ASSUME(ctrl != nullptr); } + // This constructor is used in begin() to avoid an MSan + // use-of-uninitialized-value error. Delegating from this constructor to + // the previous one doesn't avoid the error. + iterator(ctrl_t* ctrl, MaybeInitializedPtr slot, + const GenerationType* generation_ptr) + : HashSetIteratorGenerationInfo(generation_ptr), + ctrl_(ctrl), + slot_(to_slot(slot.get())) { + // This assumption helps the compiler know that any non-end iterator is + // not equal to any end iterator. + ABSL_ASSUME(ctrl != nullptr); + } // For end() iterators. explicit iterator(const GenerationType* generation_ptr) : HashSetIteratorGenerationInfo(generation_ptr), ctrl_(nullptr) {} - // Fixes up `ctrl_` to point to a full by advancing it and `slot_` until - // they reach one. - // - // If a sentinel is reached, we null `ctrl_` out instead. + // Fixes up `ctrl_` to point to a full or sentinel by advancing `ctrl_` and + // `slot_` until they reach one. void skip_empty_or_deleted() { while (IsEmptyOrDeleted(*ctrl_)) { uint32_t shift = - GroupEmptyOrDeleted{ctrl_}.CountLeadingEmptyOrDeleted(); + GroupFullEmptyOrDeleted{ctrl_}.CountLeadingEmptyOrDeleted(); ctrl_ += shift; slot_ += shift; } - if (ABSL_PREDICT_FALSE(*ctrl_ == ctrl_t::kSentinel)) ctrl_ = nullptr; } ctrl_t* control() const { return ctrl_; } @@ -2091,8 +2611,9 @@ class raw_hash_set { size_t bucket_count, const hasher& hash = hasher(), const key_equal& eq = key_equal(), const allocator_type& alloc = allocator_type()) - : settings_(CommonFields{}, hash, eq, alloc) { - if (bucket_count) { + : settings_(CommonFields::CreateDefault<SooEnabled()>(), hash, eq, + alloc) { + if (bucket_count > (SooEnabled() ? SooCapacity() : 0)) { resize(NormalizeCapacity(bucket_count)); } } @@ -2193,22 +2714,69 @@ class raw_hash_set { that.alloc_ref())) {} raw_hash_set(const raw_hash_set& that, const allocator_type& a) - : raw_hash_set(0, that.hash_ref(), that.eq_ref(), a) { + : raw_hash_set(GrowthToLowerboundCapacity(that.size()), that.hash_ref(), + that.eq_ref(), a) { const size_t size = that.size(); - if (size == 0) return; - reserve(size); - // Because the table is guaranteed to be empty, we can do something faster - // than a full `insert`. - for (const auto& v : that) { - const size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, v); - auto target = find_first_non_full_outofline(common(), hash); - SetCtrl(common(), target.offset, H2(hash), sizeof(slot_type)); - emplace_at(target.offset, v); - common().maybe_increment_generation_on_insert(); - infoz().RecordInsert(hash, target.probe_length); + if (size == 0) { + return; + } + // We don't use `that.is_soo()` here because `that` can have non-SOO + // capacity but have a size that fits into SOO capacity. + if (fits_in_soo(size)) { + assert(size == 1); + common().set_full_soo(); + emplace_at(soo_iterator(), *that.begin()); + const HashtablezInfoHandle infoz = try_sample_soo(); + if (infoz.IsSampled()) resize_with_soo_infoz(infoz); + return; + } + assert(!that.is_soo()); + const size_t cap = capacity(); + // Note about single group tables: + // 1. It is correct to have any order of elements. + // 2. Order has to be non deterministic. + // 3. We are assigning elements with arbitrary `shift` starting from + // `capacity + shift` position. + // 4. `shift` must be coprime with `capacity + 1` in order to be able to use + // modular arithmetic to traverse all positions, instead if cycling + // through a subset of positions. Odd numbers are coprime with any + // `capacity + 1` (2^N). + size_t offset = cap; + const size_t shift = + is_single_group(cap) ? (PerTableSalt(control()) | 1) : 0; + IterateOverFullSlots( + that.common(), that.slot_array(), + [&](const ctrl_t* that_ctrl, + slot_type* that_slot) ABSL_ATTRIBUTE_ALWAYS_INLINE { + if (shift == 0) { + // Big tables case. Position must be searched via probing. + // The table is guaranteed to be empty, so we can do faster than + // a full `insert`. + const size_t hash = PolicyTraits::apply( + HashElement{hash_ref()}, PolicyTraits::element(that_slot)); + FindInfo target = find_first_non_full_outofline(common(), hash); + infoz().RecordInsert(hash, target.probe_length); + offset = target.offset; + } else { + // Small tables case. Next position is computed via shift. + offset = (offset + shift) & cap; + } + const h2_t h2 = static_cast<h2_t>(*that_ctrl); + assert( // We rely that hash is not changed for small tables. + H2(PolicyTraits::apply(HashElement{hash_ref()}, + PolicyTraits::element(that_slot))) == h2 && + "hash function value changed unexpectedly during the copy"); + SetCtrl(common(), offset, h2, sizeof(slot_type)); + emplace_at(iterator_at(offset), PolicyTraits::element(that_slot)); + common().maybe_increment_generation_on_insert(); + }); + if (shift != 0) { + // On small table copy we do not record individual inserts. + // RecordInsert requires hash, but it is unknown for small tables. + infoz().RecordStorageChanged(size, cap); } common().set_size(size); - set_growth_left(growth_left() - size); + growth_info().OverwriteManyEmptyAsFull(size); } ABSL_ATTRIBUTE_NOINLINE raw_hash_set(raw_hash_set&& that) noexcept( @@ -2220,16 +2788,22 @@ class raw_hash_set { // would create a nullptr functor that cannot be called. // TODO(b/296061262): move instead of copying hash/eq/alloc. // Note: we avoid using exchange for better generated code. - settings_(std::move(that.common()), that.hash_ref(), that.eq_ref(), - that.alloc_ref()) { - that.common() = CommonFields{}; + settings_(PolicyTraits::transfer_uses_memcpy() || !that.is_full_soo() + ? std::move(that.common()) + : CommonFields{full_soo_tag_t{}}, + that.hash_ref(), that.eq_ref(), that.alloc_ref()) { + if (!PolicyTraits::transfer_uses_memcpy() && that.is_full_soo()) { + transfer(soo_slot(), that.soo_slot()); + } + that.common() = CommonFields::CreateDefault<SooEnabled()>(); maybe_increment_generation_or_rehash_on_move(); } raw_hash_set(raw_hash_set&& that, const allocator_type& a) - : settings_(CommonFields{}, that.hash_ref(), that.eq_ref(), a) { + : settings_(CommonFields::CreateDefault<SooEnabled()>(), that.hash_ref(), + that.eq_ref(), a) { if (a == that.alloc_ref()) { - std::swap(common(), that.common()); + swap_common(that); maybe_increment_generation_or_rehash_on_move(); } else { move_elements_allocs_unequal(std::move(that)); @@ -2264,8 +2838,12 @@ class raw_hash_set { ~raw_hash_set() { destructor_impl(); } iterator begin() ABSL_ATTRIBUTE_LIFETIME_BOUND { - auto it = iterator_at(0); + if (ABSL_PREDICT_FALSE(empty())) return end(); + if (is_soo()) return soo_iterator(); + iterator it = {control(), common().slots_union(), + common().generation_ptr()}; it.skip_empty_or_deleted(); + assert(IsFull(*it.control())); return it; } iterator end() ABSL_ATTRIBUTE_LIFETIME_BOUND { @@ -2285,7 +2863,14 @@ class raw_hash_set { bool empty() const { return !size(); } size_t size() const { return common().size(); } - size_t capacity() const { return common().capacity(); } + size_t capacity() const { + const size_t cap = common().capacity(); + // Compiler complains when using functions in assume so use local variables. + ABSL_ATTRIBUTE_UNUSED static constexpr bool kEnabled = SooEnabled(); + ABSL_ATTRIBUTE_UNUSED static constexpr size_t kCapacity = SooCapacity(); + ABSL_ASSUME(!kEnabled || cap >= kCapacity); + return cap; + } size_t max_size() const { return (std::numeric_limits<size_t>::max)(); } ABSL_ATTRIBUTE_REINITIALIZES void clear() { @@ -2299,9 +2884,13 @@ class raw_hash_set { const size_t cap = capacity(); if (cap == 0) { // Already guaranteed to be empty; so nothing to do. + } else if (is_soo()) { + if (!empty()) destroy(soo_slot()); + common().set_empty_soo(); } else { destroy_slots(); - ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/cap < 128); + ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/cap < 128, + SooEnabled()); } common().set_reserved_growth(0); common().set_reservation_size(0); @@ -2432,7 +3021,7 @@ class raw_hash_set { std::pair<iterator, bool> emplace(Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND { alignas(slot_type) unsigned char raw[sizeof(slot_type)]; - slot_type* slot = reinterpret_cast<slot_type*>(&raw); + slot_type* slot = to_slot(&raw); construct(slot, std::forward<Args>(args)...); const auto& elem = PolicyTraits::element(slot); @@ -2496,11 +3085,11 @@ class raw_hash_set { F&& f) ABSL_ATTRIBUTE_LIFETIME_BOUND { auto res = find_or_prepare_insert(key); if (res.second) { - slot_type* slot = slot_array() + res.first; + slot_type* slot = res.first.slot(); std::forward<F>(f)(constructor(&alloc_ref(), &slot)); assert(!slot); } - return iterator_at(res.first); + return res.first; } // Extension API: support for heterogeneous keys. @@ -2524,7 +3113,7 @@ class raw_hash_set { // this method returns void to reduce algorithmic complexity to O(1). The // iterator is invalidated, so any increment should be done before calling // erase. In order to erase while iterating across a map, use the following - // idiom (which also works for standard containers): + // idiom (which also works for some standard containers): // // for (auto it = m.begin(), end = m.end(); it != end;) { // // `erase()` will invalidate `it`, so advance `it` first. @@ -2540,7 +3129,11 @@ class raw_hash_set { void erase(iterator it) { AssertIsFull(it.control(), it.generation(), it.generation_ptr(), "erase()"); destroy(it.slot()); - erase_meta_only(it); + if (is_soo()) { + common().set_empty_soo(); + } else { + erase_meta_only(it); + } } iterator erase(const_iterator first, @@ -2548,12 +3141,19 @@ class raw_hash_set { // We check for empty first because ClearBackingArray requires that // capacity() > 0 as a precondition. if (empty()) return end(); + if (first == last) return last.inner_; + if (is_soo()) { + destroy(soo_slot()); + common().set_empty_soo(); + return end(); + } if (first == begin() && last == end()) { // TODO(ezb): we access control bytes in destroy_slots so it could make // sense to combine destroy_slots and ClearBackingArray to avoid cache // misses when the table is large. Note that we also do this in clear(). destroy_slots(); - ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/true); + ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/true, + SooEnabled()); common().set_reserved_growth(common().reservation_size()); return end(); } @@ -2568,13 +3168,21 @@ class raw_hash_set { template <typename H, typename E> void merge(raw_hash_set<Policy, H, E, Alloc>& src) { // NOLINT assert(this != &src); + // Returns whether insertion took place. + const auto insert_slot = [this](slot_type* src_slot) { + return PolicyTraits::apply(InsertSlot<false>{*this, std::move(*src_slot)}, + PolicyTraits::element(src_slot)) + .second; + }; + + if (src.is_soo()) { + if (src.empty()) return; + if (insert_slot(src.soo_slot())) src.common().set_empty_soo(); + return; + } for (auto it = src.begin(), e = src.end(); it != e;) { auto next = std::next(it); - if (PolicyTraits::apply(InsertSlot<false>{*this, std::move(*it.slot())}, - PolicyTraits::element(it.slot())) - .second) { - src.erase_meta_only(it); - } + if (insert_slot(it.slot())) src.erase_meta_only(it); it = next; } } @@ -2588,7 +3196,11 @@ class raw_hash_set { AssertIsFull(position.control(), position.inner_.generation(), position.inner_.generation_ptr(), "extract()"); auto node = CommonAccess::Transfer<node_type>(alloc_ref(), position.slot()); - erase_meta_only(position); + if (is_soo()) { + common().set_empty_soo(); + } else { + erase_meta_only(position); + } return node; } @@ -2605,7 +3217,7 @@ class raw_hash_set { IsNoThrowSwappable<allocator_type>( typename AllocTraits::propagate_on_container_swap{})) { using std::swap; - swap(common(), that.common()); + swap_common(that); swap(hash_ref(), that.hash_ref()); swap(eq_ref(), that.eq_ref()); SwapAlloc(alloc_ref(), that.alloc_ref(), @@ -2613,17 +3225,41 @@ class raw_hash_set { } void rehash(size_t n) { - if (n == 0 && capacity() == 0) return; - if (n == 0 && size() == 0) { - ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/false); - return; + const size_t cap = capacity(); + if (n == 0) { + if (cap == 0 || is_soo()) return; + if (empty()) { + ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/false, + SooEnabled()); + return; + } + if (fits_in_soo(size())) { + // When the table is already sampled, we keep it sampled. + if (infoz().IsSampled()) { + const size_t kInitialSampledCapacity = NextCapacity(SooCapacity()); + if (capacity() > kInitialSampledCapacity) { + resize(kInitialSampledCapacity); + } + // This asserts that we didn't lose sampling coverage in `resize`. + assert(infoz().IsSampled()); + return; + } + alignas(slot_type) unsigned char slot_space[sizeof(slot_type)]; + slot_type* tmp_slot = to_slot(slot_space); + transfer(tmp_slot, begin().slot()); + ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/false, + SooEnabled()); + transfer(soo_slot(), tmp_slot); + common().set_full_soo(); + return; + } } // bitor is a faster way of doing `max` here. We will round up to the next // power-of-2-minus-1, so bitor is good enough. auto m = NormalizeCapacity(n | GrowthToLowerboundCapacity(size())); // n == 0 unconditionally rehashes as per the standard. - if (n == 0 || m > capacity()) { + if (n == 0 || m > cap) { resize(m); // This is after resize, to ensure that we have completed the allocation @@ -2633,7 +3269,9 @@ class raw_hash_set { } void reserve(size_t n) { - if (n > size() + growth_left()) { + const size_t max_size_before_growth = + is_soo() ? SooCapacity() : size() + growth_left(); + if (n > max_size_before_growth) { size_t m = GrowthToLowerboundCapacity(n); resize(NormalizeCapacity(m)); @@ -2666,6 +3304,7 @@ class raw_hash_set { // specific benchmarks indicating its importance. template <class K = key_type> void prefetch(const key_arg<K>& key) const { + if (SooEnabled() ? is_soo() : capacity() == 0) return; (void)key; // Avoid probing if we won't be able to prefetch the addresses received. #ifdef ABSL_HAVE_PREFETCH @@ -2686,26 +3325,16 @@ class raw_hash_set { template <class K = key_type> iterator find(const key_arg<K>& key, size_t hash) ABSL_ATTRIBUTE_LIFETIME_BOUND { - auto seq = probe(common(), hash); - slot_type* slot_ptr = slot_array(); - const ctrl_t* ctrl = control(); - while (true) { - Group g{ctrl + seq.offset()}; - for (uint32_t i : g.Match(H2(hash))) { - if (ABSL_PREDICT_TRUE(PolicyTraits::apply( - EqualElement<K>{key, eq_ref()}, - PolicyTraits::element(slot_ptr + seq.offset(i))))) - return iterator_at(seq.offset(i)); - } - if (ABSL_PREDICT_TRUE(g.MaskEmpty())) return end(); - seq.next(); - assert(seq.index() <= capacity() && "full table!"); - } + AssertHashEqConsistent(key); + if (is_soo()) return find_soo(key); + return find_non_soo(key, hash); } template <class K = key_type> iterator find(const key_arg<K>& key) ABSL_ATTRIBUTE_LIFETIME_BOUND { + AssertHashEqConsistent(key); + if (is_soo()) return find_soo(key); prefetch_heap_block(); - return find(key, hash_ref()(key)); + return find_non_soo(key, hash_ref()(key)); } template <class K = key_type> @@ -2716,8 +3345,7 @@ class raw_hash_set { template <class K = key_type> const_iterator find(const key_arg<K>& key) const ABSL_ATTRIBUTE_LIFETIME_BOUND { - prefetch_heap_block(); - return find(key, hash_ref()(key)); + return const_cast<raw_hash_set*>(this)->find(key); } template <class K = key_type> @@ -2791,6 +3419,8 @@ class raw_hash_set { friend struct absl::container_internal::hashtable_debug_internal:: HashtableDebugAccess; + friend struct absl::container_internal::HashtableFreeFunctionsAccess; + struct FindElement { template <class K, class... Args> const_iterator operator()(const K& key, Args&&...) const { @@ -2824,7 +3454,7 @@ class raw_hash_set { if (res.second) { s.emplace_at(res.first, std::forward<Args>(args)...); } - return {s.iterator_at(res.first), res.second}; + return res; } raw_hash_set& s; }; @@ -2835,11 +3465,11 @@ class raw_hash_set { std::pair<iterator, bool> operator()(const K& key, Args&&...) && { auto res = s.find_or_prepare_insert(key); if (res.second) { - s.transfer(s.slot_array() + res.first, &slot); + s.transfer(res.first.slot(), &slot); } else if (do_destroy) { s.destroy(&slot); } - return {s.iterator_at(res.first), res.second}; + return res; } raw_hash_set& s; // Constructed slot. Either moved into place or destroyed. @@ -2858,17 +3488,55 @@ class raw_hash_set { PolicyTraits::transfer(&alloc_ref(), to, from); } - inline void destroy_slots() { - const size_t cap = capacity(); + // TODO(b/289225379): consider having a helper class that has the impls for + // SOO functionality. + template <class K = key_type> + iterator find_soo(const key_arg<K>& key) { + assert(is_soo()); + return empty() || !PolicyTraits::apply(EqualElement<K>{key, eq_ref()}, + PolicyTraits::element(soo_slot())) + ? end() + : soo_iterator(); + } + + template <class K = key_type> + iterator find_non_soo(const key_arg<K>& key, size_t hash) { + assert(!is_soo()); + auto seq = probe(common(), hash); const ctrl_t* ctrl = control(); - slot_type* slot = slot_array(); - for (size_t i = 0; i != cap; ++i) { - if (IsFull(ctrl[i])) { - destroy(slot + i); + while (true) { + Group g{ctrl + seq.offset()}; + for (uint32_t i : g.Match(H2(hash))) { + if (ABSL_PREDICT_TRUE(PolicyTraits::apply( + EqualElement<K>{key, eq_ref()}, + PolicyTraits::element(slot_array() + seq.offset(i))))) + return iterator_at(seq.offset(i)); } + if (ABSL_PREDICT_TRUE(g.MaskEmpty())) return end(); + seq.next(); + assert(seq.index() <= capacity() && "full table!"); } } + // Conditionally samples hashtablez for SOO tables. This should be called on + // insertion into an empty SOO table and in copy construction when the size + // can fit in SOO capacity. + inline HashtablezInfoHandle try_sample_soo() { + assert(is_soo()); + if (!ShouldSampleHashtablezInfo<CharAlloc>()) return HashtablezInfoHandle{}; + return Sample(sizeof(slot_type), sizeof(key_type), sizeof(value_type), + SooCapacity()); + } + + inline void destroy_slots() { + assert(!is_soo()); + if (PolicyTraits::template destroy_is_trivial<Alloc>()) return; + IterateOverFullSlots( + common(), slot_array(), + [&](const ctrl_t*, slot_type* slot) + ABSL_ATTRIBUTE_ALWAYS_INLINE { this->destroy(slot); }); + } + inline void dealloc() { assert(capacity() != 0); // Unpoison before returning the memory to the allocator. @@ -2881,6 +3549,12 @@ class raw_hash_set { inline void destructor_impl() { if (capacity() == 0) return; + if (is_soo()) { + if (!empty()) { + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED(destroy(soo_slot())); + } + return; + } destroy_slots(); dealloc(); } @@ -2890,10 +3564,16 @@ class raw_hash_set { // This merely updates the pertinent control byte. This can be used in // conjunction with Policy::transfer to move the object to another place. void erase_meta_only(const_iterator it) { + assert(!is_soo()); EraseMetaOnly(common(), static_cast<size_t>(it.control() - control()), sizeof(slot_type)); } + size_t hash_of(slot_type* slot) const { + return PolicyTraits::apply(HashElement{hash_ref()}, + PolicyTraits::element(slot)); + } + // Resizes table to the new capacity and move all elements to the new // positions accordingly. // @@ -2902,143 +3582,165 @@ class raw_hash_set { // HashSetResizeHelper::FindFirstNonFullAfterResize( // common(), old_capacity, hash) // can be called right after `resize`. - ABSL_ATTRIBUTE_NOINLINE void resize(size_t new_capacity) { + void resize(size_t new_capacity) { + raw_hash_set::resize_impl(common(), new_capacity, HashtablezInfoHandle{}); + } + + // As above, except that we also accept a pre-sampled, forced infoz for + // SOO tables, since they need to switch from SOO to heap in order to + // store the infoz. + void resize_with_soo_infoz(HashtablezInfoHandle forced_infoz) { + assert(forced_infoz.IsSampled()); + raw_hash_set::resize_impl(common(), NextCapacity(SooCapacity()), + forced_infoz); + } + + // Resizes set to the new capacity. + // It is a static function in order to use its pointer in GetPolicyFunctions. + ABSL_ATTRIBUTE_NOINLINE static void resize_impl( + CommonFields& common, size_t new_capacity, + HashtablezInfoHandle forced_infoz) { + raw_hash_set* set = reinterpret_cast<raw_hash_set*>(&common); assert(IsValidCapacity(new_capacity)); - HashSetResizeHelper resize_helper(common()); - auto* old_slots = slot_array(); - common().set_capacity(new_capacity); + assert(!set->fits_in_soo(new_capacity)); + const bool was_soo = set->is_soo(); + const bool had_soo_slot = was_soo && !set->empty(); + const ctrl_t soo_slot_h2 = + had_soo_slot ? static_cast<ctrl_t>(H2(set->hash_of(set->soo_slot()))) + : ctrl_t::kEmpty; + HashSetResizeHelper resize_helper(common, was_soo, had_soo_slot, + forced_infoz); + // Initialize HashSetResizeHelper::old_heap_or_soo_. We can't do this in + // HashSetResizeHelper constructor because it can't transfer slots when + // transfer_uses_memcpy is false. + // TODO(b/289225379): try to handle more of the SOO cases inside + // InitializeSlots. See comment on cl/555990034 snapshot #63. + if (PolicyTraits::transfer_uses_memcpy() || !had_soo_slot) { + resize_helper.old_heap_or_soo() = common.heap_or_soo(); + } else { + set->transfer(set->to_slot(resize_helper.old_soo_data()), + set->soo_slot()); + } + common.set_capacity(new_capacity); // Note that `InitializeSlots` does different number initialization steps // depending on the values of `transfer_uses_memcpy` and capacities. // Refer to the comment in `InitializeSlots` for more details. const bool grow_single_group = resize_helper.InitializeSlots<CharAlloc, sizeof(slot_type), PolicyTraits::transfer_uses_memcpy(), - alignof(slot_type)>( - common(), const_cast<std::remove_const_t<slot_type>*>(old_slots), - CharAlloc(alloc_ref())); + SooEnabled(), alignof(slot_type)>( + common, CharAlloc(set->alloc_ref()), soo_slot_h2, sizeof(key_type), + sizeof(value_type)); - if (resize_helper.old_capacity() == 0) { + // In the SooEnabled() case, capacity is never 0 so we don't check. + if (!SooEnabled() && resize_helper.old_capacity() == 0) { // InitializeSlots did all the work including infoz().RecordRehash(). return; } + assert(resize_helper.old_capacity() > 0); + // Nothing more to do in this case. + if (was_soo && !had_soo_slot) return; + slot_type* new_slots = set->slot_array(); if (grow_single_group) { if (PolicyTraits::transfer_uses_memcpy()) { // InitializeSlots did all the work. return; } - // We want GrowSizeIntoSingleGroup to be called here in order to make - // InitializeSlots not depend on PolicyTraits. - resize_helper.GrowSizeIntoSingleGroup<PolicyTraits>(common(), alloc_ref(), - old_slots); + if (was_soo) { + set->transfer(new_slots + resize_helper.SooSlotIndex(), + to_slot(resize_helper.old_soo_data())); + return; + } else { + // We want GrowSizeIntoSingleGroup to be called here in order to make + // InitializeSlots not depend on PolicyTraits. + resize_helper.GrowSizeIntoSingleGroup<PolicyTraits>(common, + set->alloc_ref()); + } } else { // InitializeSlots prepares control bytes to correspond to empty table. - auto* new_slots = slot_array(); - size_t total_probe_length = 0; - for (size_t i = 0; i != resize_helper.old_capacity(); ++i) { - if (IsFull(resize_helper.old_ctrl()[i])) { - size_t hash = PolicyTraits::apply( - HashElement{hash_ref()}, PolicyTraits::element(old_slots + i)); - auto target = find_first_non_full(common(), hash); - size_t new_i = target.offset; - total_probe_length += target.probe_length; - SetCtrl(common(), new_i, H2(hash), sizeof(slot_type)); - transfer(new_slots + new_i, old_slots + i); + const auto insert_slot = [&](slot_type* slot) { + size_t hash = PolicyTraits::apply(HashElement{set->hash_ref()}, + PolicyTraits::element(slot)); + auto target = find_first_non_full(common, hash); + SetCtrl(common, target.offset, H2(hash), sizeof(slot_type)); + set->transfer(new_slots + target.offset, slot); + return target.probe_length; + }; + if (was_soo) { + insert_slot(to_slot(resize_helper.old_soo_data())); + return; + } else { + auto* old_slots = static_cast<slot_type*>(resize_helper.old_slots()); + size_t total_probe_length = 0; + for (size_t i = 0; i != resize_helper.old_capacity(); ++i) { + if (IsFull(resize_helper.old_ctrl()[i])) { + total_probe_length += insert_slot(old_slots + i); + } } + common.infoz().RecordRehash(total_probe_length); } - infoz().RecordRehash(total_probe_length); } - resize_helper.DeallocateOld<alignof(slot_type)>( - CharAlloc(alloc_ref()), sizeof(slot_type), - const_cast<std::remove_const_t<slot_type>*>(old_slots)); + resize_helper.DeallocateOld<alignof(slot_type)>(CharAlloc(set->alloc_ref()), + sizeof(slot_type)); } - // Prunes control bytes to remove as many tombstones as possible. - // - // See the comment on `rehash_and_grow_if_necessary()`. - inline void drop_deletes_without_resize() { - // Stack-allocate space for swapping elements. - alignas(slot_type) unsigned char tmp[sizeof(slot_type)]; - DropDeletesWithoutResize(common(), GetPolicyFunctions(), tmp); - } + // Casting directly from e.g. char* to slot_type* can cause compilation errors + // on objective-C. This function converts to void* first, avoiding the issue. + static slot_type* to_slot(void* buf) { return static_cast<slot_type*>(buf); } - // Called whenever the table *might* need to conditionally grow. - // - // This function is an optimization opportunity to perform a rehash even when - // growth is unnecessary, because vacating tombstones is beneficial for - // performance in the long-run. - void rehash_and_grow_if_necessary() { - const size_t cap = capacity(); - if (cap > Group::kWidth && - // Do these calculations in 64-bit to avoid overflow. - size() * uint64_t{32} <= cap * uint64_t{25}) { - // Squash DELETED without growing if there is enough capacity. - // - // Rehash in place if the current size is <= 25/32 of capacity. - // Rationale for such a high factor: 1) drop_deletes_without_resize() is - // faster than resize, and 2) it takes quite a bit of work to add - // tombstones. In the worst case, seems to take approximately 4 - // insert/erase pairs to create a single tombstone and so if we are - // rehashing because of tombstones, we can afford to rehash-in-place as - // long as we are reclaiming at least 1/8 the capacity without doing more - // than 2X the work. (Where "work" is defined to be size() for rehashing - // or rehashing in place, and 1 for an insert or erase.) But rehashing in - // place is faster per operation than inserting or even doubling the size - // of the table, so we actually afford to reclaim even less space from a - // resize-in-place. The decision is to rehash in place if we can reclaim - // at about 1/8th of the usable capacity (specifically 3/28 of the - // capacity) which means that the total cost of rehashing will be a small - // fraction of the total work. - // - // Here is output of an experiment using the BM_CacheInSteadyState - // benchmark running the old case (where we rehash-in-place only if we can - // reclaim at least 7/16*capacity) vs. this code (which rehashes in place - // if we can recover 3/32*capacity). - // - // Note that although in the worst-case number of rehashes jumped up from - // 15 to 190, but the number of operations per second is almost the same. - // - // Abridged output of running BM_CacheInSteadyState benchmark from - // raw_hash_set_benchmark. N is the number of insert/erase operations. - // - // | OLD (recover >= 7/16 | NEW (recover >= 3/32) - // size | N/s LoadFactor NRehashes | N/s LoadFactor NRehashes - // 448 | 145284 0.44 18 | 140118 0.44 19 - // 493 | 152546 0.24 11 | 151417 0.48 28 - // 538 | 151439 0.26 11 | 151152 0.53 38 - // 583 | 151765 0.28 11 | 150572 0.57 50 - // 628 | 150241 0.31 11 | 150853 0.61 66 - // 672 | 149602 0.33 12 | 150110 0.66 90 - // 717 | 149998 0.35 12 | 149531 0.70 129 - // 762 | 149836 0.37 13 | 148559 0.74 190 - // 807 | 149736 0.39 14 | 151107 0.39 14 - // 852 | 150204 0.42 15 | 151019 0.42 15 - drop_deletes_without_resize(); + // Requires that lhs does not have a full SOO slot. + static void move_common(bool that_is_full_soo, allocator_type& rhs_alloc, + CommonFields& lhs, CommonFields&& rhs) { + if (PolicyTraits::transfer_uses_memcpy() || !that_is_full_soo) { + lhs = std::move(rhs); } else { - // Otherwise grow the container. - resize(NextCapacity(cap)); + lhs.move_non_heap_or_soo_fields(rhs); + // TODO(b/303305702): add reentrancy guard. + PolicyTraits::transfer(&rhs_alloc, to_slot(lhs.soo_data()), + to_slot(rhs.soo_data())); } } + // Swaps common fields making sure to avoid memcpy'ing a full SOO slot if we + // aren't allowed to do so. + void swap_common(raw_hash_set& that) { + using std::swap; + if (PolicyTraits::transfer_uses_memcpy()) { + swap(common(), that.common()); + return; + } + CommonFields tmp = CommonFields::CreateDefault<SooEnabled()>(); + const bool that_is_full_soo = that.is_full_soo(); + move_common(that_is_full_soo, that.alloc_ref(), tmp, + std::move(that.common())); + move_common(is_full_soo(), alloc_ref(), that.common(), std::move(common())); + move_common(that_is_full_soo, that.alloc_ref(), common(), std::move(tmp)); + } + void maybe_increment_generation_or_rehash_on_move() { - common().maybe_increment_generation_on_move(); + if (!SwisstableGenerationsEnabled() || capacity() == 0 || is_soo()) { + return; + } + common().increment_generation(); if (!empty() && common().should_rehash_for_bug_detection_on_move()) { resize(capacity()); } } - template<bool propagate_alloc> + template <bool propagate_alloc> raw_hash_set& assign_impl(raw_hash_set&& that) { // We don't bother checking for this/that aliasing. We just need to avoid // breaking the invariants in that case. destructor_impl(); - common() = std::move(that.common()); + move_common(that.is_full_soo(), that.alloc_ref(), common(), + std::move(that.common())); // TODO(b/296061262): move instead of copying hash/eq/alloc. hash_ref() = that.hash_ref(); eq_ref() = that.eq_ref(); CopyAlloc(alloc_ref(), that.alloc_ref(), std::integral_constant<bool, propagate_alloc>()); - that.common() = CommonFields{}; + that.common() = CommonFields::CreateDefault<SooEnabled()>(); maybe_increment_generation_or_rehash_on_move(); return *this; } @@ -3051,8 +3753,8 @@ class raw_hash_set { insert(std::move(PolicyTraits::element(it.slot()))); that.destroy(it.slot()); } - that.dealloc(); - that.common() = CommonFields{}; + if (!that.is_soo()) that.dealloc(); + that.common() = CommonFields::CreateDefault<SooEnabled()>(); maybe_increment_generation_or_rehash_on_move(); return *this; } @@ -3078,12 +3780,30 @@ class raw_hash_set { return move_elements_allocs_unequal(std::move(that)); } - protected: - // Attempts to find `key` in the table; if it isn't found, returns a slot that - // the value can be inserted into, with the control byte already set to - // `key`'s H2. template <class K> - std::pair<size_t, bool> find_or_prepare_insert(const K& key) { + std::pair<iterator, bool> find_or_prepare_insert_soo(const K& key) { + if (empty()) { + const HashtablezInfoHandle infoz = try_sample_soo(); + if (infoz.IsSampled()) { + resize_with_soo_infoz(infoz); + } else { + common().set_full_soo(); + return {soo_iterator(), true}; + } + } else if (PolicyTraits::apply(EqualElement<K>{key, eq_ref()}, + PolicyTraits::element(soo_slot()))) { + return {soo_iterator(), false}; + } else { + resize(NextCapacity(SooCapacity())); + } + const size_t index = + PrepareInsertAfterSoo(hash_ref()(key), sizeof(slot_type), common()); + return {iterator_at(index), true}; + } + + template <class K> + std::pair<iterator, bool> find_or_prepare_insert_non_soo(const K& key) { + assert(!is_soo()); prefetch_heap_block(); auto hash = hash_ref()(key); auto seq = probe(common(), hash); @@ -3094,65 +3814,92 @@ class raw_hash_set { if (ABSL_PREDICT_TRUE(PolicyTraits::apply( EqualElement<K>{key, eq_ref()}, PolicyTraits::element(slot_array() + seq.offset(i))))) - return {seq.offset(i), false}; + return {iterator_at(seq.offset(i)), false}; + } + auto mask_empty = g.MaskEmpty(); + if (ABSL_PREDICT_TRUE(mask_empty)) { + size_t target = seq.offset( + GetInsertionOffset(mask_empty, capacity(), hash, control())); + return {iterator_at(PrepareInsertNonSoo(common(), hash, + FindInfo{target, seq.index()}, + GetPolicyFunctions())), + true}; } - if (ABSL_PREDICT_TRUE(g.MaskEmpty())) break; seq.next(); assert(seq.index() <= capacity() && "full table!"); } - return {prepare_insert(hash), true}; } - // Given the hash of a value not currently in the table, finds the next - // viable slot index to insert it at. - // - // REQUIRES: At least one non-full slot available. - size_t prepare_insert(size_t hash) ABSL_ATTRIBUTE_NOINLINE { - const bool rehash_for_bug_detection = - common().should_rehash_for_bug_detection_on_insert(); - if (rehash_for_bug_detection) { - // Move to a different heap allocation in order to detect bugs. - const size_t cap = capacity(); - resize(growth_left() > 0 ? cap : NextCapacity(cap)); - } - auto target = find_first_non_full(common(), hash); - if (!rehash_for_bug_detection && - ABSL_PREDICT_FALSE(growth_left() == 0 && - !IsDeleted(control()[target.offset]))) { - size_t old_capacity = capacity(); - rehash_and_grow_if_necessary(); - // NOTE: It is safe to use `FindFirstNonFullAfterResize`. - // `FindFirstNonFullAfterResize` must be called right after resize. - // `rehash_and_grow_if_necessary` may *not* call `resize` - // and perform `drop_deletes_without_resize` instead. But this - // could happen only on big tables. - // For big tables `FindFirstNonFullAfterResize` will always - // fallback to normal `find_first_non_full`, so it is safe to use it. - target = HashSetResizeHelper::FindFirstNonFullAfterResize( - common(), old_capacity, hash); - } - common().increment_size(); - set_growth_left(growth_left() - IsEmpty(control()[target.offset])); - SetCtrl(common(), target.offset, H2(hash), sizeof(slot_type)); - common().maybe_increment_generation_on_insert(); - infoz().RecordInsert(hash, target.probe_length); - return target.offset; + protected: + // Asserts that hash and equal functors provided by the user are consistent, + // meaning that `eq(k1, k2)` implies `hash(k1)==hash(k2)`. + template <class K> + void AssertHashEqConsistent(ABSL_ATTRIBUTE_UNUSED const K& key) { +#ifndef NDEBUG + if (empty()) return; + + const size_t hash_of_arg = hash_ref()(key); + const auto assert_consistent = [&](const ctrl_t*, slot_type* slot) { + const value_type& element = PolicyTraits::element(slot); + const bool is_key_equal = + PolicyTraits::apply(EqualElement<K>{key, eq_ref()}, element); + if (!is_key_equal) return; + + const size_t hash_of_slot = + PolicyTraits::apply(HashElement{hash_ref()}, element); + const bool is_hash_equal = hash_of_arg == hash_of_slot; + if (!is_hash_equal) { + // In this case, we're going to crash. Do a couple of other checks for + // idempotence issues. Recalculating hash/eq here is also convenient for + // debugging with gdb/lldb. + const size_t once_more_hash_arg = hash_ref()(key); + assert(hash_of_arg == once_more_hash_arg && "hash is not idempotent."); + const size_t once_more_hash_slot = + PolicyTraits::apply(HashElement{hash_ref()}, element); + assert(hash_of_slot == once_more_hash_slot && + "hash is not idempotent."); + const bool once_more_eq = + PolicyTraits::apply(EqualElement<K>{key, eq_ref()}, element); + assert(is_key_equal == once_more_eq && "equality is not idempotent."); + } + assert((!is_key_equal || is_hash_equal) && + "eq(k1, k2) must imply that hash(k1) == hash(k2). " + "hash/eq functors are inconsistent."); + }; + + if (is_soo()) { + assert_consistent(/*unused*/ nullptr, soo_slot()); + return; + } + // We only do validation for small tables so that it's constant time. + if (capacity() > 16) return; + IterateOverFullSlots(common(), slot_array(), assert_consistent); +#endif + } + + // Attempts to find `key` in the table; if it isn't found, returns an iterator + // where the value can be inserted into, with the control byte already set to + // `key`'s H2. Returns a bool indicating whether an insertion can take place. + template <class K> + std::pair<iterator, bool> find_or_prepare_insert(const K& key) { + AssertHashEqConsistent(key); + if (is_soo()) return find_or_prepare_insert_soo(key); + return find_or_prepare_insert_non_soo(key); } // Constructs the value in the space pointed by the iterator. This only works // after an unsuccessful find_or_prepare_insert() and before any other // modifications happen in the raw_hash_set. // - // PRECONDITION: i is an index returned from find_or_prepare_insert(k), where - // k is the key decomposed from `forward<Args>(args)...`, and the bool - // returned by find_or_prepare_insert(k) was true. + // PRECONDITION: iter was returned from find_or_prepare_insert(k), where k is + // the key decomposed from `forward<Args>(args)...`, and the bool returned by + // find_or_prepare_insert(k) was true. // POSTCONDITION: *m.iterator_at(i) == value_type(forward<Args>(args)...). template <class... Args> - void emplace_at(size_t i, Args&&... args) { - construct(slot_array() + i, std::forward<Args>(args)...); + void emplace_at(iterator iter, Args&&... args) { + construct(iter.slot(), std::forward<Args>(args)...); - assert(PolicyTraits::apply(FindElement{*this}, *iterator_at(i)) == - iterator_at(i) && + assert(PolicyTraits::apply(FindElement{*this}, *iter) == iter && "constructed value does not match the lookup key"); } @@ -3160,7 +3907,7 @@ class raw_hash_set { return {control() + i, slot_array() + i, common().generation_ptr()}; } const_iterator iterator_at(size_t i) const ABSL_ATTRIBUTE_LIFETIME_BOUND { - return {control() + i, slot_array() + i, common().generation_ptr()}; + return const_cast<raw_hash_set*>(this)->iterator_at(i); } reference unchecked_deref(iterator it) { return it.unchecked_deref(); } @@ -3178,13 +3925,25 @@ class raw_hash_set { // side-effect. // // See `CapacityToGrowth()`. - size_t growth_left() const { return common().growth_left(); } - void set_growth_left(size_t gl) { return common().set_growth_left(gl); } + size_t growth_left() const { + assert(!is_soo()); + return common().growth_left(); + } + + GrowthInfo& growth_info() { + assert(!is_soo()); + return common().growth_info(); + } + GrowthInfo growth_info() const { + assert(!is_soo()); + return common().growth_info(); + } // Prefetch the heap-allocated memory region to resolve potential TLB and // cache misses. This is intended to overlap with execution of calculating the // hash for a key. void prefetch_heap_block() const { + assert(!is_soo()); #if ABSL_HAVE_BUILTIN(__builtin_prefetch) || defined(__GNUC__) __builtin_prefetch(control(), 0, 1); #endif @@ -3193,11 +3952,31 @@ class raw_hash_set { CommonFields& common() { return settings_.template get<0>(); } const CommonFields& common() const { return settings_.template get<0>(); } - ctrl_t* control() const { return common().control(); } + ctrl_t* control() const { + assert(!is_soo()); + return common().control(); + } slot_type* slot_array() const { + assert(!is_soo()); return static_cast<slot_type*>(common().slot_array()); } - HashtablezInfoHandle infoz() { return common().infoz(); } + slot_type* soo_slot() { + assert(is_soo()); + return static_cast<slot_type*>(common().soo_data()); + } + const slot_type* soo_slot() const { + return const_cast<raw_hash_set*>(this)->soo_slot(); + } + iterator soo_iterator() { + return {SooControl(), soo_slot(), common().generation_ptr()}; + } + const_iterator soo_iterator() const { + return const_cast<raw_hash_set*>(this)->soo_iterator(); + } + HashtablezInfoHandle infoz() { + assert(!is_soo()); + return common().infoz(); + } hasher& hash_ref() { return settings_.template get<1>(); } const hasher& hash_ref() const { return settings_.template get<1>(); } @@ -3208,12 +3987,9 @@ class raw_hash_set { return settings_.template get<3>(); } - // Make type-specific functions for this type's PolicyFunctions struct. - static size_t hash_slot_fn(void* set, void* slot) { - auto* h = static_cast<raw_hash_set*>(set); - return PolicyTraits::apply( - HashElement{h->hash_ref()}, - PolicyTraits::element(static_cast<slot_type*>(slot))); + static const void* get_hash_ref_fn(const CommonFields& common) { + auto* h = reinterpret_cast<const raw_hash_set*>(&common); + return &h->hash_ref(); } static void transfer_slot_fn(void* set, void* dst, void* src) { auto* h = static_cast<raw_hash_set*>(set); @@ -3236,13 +4012,18 @@ class raw_hash_set { static const PolicyFunctions& GetPolicyFunctions() { static constexpr PolicyFunctions value = { sizeof(slot_type), - &raw_hash_set::hash_slot_fn, + // TODO(b/328722020): try to type erase + // for standard layout and alignof(Hash) <= alignof(CommonFields). + std::is_empty<hasher>::value ? &GetHashRefForEmptyHasher + : &raw_hash_set::get_hash_ref_fn, + PolicyTraits::template get_hash_slot_fn<hasher>(), PolicyTraits::transfer_uses_memcpy() ? TransferRelocatable<sizeof(slot_type)> : &raw_hash_set::transfer_slot_fn, (std::is_same<SlotAlloc, std::allocator<slot_type>>::value ? &DeallocateStandard<alignof(slot_type)> : &raw_hash_set::dealloc_fn), + &raw_hash_set::resize_impl, }; return value; } @@ -3252,22 +4033,78 @@ class raw_hash_set { // fields that occur after CommonFields. absl::container_internal::CompressedTuple<CommonFields, hasher, key_equal, allocator_type> - settings_{CommonFields{}, hasher{}, key_equal{}, allocator_type{}}; + settings_{CommonFields::CreateDefault<SooEnabled()>(), hasher{}, + key_equal{}, allocator_type{}}; +}; + +// Friend access for free functions in raw_hash_set.h. +struct HashtableFreeFunctionsAccess { + template <class Predicate, typename Set> + static typename Set::size_type EraseIf(Predicate& pred, Set* c) { + if (c->empty()) { + return 0; + } + if (c->is_soo()) { + auto it = c->soo_iterator(); + if (!pred(*it)) { + assert(c->size() == 1 && "hash table was modified unexpectedly"); + return 0; + } + c->destroy(it.slot()); + c->common().set_empty_soo(); + return 1; + } + ABSL_ATTRIBUTE_UNUSED const size_t original_size_for_assert = c->size(); + size_t num_deleted = 0; + IterateOverFullSlots( + c->common(), c->slot_array(), [&](const ctrl_t* ctrl, auto* slot) { + if (pred(Set::PolicyTraits::element(slot))) { + c->destroy(slot); + EraseMetaOnly(c->common(), static_cast<size_t>(ctrl - c->control()), + sizeof(*slot)); + ++num_deleted; + } + }); + // NOTE: IterateOverFullSlots allow removal of the current element, so we + // verify the size additionally here. + assert(original_size_for_assert - num_deleted == c->size() && + "hash table was modified unexpectedly"); + return num_deleted; + } + + template <class Callback, typename Set> + static void ForEach(Callback& cb, Set* c) { + if (c->empty()) { + return; + } + if (c->is_soo()) { + cb(*c->soo_iterator()); + return; + } + using ElementTypeWithConstness = decltype(*c->begin()); + IterateOverFullSlots( + c->common(), c->slot_array(), [&cb](const ctrl_t*, auto* slot) { + ElementTypeWithConstness& element = Set::PolicyTraits::element(slot); + cb(element); + }); + } }; // Erases all elements that satisfy the predicate `pred` from the container `c`. template <typename P, typename H, typename E, typename A, typename Predicate> typename raw_hash_set<P, H, E, A>::size_type EraseIf( Predicate& pred, raw_hash_set<P, H, E, A>* c) { - const auto initial_size = c->size(); - for (auto it = c->begin(), last = c->end(); it != last;) { - if (pred(*it)) { - c->erase(it++); - } else { - ++it; - } - } - return initial_size - c->size(); + return HashtableFreeFunctionsAccess::EraseIf(pred, c); +} + +// Calls `cb` for all elements in the container `c`. +template <typename P, typename H, typename E, typename A, typename Callback> +void ForEach(Callback& cb, raw_hash_set<P, H, E, A>* c) { + return HashtableFreeFunctionsAccess::ForEach(cb, c); +} +template <typename P, typename H, typename E, typename A, typename Callback> +void ForEach(Callback& cb, const raw_hash_set<P, H, E, A>* c) { + return HashtableFreeFunctionsAccess::ForEach(cb, c); } namespace hashtable_debug_internal { @@ -3278,6 +4115,7 @@ struct HashtableDebugAccess<Set, absl::void_t<typename Set::raw_hash_set>> { static size_t GetNumProbes(const Set& set, const typename Set::key_type& key) { + if (set.is_soo()) return 0; size_t num_probes = 0; size_t hash = set.hash_ref()(key); auto seq = probe(set.common(), hash); @@ -3301,7 +4139,8 @@ struct HashtableDebugAccess<Set, absl::void_t<typename Set::raw_hash_set>> { static size_t AllocatedByteSize(const Set& c) { size_t capacity = c.capacity(); if (capacity == 0) return 0; - size_t m = c.common().alloc_size(sizeof(Slot), alignof(Slot)); + size_t m = + c.is_soo() ? 0 : c.common().alloc_size(sizeof(Slot), alignof(Slot)); size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr)); if (per_slot != ~size_t{}) { @@ -3321,5 +4160,7 @@ ABSL_NAMESPACE_END } // namespace absl #undef ABSL_SWISSTABLE_ENABLE_GENERATIONS +#undef ABSL_SWISSTABLE_IGNORE_UNINITIALIZED +#undef ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN #endif // ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ diff --git a/absl/container/internal/raw_hash_set_allocator_test.cc b/absl/container/internal/raw_hash_set_allocator_test.cc index 05dcfaa6..7e7a5063 100644 --- a/absl/container/internal/raw_hash_set_allocator_test.cc +++ b/absl/container/internal/raw_hash_set_allocator_test.cc @@ -25,6 +25,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/base/config.h" +#include "absl/container/internal/container_memory.h" #include "absl/container/internal/raw_hash_set.h" #include "absl/container/internal/tracked.h" @@ -133,7 +134,7 @@ class CheckedAlloc { }; struct Identity { - int32_t operator()(int32_t v) const { return v; } + size_t operator()(int32_t v) const { return static_cast<size_t>(v); } }; struct Policy { @@ -178,6 +179,11 @@ struct Policy { } static slot_type& element(slot_type* slot) { return *slot; } + + template <class Hash> + static constexpr HashSlotFn get_hash_slot_fn() { + return nullptr; + } }; template <int Spec> diff --git a/absl/container/internal/raw_hash_set_benchmark.cc b/absl/container/internal/raw_hash_set_benchmark.cc index 88b07373..424b72cf 100644 --- a/absl/container/internal/raw_hash_set_benchmark.cc +++ b/absl/container/internal/raw_hash_set_benchmark.cc @@ -12,19 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include <algorithm> #include <array> #include <cmath> #include <cstddef> #include <cstdint> +#include <limits> #include <numeric> #include <random> +#include <string> #include <tuple> #include <utility> #include <vector> #include "absl/base/internal/raw_logging.h" +#include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_function_defaults.h" #include "absl/container/internal/raw_hash_set.h" +#include "absl/random/random.h" #include "absl/strings/str_format.h" #include "benchmark/benchmark.h" @@ -58,6 +63,11 @@ struct IntPolicy { static auto apply(F&& f, int64_t x) -> decltype(std::forward<F>(f)(x, x)) { return std::forward<F>(f)(x, x); } + + template <class Hash> + static constexpr HashSlotFn get_hash_slot_fn() { + return nullptr; + } }; class StringPolicy { @@ -116,6 +126,11 @@ class StringPolicy { return apply_impl(std::forward<F>(f), PairArgs(std::forward<Args>(args)...)); } + + template <class Hash> + static constexpr HashSlotFn get_hash_slot_fn() { + return nullptr; + } }; struct StringHash : container_internal::hash_default_hash<absl::string_view> { @@ -294,7 +309,7 @@ void BM_CopyCtorSparseInt(benchmark::State& state) { benchmark::DoNotOptimize(t2); } } -BENCHMARK(BM_CopyCtorSparseInt)->Range(128, 4096); +BENCHMARK(BM_CopyCtorSparseInt)->Range(1, 4096); void BM_CopyCtorInt(benchmark::State& state) { std::random_device rd; @@ -312,7 +327,7 @@ void BM_CopyCtorInt(benchmark::State& state) { benchmark::DoNotOptimize(t2); } } -BENCHMARK(BM_CopyCtorInt)->Range(128, 4096); +BENCHMARK(BM_CopyCtorInt)->Range(0, 4096); void BM_CopyCtorString(benchmark::State& state) { std::random_device rd; @@ -330,7 +345,7 @@ void BM_CopyCtorString(benchmark::State& state) { benchmark::DoNotOptimize(t2); } } -BENCHMARK(BM_CopyCtorString)->Range(128, 4096); +BENCHMARK(BM_CopyCtorString)->Range(0, 4096); void BM_CopyAssign(benchmark::State& state) { std::random_device rd; @@ -445,6 +460,19 @@ void BM_Group_Match(benchmark::State& state) { } BENCHMARK(BM_Group_Match); +void BM_GroupPortable_Match(benchmark::State& state) { + std::array<ctrl_t, GroupPortableImpl::kWidth> group; + Iota(group.begin(), group.end(), -4); + GroupPortableImpl g{group.data()}; + h2_t h = 1; + for (auto _ : state) { + ::benchmark::DoNotOptimize(h); + ::benchmark::DoNotOptimize(g); + ::benchmark::DoNotOptimize(g.Match(h)); + } +} +BENCHMARK(BM_GroupPortable_Match); + void BM_Group_MaskEmpty(benchmark::State& state) { std::array<ctrl_t, Group::kWidth> group; Iota(group.begin(), group.end(), -4); @@ -467,6 +495,17 @@ void BM_Group_MaskEmptyOrDeleted(benchmark::State& state) { } BENCHMARK(BM_Group_MaskEmptyOrDeleted); +void BM_Group_MaskNonFull(benchmark::State& state) { + std::array<ctrl_t, Group::kWidth> group; + Iota(group.begin(), group.end(), -4); + Group g{group.data()}; + for (auto _ : state) { + ::benchmark::DoNotOptimize(g); + ::benchmark::DoNotOptimize(g.MaskNonFull()); + } +} +BENCHMARK(BM_Group_MaskNonFull); + void BM_Group_CountLeadingEmptyOrDeleted(benchmark::State& state) { std::array<ctrl_t, Group::kWidth> group; Iota(group.begin(), group.end(), -2); @@ -489,6 +528,17 @@ void BM_Group_MatchFirstEmptyOrDeleted(benchmark::State& state) { } BENCHMARK(BM_Group_MatchFirstEmptyOrDeleted); +void BM_Group_MatchFirstNonFull(benchmark::State& state) { + std::array<ctrl_t, Group::kWidth> group; + Iota(group.begin(), group.end(), -2); + Group g{group.data()}; + for (auto _ : state) { + ::benchmark::DoNotOptimize(g); + ::benchmark::DoNotOptimize(g.MaskNonFull().LowestBitSet()); + } +} +BENCHMARK(BM_Group_MatchFirstNonFull); + void BM_DropDeletes(benchmark::State& state) { constexpr size_t capacity = (1 << 20) - 1; std::vector<ctrl_t> ctrl(capacity + 1 + Group::kWidth); @@ -528,6 +578,67 @@ void BM_Resize(benchmark::State& state) { } BENCHMARK(BM_Resize); +void BM_EraseIf(benchmark::State& state) { + int64_t num_elements = state.range(0); + size_t num_erased = static_cast<size_t>(state.range(1)); + + constexpr size_t kRepetitions = 64; + + absl::BitGen rng; + + std::vector<std::vector<int64_t>> keys(kRepetitions); + std::vector<IntTable> tables; + std::vector<int64_t> threshold; + for (auto& k : keys) { + tables.push_back(IntTable()); + auto& table = tables.back(); + for (int64_t i = 0; i < num_elements; i++) { + // We use random keys to reduce noise. + k.push_back( + absl::Uniform<int64_t>(rng, 0, std::numeric_limits<int64_t>::max())); + if (!table.insert(k.back()).second) { + k.pop_back(); + --i; // duplicated value, retrying + } + } + std::sort(k.begin(), k.end()); + threshold.push_back(static_cast<int64_t>(num_erased) < num_elements + ? k[num_erased] + : std::numeric_limits<int64_t>::max()); + } + + while (state.KeepRunningBatch(static_cast<int64_t>(kRepetitions) * + std::max(num_elements, int64_t{1}))) { + benchmark::DoNotOptimize(tables); + for (size_t t_id = 0; t_id < kRepetitions; t_id++) { + auto& table = tables[t_id]; + benchmark::DoNotOptimize(num_erased); + auto pred = [t = threshold[t_id]](int64_t key) { return key < t; }; + benchmark::DoNotOptimize(pred); + benchmark::DoNotOptimize(table); + absl::container_internal::EraseIf(pred, &table); + } + state.PauseTiming(); + for (size_t t_id = 0; t_id < kRepetitions; t_id++) { + auto& k = keys[t_id]; + auto& table = tables[t_id]; + for (size_t i = 0; i < num_erased; i++) { + table.insert(k[i]); + } + } + state.ResumeTiming(); + } +} + +BENCHMARK(BM_EraseIf) + ->ArgNames({"num_elements", "num_erased"}) + ->ArgPair(10, 0) + ->ArgPair(1000, 0) + ->ArgPair(10, 5) + ->ArgPair(1000, 500) + ->ArgPair(10, 10) + ->ArgPair(1000, 1000); + } // namespace } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/internal/raw_hash_set_probe_benchmark.cc b/absl/container/internal/raw_hash_set_probe_benchmark.cc index 5d4184b2..8f36305d 100644 --- a/absl/container/internal/raw_hash_set_probe_benchmark.cc +++ b/absl/container/internal/raw_hash_set_probe_benchmark.cc @@ -70,6 +70,11 @@ struct Policy { -> decltype(std::forward<F>(f)(arg, arg)) { return std::forward<F>(f)(arg, arg); } + + template <class Hash> + static constexpr auto get_hash_slot_fn() { + return nullptr; + } }; absl::BitGen& GlobalBitGen() { diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index f9797f56..f1257d4b 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -15,6 +15,7 @@ #include "absl/container/internal/raw_hash_set.h" #include <algorithm> +#include <array> #include <atomic> #include <cmath> #include <cstddef> @@ -51,10 +52,15 @@ #include "absl/container/internal/hashtable_debug.h" #include "absl/container/internal/hashtablez_sampler.h" #include "absl/container/internal/test_allocator.h" +#include "absl/container/internal/test_instance_tracker.h" +#include "absl/container/node_hash_set.h" +#include "absl/functional/function_ref.h" #include "absl/hash/hash.h" +#include "absl/log/check.h" #include "absl/log/log.h" #include "absl/memory/memory.h" #include "absl/meta/type_traits.h" +#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" namespace absl { @@ -63,6 +69,10 @@ namespace container_internal { struct RawHashSetTestOnlyAccess { template <typename C> + static auto GetCommon(const C& c) -> decltype(c.common()) { + return c.common(); + } + template <typename C> static auto GetSlots(const C& c) -> decltype(c.slot_array()) { return c.slot_array(); } @@ -75,6 +85,7 @@ struct RawHashSetTestOnlyAccess { namespace { using ::testing::ElementsAre; +using ::testing::ElementsAreArray; using ::testing::Eq; using ::testing::Ge; using ::testing::Lt; @@ -84,6 +95,94 @@ using ::testing::UnorderedElementsAre; // Convenience function to static cast to ctrl_t. ctrl_t CtrlT(int i) { return static_cast<ctrl_t>(i); } +TEST(GrowthInfoTest, GetGrowthLeft) { + GrowthInfo gi; + gi.InitGrowthLeftNoDeleted(5); + EXPECT_EQ(gi.GetGrowthLeft(), 5); + gi.OverwriteFullAsDeleted(); + EXPECT_EQ(gi.GetGrowthLeft(), 5); +} + +TEST(GrowthInfoTest, HasNoDeleted) { + GrowthInfo gi; + gi.InitGrowthLeftNoDeleted(5); + EXPECT_TRUE(gi.HasNoDeleted()); + gi.OverwriteFullAsDeleted(); + EXPECT_FALSE(gi.HasNoDeleted()); + // After reinitialization we have no deleted slots. + gi.InitGrowthLeftNoDeleted(5); + EXPECT_TRUE(gi.HasNoDeleted()); +} + +TEST(GrowthInfoTest, HasNoDeletedAndGrowthLeft) { + GrowthInfo gi; + gi.InitGrowthLeftNoDeleted(5); + EXPECT_TRUE(gi.HasNoDeletedAndGrowthLeft()); + gi.OverwriteFullAsDeleted(); + EXPECT_FALSE(gi.HasNoDeletedAndGrowthLeft()); + gi.InitGrowthLeftNoDeleted(0); + EXPECT_FALSE(gi.HasNoDeletedAndGrowthLeft()); + gi.OverwriteFullAsDeleted(); + EXPECT_FALSE(gi.HasNoDeletedAndGrowthLeft()); + // After reinitialization we have no deleted slots. + gi.InitGrowthLeftNoDeleted(5); + EXPECT_TRUE(gi.HasNoDeletedAndGrowthLeft()); +} + +TEST(GrowthInfoTest, HasNoGrowthLeftAndNoDeleted) { + GrowthInfo gi; + gi.InitGrowthLeftNoDeleted(1); + EXPECT_FALSE(gi.HasNoGrowthLeftAndNoDeleted()); + gi.OverwriteEmptyAsFull(); + EXPECT_TRUE(gi.HasNoGrowthLeftAndNoDeleted()); + gi.OverwriteFullAsDeleted(); + EXPECT_FALSE(gi.HasNoGrowthLeftAndNoDeleted()); + gi.OverwriteFullAsEmpty(); + EXPECT_FALSE(gi.HasNoGrowthLeftAndNoDeleted()); + gi.InitGrowthLeftNoDeleted(0); + EXPECT_TRUE(gi.HasNoGrowthLeftAndNoDeleted()); + gi.OverwriteFullAsEmpty(); + EXPECT_FALSE(gi.HasNoGrowthLeftAndNoDeleted()); +} + +TEST(GrowthInfoTest, OverwriteFullAsEmpty) { + GrowthInfo gi; + gi.InitGrowthLeftNoDeleted(5); + gi.OverwriteFullAsEmpty(); + EXPECT_EQ(gi.GetGrowthLeft(), 6); + gi.OverwriteFullAsDeleted(); + EXPECT_EQ(gi.GetGrowthLeft(), 6); + gi.OverwriteFullAsEmpty(); + EXPECT_EQ(gi.GetGrowthLeft(), 7); + EXPECT_FALSE(gi.HasNoDeleted()); +} + +TEST(GrowthInfoTest, OverwriteEmptyAsFull) { + GrowthInfo gi; + gi.InitGrowthLeftNoDeleted(5); + gi.OverwriteEmptyAsFull(); + EXPECT_EQ(gi.GetGrowthLeft(), 4); + gi.OverwriteFullAsDeleted(); + EXPECT_EQ(gi.GetGrowthLeft(), 4); + gi.OverwriteEmptyAsFull(); + EXPECT_EQ(gi.GetGrowthLeft(), 3); + EXPECT_FALSE(gi.HasNoDeleted()); +} + +TEST(GrowthInfoTest, OverwriteControlAsFull) { + GrowthInfo gi; + gi.InitGrowthLeftNoDeleted(5); + gi.OverwriteControlAsFull(ctrl_t::kEmpty); + EXPECT_EQ(gi.GetGrowthLeft(), 4); + gi.OverwriteControlAsFull(ctrl_t::kDeleted); + EXPECT_EQ(gi.GetGrowthLeft(), 4); + gi.OverwriteFullAsDeleted(); + gi.OverwriteControlAsFull(ctrl_t::kDeleted); + // We do not count number of deleted, so the bit sticks till the next rehash. + EXPECT_FALSE(gi.HasNoDeletedAndGrowthLeft()); + EXPECT_FALSE(gi.HasNoDeleted()); +} + TEST(Util, NormalizeCapacity) { EXPECT_EQ(1, NormalizeCapacity(0)); EXPECT_EQ(1, NormalizeCapacity(1)); @@ -156,20 +255,66 @@ TEST(BitMask, Smoke) { EXPECT_THAT((BitMask<uint8_t, 8>(0xAA)), ElementsAre(1, 3, 5, 7)); } -TEST(BitMask, WithShift) { +TEST(BitMask, WithShift_MatchPortable) { // See the non-SSE version of Group for details on what this math is for. uint64_t ctrl = 0x1716151413121110; uint64_t hash = 0x12; - constexpr uint64_t msbs = 0x8080808080808080ULL; constexpr uint64_t lsbs = 0x0101010101010101ULL; auto x = ctrl ^ (lsbs * hash); - uint64_t mask = (x - lsbs) & ~x & msbs; + uint64_t mask = (x - lsbs) & ~x & kMsbs8Bytes; EXPECT_EQ(0x0000000080800000, mask); BitMask<uint64_t, 8, 3> b(mask); EXPECT_EQ(*b, 2); } +constexpr uint64_t kSome8BytesMask = /* */ 0x8000808080008000ULL; +constexpr uint64_t kSome8BytesMaskAllOnes = 0xff00ffffff00ff00ULL; +constexpr auto kSome8BytesMaskBits = std::array<int, 5>{1, 3, 4, 5, 7}; + + +TEST(BitMask, WithShift_FullMask) { + EXPECT_THAT((BitMask<uint64_t, 8, 3>(kMsbs8Bytes)), + ElementsAre(0, 1, 2, 3, 4, 5, 6, 7)); + EXPECT_THAT( + (BitMask<uint64_t, 8, 3, /*NullifyBitsOnIteration=*/true>(kMsbs8Bytes)), + ElementsAre(0, 1, 2, 3, 4, 5, 6, 7)); + EXPECT_THAT( + (BitMask<uint64_t, 8, 3, /*NullifyBitsOnIteration=*/true>(~uint64_t{0})), + ElementsAre(0, 1, 2, 3, 4, 5, 6, 7)); +} + +TEST(BitMask, WithShift_EmptyMask) { + EXPECT_THAT((BitMask<uint64_t, 8, 3>(0)), ElementsAre()); + EXPECT_THAT((BitMask<uint64_t, 8, 3, /*NullifyBitsOnIteration=*/true>(0)), + ElementsAre()); +} + +TEST(BitMask, WithShift_SomeMask) { + EXPECT_THAT((BitMask<uint64_t, 8, 3>(kSome8BytesMask)), + ElementsAreArray(kSome8BytesMaskBits)); + EXPECT_THAT((BitMask<uint64_t, 8, 3, /*NullifyBitsOnIteration=*/true>( + kSome8BytesMask)), + ElementsAreArray(kSome8BytesMaskBits)); + EXPECT_THAT((BitMask<uint64_t, 8, 3, /*NullifyBitsOnIteration=*/true>( + kSome8BytesMaskAllOnes)), + ElementsAreArray(kSome8BytesMaskBits)); +} + +TEST(BitMask, WithShift_SomeMaskExtraBitsForNullify) { + // Verify that adding extra bits into non zero bytes is fine. + uint64_t extra_bits = 77; + for (int i = 0; i < 100; ++i) { + // Add extra bits, but keep zero bytes untouched. + uint64_t extra_mask = extra_bits & kSome8BytesMaskAllOnes; + EXPECT_THAT((BitMask<uint64_t, 8, 3, /*NullifyBitsOnIteration=*/true>( + kSome8BytesMask | extra_mask)), + ElementsAreArray(kSome8BytesMaskBits)) + << i << " " << extra_mask; + extra_bits = (extra_bits + 1) * 3; + } +} + TEST(BitMask, LeadingTrailing) { EXPECT_EQ((BitMask<uint32_t, 16>(0x00001a40).LeadingZeros()), 3); EXPECT_EQ((BitMask<uint32_t, 16>(0x00001a40).TrailingZeros()), 6); @@ -255,6 +400,25 @@ TEST(Group, MaskFull) { } } +TEST(Group, MaskNonFull) { + if (Group::kWidth == 16) { + ctrl_t group[] = { + ctrl_t::kEmpty, CtrlT(1), ctrl_t::kDeleted, CtrlT(3), + ctrl_t::kEmpty, CtrlT(5), ctrl_t::kSentinel, CtrlT(7), + CtrlT(7), CtrlT(5), ctrl_t::kDeleted, CtrlT(1), + CtrlT(1), ctrl_t::kSentinel, ctrl_t::kEmpty, CtrlT(1)}; + EXPECT_THAT(Group{group}.MaskNonFull(), + ElementsAre(0, 2, 4, 6, 10, 13, 14)); + } else if (Group::kWidth == 8) { + ctrl_t group[] = {ctrl_t::kEmpty, CtrlT(1), ctrl_t::kEmpty, + ctrl_t::kDeleted, CtrlT(2), ctrl_t::kSentinel, + ctrl_t::kSentinel, CtrlT(1)}; + EXPECT_THAT(Group{group}.MaskNonFull(), ElementsAre(0, 2, 3, 5, 6)); + } else { + FAIL() << "No test coverage for Group::kWidth==" << Group::kWidth; + } +} + TEST(Group, MaskEmptyOrDeleted) { if (Group::kWidth == 16) { ctrl_t group[] = {ctrl_t::kEmpty, CtrlT(1), ctrl_t::kEmpty, CtrlT(3), @@ -323,7 +487,7 @@ TEST(Group, CountLeadingEmptyOrDeleted) { } } -template <class T, bool kTransferable = false> +template <class T, bool kTransferable = false, bool kSoo = false> struct ValuePolicy { using slot_type = T; using key_type = T; @@ -357,6 +521,13 @@ struct ValuePolicy { return absl::container_internal::DecomposeValue( std::forward<F>(f), std::forward<Args>(args)...); } + + template <class Hash> + static constexpr HashSlotFn get_hash_slot_fn() { + return nullptr; + } + + static constexpr bool soo_enabled() { return kSoo; } }; using IntPolicy = ValuePolicy<int64_t>; @@ -364,6 +535,44 @@ using Uint8Policy = ValuePolicy<uint8_t>; using TranferableIntPolicy = ValuePolicy<int64_t, /*kTransferable=*/true>; +// For testing SOO. +template <int N> +class SizedValue { + public: + SizedValue(int64_t v) { // NOLINT + vals_[0] = v; + } + SizedValue() : SizedValue(0) {} + SizedValue(const SizedValue&) = default; + SizedValue& operator=(const SizedValue&) = default; + + int64_t operator*() const { + // Suppress erroneous uninitialized memory errors on GCC. +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + return vals_[0]; +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + } + explicit operator int() const { return **this; } + explicit operator int64_t() const { return **this; } + + template <typename H> + friend H AbslHashValue(H h, SizedValue sv) { + return H::combine(std::move(h), *sv); + } + bool operator==(const SizedValue& rhs) const { return **this == *rhs; } + + private: + int64_t vals_[N / sizeof(int64_t)]; +}; +template <int N, bool kSoo> +using SizedValuePolicy = + ValuePolicy<SizedValue<N>, /*kTransferable=*/true, kSoo>; + class StringPolicy { template <class F, class K, class V, class = typename std::enable_if< @@ -420,6 +629,11 @@ class StringPolicy { return apply_impl(std::forward<F>(f), PairArgs(std::forward<Args>(args)...)); } + + template <class Hash> + static constexpr HashSlotFn get_hash_slot_fn() { + return nullptr; + } }; struct StringHash : absl::Hash<absl::string_view> { @@ -436,9 +650,9 @@ struct StringTable using Base::Base; }; -template <typename T, bool kTransferable = false> +template <typename T, bool kTransferable = false, bool kSoo = false> struct ValueTable - : raw_hash_set<ValuePolicy<T, kTransferable>, hash_default_hash<T>, + : raw_hash_set<ValuePolicy<T, kTransferable, kSoo>, hash_default_hash<T>, std::equal_to<T>, std::allocator<T>> { using Base = typename ValueTable::raw_hash_set; using Base::Base; @@ -449,6 +663,11 @@ using Uint8Table = ValueTable<uint8_t>; using TransferableIntTable = ValueTable<int64_t, /*kTransferable=*/true>; +constexpr size_t kNonSooSize = sizeof(HeapOrSoo) + 8; +static_assert(sizeof(SizedValue<kNonSooSize>) >= kNonSooSize, "too small"); +using NonSooIntTable = ValueTable<SizedValue<kNonSooSize>>; +using SooIntTable = ValueTable<int64_t, /*kTransferable=*/true, /*kSoo=*/true>; + template <typename T> struct CustomAlloc : std::allocator<T> { CustomAlloc() = default; @@ -498,6 +717,16 @@ struct FreezableAlloc : std::allocator<T> { bool* frozen; }; +template <int N> +struct FreezableSizedValueSooTable + : raw_hash_set<SizedValuePolicy<N, /*kSoo=*/true>, + container_internal::hash_default_hash<SizedValue<N>>, + std::equal_to<SizedValue<N>>, + FreezableAlloc<SizedValue<N>>> { + using Base = typename FreezableSizedValueSooTable::raw_hash_set; + using Base::Base; +}; + struct BadFastHash { template <class T> size_t operator()(const T&) const { @@ -568,20 +797,26 @@ TEST(Table, EmptyFunctorOptimization) { std::equal_to<absl::string_view>, std::allocator<int>>)); } -TEST(Table, Empty) { - IntTable t; +template <class TableType> +class SooTest : public testing::Test {}; + +using SooTableTypes = ::testing::Types<SooIntTable, NonSooIntTable>; +TYPED_TEST_SUITE(SooTest, SooTableTypes); + +TYPED_TEST(SooTest, Empty) { + TypeParam t; EXPECT_EQ(0, t.size()); EXPECT_TRUE(t.empty()); } -TEST(Table, LookupEmpty) { - IntTable t; +TYPED_TEST(SooTest, LookupEmpty) { + TypeParam t; auto it = t.find(0); EXPECT_TRUE(it == t.end()); } -TEST(Table, Insert1) { - IntTable t; +TYPED_TEST(SooTest, Insert1) { + TypeParam t; EXPECT_TRUE(t.find(0) == t.end()); auto res = t.emplace(0); EXPECT_TRUE(res.second); @@ -590,8 +825,8 @@ TEST(Table, Insert1) { EXPECT_THAT(*t.find(0), 0); } -TEST(Table, Insert2) { - IntTable t; +TYPED_TEST(SooTest, Insert2) { + TypeParam t; EXPECT_TRUE(t.find(0) == t.end()); auto res = t.emplace(0); EXPECT_TRUE(res.second); @@ -653,9 +888,9 @@ TEST(Table, InsertCollisionAndFindAfterDelete) { EXPECT_TRUE(t.empty()); } -TEST(Table, EraseInSmallTables) { +TYPED_TEST(SooTest, EraseInSmallTables) { for (int64_t size = 0; size < 64; ++size) { - IntTable t; + TypeParam t; for (int64_t i = 0; i < size; ++i) { t.insert(i); } @@ -670,8 +905,8 @@ TEST(Table, EraseInSmallTables) { } } -TEST(Table, InsertWithinCapacity) { - IntTable t; +TYPED_TEST(SooTest, InsertWithinCapacity) { + TypeParam t; t.reserve(10); const size_t original_capacity = t.capacity(); const auto addr = [&](int i) { @@ -704,9 +939,11 @@ TEST(Table, InsertWithinCapacity) { template <class TableType> class SmallTableResizeTest : public testing::Test {}; -TYPED_TEST_SUITE_P(SmallTableResizeTest); +using SmallTableTypes = + ::testing::Types<IntTable, TransferableIntTable, SooIntTable>; +TYPED_TEST_SUITE(SmallTableResizeTest, SmallTableTypes); -TYPED_TEST_P(SmallTableResizeTest, InsertIntoSmallTable) { +TYPED_TEST(SmallTableResizeTest, InsertIntoSmallTable) { TypeParam t; for (int i = 0; i < 32; ++i) { t.insert(i); @@ -718,11 +955,11 @@ TYPED_TEST_P(SmallTableResizeTest, InsertIntoSmallTable) { } } -TYPED_TEST_P(SmallTableResizeTest, ResizeGrowSmallTables) { - TypeParam t; +TYPED_TEST(SmallTableResizeTest, ResizeGrowSmallTables) { for (size_t source_size = 0; source_size < 32; ++source_size) { for (size_t target_size = source_size; target_size < 32; ++target_size) { for (bool rehash : {false, true}) { + TypeParam t; for (size_t i = 0; i < source_size; ++i) { t.insert(static_cast<int>(i)); } @@ -740,15 +977,21 @@ TYPED_TEST_P(SmallTableResizeTest, ResizeGrowSmallTables) { } } -TYPED_TEST_P(SmallTableResizeTest, ResizeReduceSmallTables) { - TypeParam t; +TYPED_TEST(SmallTableResizeTest, ResizeReduceSmallTables) { for (size_t source_size = 0; source_size < 32; ++source_size) { for (size_t target_size = 0; target_size <= source_size; ++target_size) { + TypeParam t; size_t inserted_count = std::min<size_t>(source_size, 5); for (size_t i = 0; i < inserted_count; ++i) { t.insert(static_cast<int>(i)); } + const size_t minimum_capacity = t.capacity(); + t.reserve(source_size); t.rehash(target_size); + if (target_size == 0) { + EXPECT_EQ(t.capacity(), minimum_capacity) + << "rehash(0) must resize to the minimum capacity"; + } for (size_t i = 0; i < inserted_count; ++i) { EXPECT_TRUE(t.find(static_cast<int>(i)) != t.end()); EXPECT_EQ(*t.find(static_cast<int>(i)), static_cast<int>(i)); @@ -757,12 +1000,6 @@ TYPED_TEST_P(SmallTableResizeTest, ResizeReduceSmallTables) { } } -REGISTER_TYPED_TEST_SUITE_P(SmallTableResizeTest, InsertIntoSmallTable, - ResizeGrowSmallTables, ResizeReduceSmallTables); -using SmallTableTypes = ::testing::Types<IntTable, TransferableIntTable>; -INSTANTIATE_TYPED_TEST_SUITE_P(InstanceSmallTableResizeTest, - SmallTableResizeTest, SmallTableTypes); - TEST(Table, LazyEmplace) { StringTable t; bool called = false; @@ -781,14 +1018,14 @@ TEST(Table, LazyEmplace) { EXPECT_THAT(*it, Pair("abc", "ABC")); } -TEST(Table, ContainsEmpty) { - IntTable t; +TYPED_TEST(SooTest, ContainsEmpty) { + TypeParam t; EXPECT_FALSE(t.contains(0)); } -TEST(Table, Contains1) { - IntTable t; +TYPED_TEST(SooTest, Contains1) { + TypeParam t; EXPECT_TRUE(t.insert(0).second); EXPECT_TRUE(t.contains(0)); @@ -798,8 +1035,8 @@ TEST(Table, Contains1) { EXPECT_FALSE(t.contains(0)); } -TEST(Table, Contains2) { - IntTable t; +TYPED_TEST(SooTest, Contains2) { + TypeParam t; EXPECT_TRUE(t.insert(0).second); EXPECT_TRUE(t.contains(0)); @@ -875,6 +1112,11 @@ struct DecomposePolicy { static auto apply(F&& f, const T& x) -> decltype(std::forward<F>(f)(x, x)) { return std::forward<F>(f)(x, x); } + + template <class Hash> + static constexpr HashSlotFn get_hash_slot_fn() { + return nullptr; + } }; template <typename Hash, typename Eq> @@ -1035,7 +1277,7 @@ size_t MaxDensitySize(size_t n) { } struct Modulo1000Hash { - size_t operator()(int x) const { return x % 1000; } + size_t operator()(int64_t x) const { return static_cast<size_t>(x) % 1000; } }; struct Modulo1000HashTable @@ -1091,8 +1333,8 @@ TEST(Table, RehashWithNoResize) { } } -TEST(Table, InsertEraseStressTest) { - IntTable t; +TYPED_TEST(SooTest, InsertEraseStressTest) { + TypeParam t; const size_t kMinElementCount = 250; std::deque<int> keys; size_t i = 0; @@ -1120,32 +1362,33 @@ TEST(Table, InsertOverloads) { Pair("DEF", "!!!"))); } -TEST(Table, LargeTable) { - IntTable t; +TYPED_TEST(SooTest, LargeTable) { + TypeParam t; for (int64_t i = 0; i != 100000; ++i) t.emplace(i << 40); - for (int64_t i = 0; i != 100000; ++i) ASSERT_EQ(i << 40, *t.find(i << 40)); + for (int64_t i = 0; i != 100000; ++i) + ASSERT_EQ(i << 40, static_cast<int64_t>(*t.find(i << 40))); } // Timeout if copy is quadratic as it was in Rust. -TEST(Table, EnsureNonQuadraticAsInRust) { +TYPED_TEST(SooTest, EnsureNonQuadraticAsInRust) { static const size_t kLargeSize = 1 << 15; - IntTable t; + TypeParam t; for (size_t i = 0; i != kLargeSize; ++i) { t.insert(i); } // If this is quadratic, the test will timeout. - IntTable t2; + TypeParam t2; for (const auto& entry : t) t2.insert(entry); } -TEST(Table, ClearBug) { +TYPED_TEST(SooTest, ClearBug) { if (SwisstableGenerationsEnabled()) { GTEST_SKIP() << "Generations being enabled causes extra rehashes."; } - IntTable t; + TypeParam t; constexpr size_t capacity = container_internal::Group::kWidth - 1; constexpr size_t max_size = capacity / 2 + 1; for (size_t i = 0; i < max_size; ++i) { @@ -1164,11 +1407,11 @@ TEST(Table, ClearBug) { // that they are probably still in the same group. This is not strictly // guaranteed. EXPECT_LT(static_cast<size_t>(std::abs(original - second)), - capacity * sizeof(IntTable::value_type)); + capacity * sizeof(typename TypeParam::value_type)); } -TEST(Table, Erase) { - IntTable t; +TYPED_TEST(SooTest, Erase) { + TypeParam t; EXPECT_TRUE(t.find(0) == t.end()); auto res = t.emplace(0); EXPECT_TRUE(res.second); @@ -1178,8 +1421,8 @@ TEST(Table, Erase) { EXPECT_TRUE(t.find(0) == t.end()); } -TEST(Table, EraseMaintainsValidIterator) { - IntTable t; +TYPED_TEST(SooTest, EraseMaintainsValidIterator) { + TypeParam t; const int kNumElements = 100; for (int i = 0; i < kNumElements; i++) { EXPECT_TRUE(t.emplace(i).second); @@ -1197,8 +1440,8 @@ TEST(Table, EraseMaintainsValidIterator) { EXPECT_EQ(num_erase_calls, kNumElements); } -TEST(Table, EraseBeginEnd) { - IntTable t; +TYPED_TEST(SooTest, EraseBeginEnd) { + TypeParam t; for (int i = 0; i < 10; ++i) t.insert(i); EXPECT_EQ(t.size(), 10); t.erase(t.begin(), t.end()); @@ -1597,8 +1840,29 @@ TEST(Table, EraseInsertProbing) { EXPECT_THAT(t, UnorderedElementsAre(1, 10, 3, 11, 12)); } -TEST(Table, Clear) { - IntTable t; +TEST(Table, GrowthInfoDeletedBit) { + BadTable t; + EXPECT_TRUE( + RawHashSetTestOnlyAccess::GetCommon(t).growth_info().HasNoDeleted()); + int64_t init_count = static_cast<int64_t>( + CapacityToGrowth(NormalizeCapacity(Group::kWidth + 1))); + for (int64_t i = 0; i < init_count; ++i) { + t.insert(i); + } + EXPECT_TRUE( + RawHashSetTestOnlyAccess::GetCommon(t).growth_info().HasNoDeleted()); + t.erase(0); + EXPECT_EQ(RawHashSetTestOnlyAccess::CountTombstones(t), 1); + EXPECT_FALSE( + RawHashSetTestOnlyAccess::GetCommon(t).growth_info().HasNoDeleted()); + t.rehash(0); + EXPECT_EQ(RawHashSetTestOnlyAccess::CountTombstones(t), 0); + EXPECT_TRUE( + RawHashSetTestOnlyAccess::GetCommon(t).growth_info().HasNoDeleted()); +} + +TYPED_TEST(SooTest, Clear) { + TypeParam t; EXPECT_TRUE(t.find(0) == t.end()); t.clear(); EXPECT_TRUE(t.find(0) == t.end()); @@ -1610,13 +1874,13 @@ TEST(Table, Clear) { EXPECT_TRUE(t.find(0) == t.end()); } -TEST(Table, Swap) { - IntTable t; +TYPED_TEST(SooTest, Swap) { + TypeParam t; EXPECT_TRUE(t.find(0) == t.end()); auto res = t.emplace(0); EXPECT_TRUE(res.second); EXPECT_EQ(1, t.size()); - IntTable u; + TypeParam u; t.swap(u); EXPECT_EQ(0, t.size()); EXPECT_EQ(1, u.size()); @@ -1624,8 +1888,8 @@ TEST(Table, Swap) { EXPECT_THAT(*u.find(0), 0); } -TEST(Table, Rehash) { - IntTable t; +TYPED_TEST(SooTest, Rehash) { + TypeParam t; EXPECT_TRUE(t.find(0) == t.end()); t.emplace(0); t.emplace(1); @@ -1636,8 +1900,8 @@ TEST(Table, Rehash) { EXPECT_THAT(*t.find(1), 1); } -TEST(Table, RehashDoesNotRehashWhenNotNecessary) { - IntTable t; +TYPED_TEST(SooTest, RehashDoesNotRehashWhenNotNecessary) { + TypeParam t; t.emplace(0); t.emplace(1); auto* p = &*t.find(0); @@ -1645,14 +1909,15 @@ TEST(Table, RehashDoesNotRehashWhenNotNecessary) { EXPECT_EQ(p, &*t.find(0)); } +// Following two tests use non-SOO table because they test for 0 capacity. TEST(Table, RehashZeroDoesNotAllocateOnEmptyTable) { - IntTable t; + NonSooIntTable t; t.rehash(0); EXPECT_EQ(0, t.bucket_count()); } TEST(Table, RehashZeroDeallocatesEmptyTable) { - IntTable t; + NonSooIntTable t; t.emplace(0); t.clear(); EXPECT_NE(0, t.bucket_count()); @@ -1660,8 +1925,8 @@ TEST(Table, RehashZeroDeallocatesEmptyTable) { EXPECT_EQ(0, t.bucket_count()); } -TEST(Table, RehashZeroForcesRehash) { - IntTable t; +TYPED_TEST(SooTest, RehashZeroForcesRehash) { + TypeParam t; t.emplace(0); t.emplace(1); auto* p = &*t.find(0); @@ -1677,27 +1942,61 @@ TEST(Table, ConstructFromInitList) { StringTable t = {P(), Q(), {}, {{}, {}}}; } -TEST(Table, CopyConstruct) { - IntTable t; +TYPED_TEST(SooTest, CopyConstruct) { + TypeParam t; t.emplace(0); EXPECT_EQ(1, t.size()); { - IntTable u(t); + TypeParam u(t); EXPECT_EQ(1, u.size()); EXPECT_THAT(*u.find(0), 0); } { - IntTable u{t}; + TypeParam u{t}; EXPECT_EQ(1, u.size()); EXPECT_THAT(*u.find(0), 0); } { - IntTable u = t; + TypeParam u = t; EXPECT_EQ(1, u.size()); EXPECT_THAT(*u.find(0), 0); } } +TYPED_TEST(SooTest, CopyDifferentSizes) { + TypeParam t; + + for (int i = 0; i < 100; ++i) { + t.emplace(i); + TypeParam c = t; + for (int j = 0; j <= i; ++j) { + ASSERT_TRUE(c.find(j) != c.end()) << "i=" << i << " j=" << j; + } + // Testing find miss to verify that table is not full. + ASSERT_TRUE(c.find(-1) == c.end()); + } +} + +TYPED_TEST(SooTest, CopyDifferentCapacities) { + for (int cap = 1; cap < 100; cap = cap * 2 + 1) { + TypeParam t; + t.reserve(static_cast<size_t>(cap)); + for (int i = 0; i <= cap; ++i) { + t.emplace(i); + if (i != cap && i % 5 != 0) { + continue; + } + TypeParam c = t; + for (int j = 0; j <= i; ++j) { + ASSERT_TRUE(c.find(j) != c.end()) + << "cap=" << cap << " i=" << i << " j=" << j; + } + // Testing find miss to verify that table is not full. + ASSERT_TRUE(c.find(-1) == c.end()); + } + } +} + TEST(Table, CopyConstructWithAlloc) { StringTable t; t.emplace("a", "b"); @@ -1827,8 +2126,8 @@ TEST(Table, Equality3) { EXPECT_NE(u, t); } -TEST(Table, NumDeletedRegression) { - IntTable t; +TYPED_TEST(SooTest, NumDeletedRegression) { + TypeParam t; t.emplace(0); t.erase(t.find(0)); // construct over a deleted slot. @@ -1836,8 +2135,8 @@ TEST(Table, NumDeletedRegression) { t.clear(); } -TEST(Table, FindFullDeletedRegression) { - IntTable t; +TYPED_TEST(SooTest, FindFullDeletedRegression) { + TypeParam t; for (int i = 0; i < 1000; ++i) { t.emplace(i); t.erase(t.find(i)); @@ -1845,17 +2144,20 @@ TEST(Table, FindFullDeletedRegression) { EXPECT_EQ(0, t.size()); } -TEST(Table, ReplacingDeletedSlotDoesNotRehash) { +TYPED_TEST(SooTest, ReplacingDeletedSlotDoesNotRehash) { + // We need to disable hashtablez to avoid issues related to SOO and sampling. + SetHashtablezEnabled(false); + size_t n; { // Compute n such that n is the maximum number of elements before rehash. - IntTable t; + TypeParam t; t.emplace(0); size_t c = t.bucket_count(); for (n = 1; c == t.bucket_count(); ++n) t.emplace(n); --n; } - IntTable t; + TypeParam t; t.rehash(n); const size_t c = t.bucket_count(); for (size_t i = 0; i != n; ++i) t.emplace(i); @@ -2106,8 +2408,8 @@ TEST(Nodes, ExtractInsert) { EXPECT_FALSE(node); // NOLINT(bugprone-use-after-move) } -TEST(Nodes, HintInsert) { - IntTable t = {1, 2, 3}; +TYPED_TEST(SooTest, HintInsert) { + TypeParam t = {1, 2, 3}; auto node = t.extract(1); EXPECT_THAT(t, UnorderedElementsAre(2, 3)); auto it = t.insert(t.begin(), std::move(node)); @@ -2126,14 +2428,18 @@ TEST(Nodes, HintInsert) { EXPECT_TRUE(node); // NOLINT(bugprone-use-after-move) } -IntTable MakeSimpleTable(size_t size) { - IntTable t; +template <typename T> +T MakeSimpleTable(size_t size) { + T t; while (t.size() < size) t.insert(t.size()); return t; } -std::vector<int> OrderOfIteration(const IntTable& t) { - return {t.begin(), t.end()}; +template <typename T> +std::vector<int> OrderOfIteration(const T& t) { + std::vector<int> res; + for (auto i : t) res.push_back(static_cast<int>(i)); + return res; } // These IterationOrderChanges tests depend on non-deterministic behavior. @@ -2142,15 +2448,15 @@ std::vector<int> OrderOfIteration(const IntTable& t) { // we are touching different memory pages to cause the ordering to change. // We also need to keep the old tables around to avoid getting the same memory // blocks over and over. -TEST(Table, IterationOrderChangesByInstance) { +TYPED_TEST(SooTest, IterationOrderChangesByInstance) { for (size_t size : {2, 6, 12, 20}) { - const auto reference_table = MakeSimpleTable(size); + const auto reference_table = MakeSimpleTable<TypeParam>(size); const auto reference = OrderOfIteration(reference_table); - std::vector<IntTable> tables; + std::vector<TypeParam> tables; bool found_difference = false; for (int i = 0; !found_difference && i < 5000; ++i) { - tables.push_back(MakeSimpleTable(size)); + tables.push_back(MakeSimpleTable<TypeParam>(size)); found_difference = OrderOfIteration(tables.back()) != reference; } if (!found_difference) { @@ -2161,27 +2467,44 @@ TEST(Table, IterationOrderChangesByInstance) { } } -TEST(Table, IterationOrderChangesOnRehash) { - std::vector<IntTable> garbage; - for (int i = 0; i < 5000; ++i) { - auto t = MakeSimpleTable(20); - const auto reference = OrderOfIteration(t); - // Force rehash to the same size. - t.rehash(0); - auto trial = OrderOfIteration(t); - if (trial != reference) { - // We are done. - return; +TYPED_TEST(SooTest, IterationOrderChangesOnRehash) { + // We test different sizes with many small numbers, because small table + // resize has a different codepath. + // Note: iteration order for size() <= 1 is always the same. + for (size_t size : std::vector<size_t>{2, 3, 6, 7, 12, 15, 20, 50}) { + for (size_t rehash_size : { + size_t{0}, // Force rehash is guaranteed. + size * 10 // Rehash to the larger capacity is guaranteed. + }) { + std::vector<TypeParam> garbage; + bool ok = false; + for (int i = 0; i < 5000; ++i) { + auto t = MakeSimpleTable<TypeParam>(size); + const auto reference = OrderOfIteration(t); + // Force rehash. + t.rehash(rehash_size); + auto trial = OrderOfIteration(t); + if (trial != reference) { + // We are done. + ok = true; + break; + } + garbage.push_back(std::move(t)); + } + EXPECT_TRUE(ok) + << "Iteration order remained the same across many attempts " << size + << "->" << rehash_size << "."; } - garbage.push_back(std::move(t)); } - FAIL() << "Iteration order remained the same across many attempts."; } // Verify that pointers are invalidated as soon as a second element is inserted. // This prevents dependency on pointer stability on small tables. -TEST(Table, UnstablePointers) { - IntTable table; +TYPED_TEST(SooTest, UnstablePointers) { + // We need to disable hashtablez to avoid issues related to SOO and sampling. + SetHashtablezEnabled(false); + + TypeParam table; const auto addr = [&](int i) { return reinterpret_cast<uintptr_t>(&*table.find(i)); @@ -2200,11 +2523,11 @@ TEST(TableDeathTest, InvalidIteratorAsserts) { if (!IsAssertEnabled() && !SwisstableGenerationsEnabled()) GTEST_SKIP() << "Assertions not enabled."; - IntTable t; + NonSooIntTable t; // Extra simple "regexp" as regexp support is highly varied across platforms. EXPECT_DEATH_IF_SUPPORTED(t.erase(t.end()), "erase.* called on end.. iterator."); - typename IntTable::iterator iter; + typename NonSooIntTable::iterator iter; EXPECT_DEATH_IF_SUPPORTED( ++iter, "operator.* called on default-constructed iterator."); t.insert(0); @@ -2218,6 +2541,22 @@ TEST(TableDeathTest, InvalidIteratorAsserts) { EXPECT_DEATH_IF_SUPPORTED(++iter, kErasedDeathMessage); } +TEST(TableDeathTest, InvalidIteratorAssertsSoo) { + if (!IsAssertEnabled() && !SwisstableGenerationsEnabled()) + GTEST_SKIP() << "Assertions not enabled."; + + SooIntTable t; + // Extra simple "regexp" as regexp support is highly varied across platforms. + EXPECT_DEATH_IF_SUPPORTED(t.erase(t.end()), + "erase.* called on end.. iterator."); + typename SooIntTable::iterator iter; + EXPECT_DEATH_IF_SUPPORTED( + ++iter, "operator.* called on default-constructed iterator."); + + // We can't detect the erased iterator case as invalid in SOO mode because + // the control is static constant. +} + // Invalid iterator use can trigger use-after-free in asan/hwasan, // use-of-uninitialized-value in msan, or invalidated iterator assertions. constexpr const char* kInvalidIteratorDeathMessage = @@ -2231,11 +2570,11 @@ constexpr bool kMsvc = true; constexpr bool kMsvc = false; #endif -TEST(TableDeathTest, IteratorInvalidAssertsEqualityOperator) { +TYPED_TEST(SooTest, IteratorInvalidAssertsEqualityOperator) { if (!IsAssertEnabled() && !SwisstableGenerationsEnabled()) GTEST_SKIP() << "Assertions not enabled."; - IntTable t; + TypeParam t; t.insert(1); t.insert(2); t.insert(3); @@ -2254,38 +2593,55 @@ TEST(TableDeathTest, IteratorInvalidAssertsEqualityOperator) { t.erase(iter2); EXPECT_DEATH_IF_SUPPORTED(void(iter1 == iter2), kErasedDeathMessage); - IntTable t1, t2; + TypeParam t1, t2; t1.insert(0); t2.insert(0); iter1 = t1.begin(); iter2 = t2.begin(); const char* const kContainerDiffDeathMessage = SwisstableGenerationsEnabled() - ? "Invalid iterator comparison.*iterators from different hashtables" + ? "Invalid iterator comparison.*iterators from different.* hashtables" : "Invalid iterator comparison.*may be from different " ".*containers.*config=asan"; EXPECT_DEATH_IF_SUPPORTED(void(iter1 == iter2), kContainerDiffDeathMessage); EXPECT_DEATH_IF_SUPPORTED(void(iter2 == iter1), kContainerDiffDeathMessage); +} + +TYPED_TEST(SooTest, IteratorInvalidAssertsEqualityOperatorRehash) { + if (!IsAssertEnabled() && !SwisstableGenerationsEnabled()) + GTEST_SKIP() << "Assertions not enabled."; + if (kMsvc) GTEST_SKIP() << "MSVC doesn't support | in regex."; +#ifdef ABSL_HAVE_THREAD_SANITIZER + GTEST_SKIP() << "ThreadSanitizer test runs fail on use-after-free even in " + "EXPECT_DEATH."; +#endif + + TypeParam t; + t.insert(0); + auto iter = t.begin(); - for (int i = 0; i < 10; ++i) t1.insert(i); - // There should have been a rehash in t1. - if (kMsvc) return; // MSVC doesn't support | in regex. + // Trigger a rehash in t. + for (int i = 0; i < 10; ++i) t.insert(i); - // NOTE(b/293887834): After rehashing, iterators will contain pointers to - // freed memory, which may be detected by ThreadSanitizer. const char* const kRehashedDeathMessage = SwisstableGenerationsEnabled() ? kInvalidIteratorDeathMessage - : "Invalid iterator comparison.*might have rehashed.*config=asan" - "|ThreadSanitizer: heap-use-after-free"; - EXPECT_DEATH_IF_SUPPORTED(void(iter1 == t1.begin()), kRehashedDeathMessage); + : "Invalid iterator comparison.*might have rehashed.*config=asan"; + EXPECT_DEATH_IF_SUPPORTED(void(iter == t.begin()), kRehashedDeathMessage); } #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) -TEST(RawHashSamplerTest, Sample) { +template <typename T> +class RawHashSamplerTest : public testing::Test {}; + +using RawHashSamplerTestTypes = ::testing::Types<SooIntTable, NonSooIntTable>; +TYPED_TEST_SUITE(RawHashSamplerTest, RawHashSamplerTestTypes); + +TYPED_TEST(RawHashSamplerTest, Sample) { + constexpr bool soo_enabled = std::is_same<SooIntTable, TypeParam>::value; // Enable the feature even if the prod default is off. SetHashtablezEnabled(true); - SetHashtablezSampleParameter(100); + SetHashtablezSampleParameter(100); // Sample ~1% of tables. auto& sampler = GlobalHashtablezSampler(); size_t start_size = 0; @@ -2295,7 +2651,7 @@ TEST(RawHashSamplerTest, Sample) { ++start_size; }); - std::vector<IntTable> tables; + std::vector<TypeParam> tables; for (int i = 0; i < 1000000; ++i) { tables.emplace_back(); @@ -2319,15 +2675,23 @@ TEST(RawHashSamplerTest, Sample) { absl::flat_hash_map<size_t, int> observed_checksums; absl::flat_hash_map<ssize_t, int> reservations; end_size += sampler.Iterate([&](const HashtablezInfo& info) { - if (preexisting_info.count(&info) == 0) { - observed_checksums[info.hashes_bitwise_xor.load( - std::memory_order_relaxed)]++; - reservations[info.max_reserve.load(std::memory_order_relaxed)]++; - } - EXPECT_EQ(info.inline_element_size, sizeof(int64_t)); ++end_size; + if (preexisting_info.contains(&info)) return; + observed_checksums[info.hashes_bitwise_xor.load( + std::memory_order_relaxed)]++; + reservations[info.max_reserve.load(std::memory_order_relaxed)]++; + EXPECT_EQ(info.inline_element_size, sizeof(typename TypeParam::value_type)); + EXPECT_EQ(info.key_size, sizeof(typename TypeParam::key_type)); + EXPECT_EQ(info.value_size, sizeof(typename TypeParam::value_type)); + + if (soo_enabled) { + EXPECT_EQ(info.soo_capacity, SooCapacity()); + } else { + EXPECT_EQ(info.soo_capacity, 0); + } }); + // Expect that we sampled at the requested sampling rate of ~1%. EXPECT_NEAR((end_size - start_size) / static_cast<double>(tables.size()), 0.01, 0.005); EXPECT_EQ(observed_checksums.size(), 5); @@ -2344,12 +2708,141 @@ TEST(RawHashSamplerTest, Sample) { << reservation; } } + +std::vector<const HashtablezInfo*> SampleSooMutation( + absl::FunctionRef<void(SooIntTable&)> mutate_table) { + // Enable the feature even if the prod default is off. + SetHashtablezEnabled(true); + SetHashtablezSampleParameter(100); // Sample ~1% of tables. + + auto& sampler = GlobalHashtablezSampler(); + size_t start_size = 0; + absl::flat_hash_set<const HashtablezInfo*> preexisting_info; + start_size += sampler.Iterate([&](const HashtablezInfo& info) { + preexisting_info.insert(&info); + ++start_size; + }); + + std::vector<SooIntTable> tables; + for (int i = 0; i < 1000000; ++i) { + tables.emplace_back(); + mutate_table(tables.back()); + } + size_t end_size = 0; + std::vector<const HashtablezInfo*> infos; + end_size += sampler.Iterate([&](const HashtablezInfo& info) { + ++end_size; + if (preexisting_info.contains(&info)) return; + infos.push_back(&info); + }); + + // Expect that we sampled at the requested sampling rate of ~1%. + EXPECT_NEAR((end_size - start_size) / static_cast<double>(tables.size()), + 0.01, 0.005); + return infos; +} + +TEST(RawHashSamplerTest, SooTableInsertToEmpty) { + if (SooIntTable().capacity() != SooCapacity()) { + CHECK_LT(sizeof(void*), 8) << "missing SOO coverage"; + GTEST_SKIP() << "not SOO on this platform"; + } + std::vector<const HashtablezInfo*> infos = + SampleSooMutation([](SooIntTable& t) { t.insert(1); }); + + for (const HashtablezInfo* info : infos) { + ASSERT_EQ(info->inline_element_size, + sizeof(typename SooIntTable::value_type)); + ASSERT_EQ(info->soo_capacity, SooCapacity()); + ASSERT_EQ(info->capacity, NextCapacity(SooCapacity())); + ASSERT_EQ(info->size, 1); + ASSERT_EQ(info->max_reserve, 0); + ASSERT_EQ(info->num_erases, 0); + ASSERT_EQ(info->max_probe_length, 0); + ASSERT_EQ(info->total_probe_length, 0); + } +} + +TEST(RawHashSamplerTest, SooTableReserveToEmpty) { + if (SooIntTable().capacity() != SooCapacity()) { + CHECK_LT(sizeof(void*), 8) << "missing SOO coverage"; + GTEST_SKIP() << "not SOO on this platform"; + } + std::vector<const HashtablezInfo*> infos = + SampleSooMutation([](SooIntTable& t) { t.reserve(100); }); + + for (const HashtablezInfo* info : infos) { + ASSERT_EQ(info->inline_element_size, + sizeof(typename SooIntTable::value_type)); + ASSERT_EQ(info->soo_capacity, SooCapacity()); + ASSERT_GE(info->capacity, 100); + ASSERT_EQ(info->size, 0); + ASSERT_EQ(info->max_reserve, 100); + ASSERT_EQ(info->num_erases, 0); + ASSERT_EQ(info->max_probe_length, 0); + ASSERT_EQ(info->total_probe_length, 0); + } +} + +// This tests that reserve on a full SOO table doesn't incorrectly result in new +// (over-)sampling. +TEST(RawHashSamplerTest, SooTableReserveToFullSoo) { + if (SooIntTable().capacity() != SooCapacity()) { + CHECK_LT(sizeof(void*), 8) << "missing SOO coverage"; + GTEST_SKIP() << "not SOO on this platform"; + } + std::vector<const HashtablezInfo*> infos = + SampleSooMutation([](SooIntTable& t) { + t.insert(1); + t.reserve(100); + }); + + for (const HashtablezInfo* info : infos) { + ASSERT_EQ(info->inline_element_size, + sizeof(typename SooIntTable::value_type)); + ASSERT_EQ(info->soo_capacity, SooCapacity()); + ASSERT_GE(info->capacity, 100); + ASSERT_EQ(info->size, 1); + ASSERT_EQ(info->max_reserve, 100); + ASSERT_EQ(info->num_erases, 0); + ASSERT_EQ(info->max_probe_length, 0); + ASSERT_EQ(info->total_probe_length, 0); + } +} + +// This tests that rehash(0) on a sampled table with size that fits in SOO +// doesn't incorrectly result in losing sampling. +TEST(RawHashSamplerTest, SooTableRehashShrinkWhenSizeFitsInSoo) { + if (SooIntTable().capacity() != SooCapacity()) { + CHECK_LT(sizeof(void*), 8) << "missing SOO coverage"; + GTEST_SKIP() << "not SOO on this platform"; + } + std::vector<const HashtablezInfo*> infos = + SampleSooMutation([](SooIntTable& t) { + t.reserve(100); + t.insert(1); + EXPECT_GE(t.capacity(), 100); + t.rehash(0); + }); + + for (const HashtablezInfo* info : infos) { + ASSERT_EQ(info->inline_element_size, + sizeof(typename SooIntTable::value_type)); + ASSERT_EQ(info->soo_capacity, SooCapacity()); + ASSERT_EQ(info->capacity, NextCapacity(SooCapacity())); + ASSERT_EQ(info->size, 1); + ASSERT_EQ(info->max_reserve, 100); + ASSERT_EQ(info->num_erases, 0); + ASSERT_EQ(info->max_probe_length, 0); + ASSERT_EQ(info->total_probe_length, 0); + } +} #endif // ABSL_INTERNAL_HASHTABLEZ_SAMPLE TEST(RawHashSamplerTest, DoNotSampleCustomAllocators) { // Enable the feature even if the prod default is off. SetHashtablezEnabled(true); - SetHashtablezSampleParameter(100); + SetHashtablezSampleParameter(100); // Sample ~1% of tables. auto& sampler = GlobalHashtablezSampler(); size_t start_size = 0; @@ -2371,9 +2864,10 @@ TEST(RawHashSamplerTest, DoNotSampleCustomAllocators) { template <class TableType> class SanitizerTest : public testing::Test {}; -TYPED_TEST_SUITE_P(SanitizerTest); +using SanitizerTableTypes = ::testing::Types<IntTable, TransferableIntTable>; +TYPED_TEST_SUITE(SanitizerTest, SanitizerTableTypes); -TYPED_TEST_P(SanitizerTest, PoisoningUnused) { +TYPED_TEST(SanitizerTest, PoisoningUnused) { TypeParam t; for (size_t reserve_size = 2; reserve_size < 1024; reserve_size = reserve_size * 3 / 2) { @@ -2391,14 +2885,10 @@ TYPED_TEST_P(SanitizerTest, PoisoningUnused) { } } -REGISTER_TYPED_TEST_SUITE_P(SanitizerTest, PoisoningUnused); -using SanitizerTableTypes = ::testing::Types<IntTable, TransferableIntTable>; -INSTANTIATE_TYPED_TEST_SUITE_P(InstanceSanitizerTest, SanitizerTest, - SanitizerTableTypes); - +// TODO(b/289225379): poison inline space when empty SOO. TEST(Sanitizer, PoisoningOnErase) { - IntTable t; - int64_t& v = *t.insert(0).first; + NonSooIntTable t; + auto& v = *t.insert(0).first; EXPECT_FALSE(__asan_address_is_poisoned(&v)); t.erase(0); @@ -2446,7 +2936,7 @@ TEST(Iterator, InvalidUseCrashesWithSanitizers) { if (!SwisstableGenerationsEnabled()) GTEST_SKIP() << "Generations disabled."; if (kMsvc) GTEST_SKIP() << "MSVC doesn't support | in regexp."; - IntTable t; + NonSooIntTable t; // Start with 1 element so that `it` is never an end iterator. t.insert(-1); for (int i = 0; i < 10; ++i) { @@ -2493,11 +2983,11 @@ TEST(Iterator, InvalidUseWithMoveCrashesWithSanitizers) { if (!SwisstableGenerationsEnabled()) GTEST_SKIP() << "Generations disabled."; if (kMsvc) GTEST_SKIP() << "MSVC doesn't support | in regexp."; - IntTable t1, t2; + NonSooIntTable t1, t2; t1.insert(1); auto it = t1.begin(); // ptr will become invalidated on rehash. - const int64_t* ptr = &*it; + const auto* ptr = &*it; (void)ptr; t2 = std::move(t1); @@ -2505,12 +2995,12 @@ TEST(Iterator, InvalidUseWithMoveCrashesWithSanitizers) { EXPECT_DEATH_IF_SUPPORTED(void(it == t2.begin()), kInvalidIteratorDeathMessage); #ifdef ABSL_HAVE_ADDRESS_SANITIZER - EXPECT_DEATH_IF_SUPPORTED(std::cout << *ptr, "heap-use-after-free"); + EXPECT_DEATH_IF_SUPPORTED(std::cout << **ptr, "heap-use-after-free"); #endif } -TEST(Table, ReservedGrowthUpdatesWhenTableDoesntGrow) { - IntTable t; +TYPED_TEST(SooTest, ReservedGrowthUpdatesWhenTableDoesntGrow) { + TypeParam t; for (int i = 0; i < 8; ++i) t.insert(i); // Want to insert twice without invalidating iterators so reserve. const size_t cap = t.capacity(); @@ -2524,6 +3014,213 @@ TEST(Table, ReservedGrowthUpdatesWhenTableDoesntGrow) { EXPECT_EQ(*it, 0); } +template <class TableType> +class InstanceTrackerTest : public testing::Test {}; + +using ::absl::test_internal::CopyableMovableInstance; +using ::absl::test_internal::InstanceTracker; + +struct InstanceTrackerHash { + size_t operator()(const CopyableMovableInstance& t) const { + return absl::HashOf(t.value()); + } +}; + +using InstanceTrackerTableTypes = ::testing::Types< + absl::node_hash_set<CopyableMovableInstance, InstanceTrackerHash>, + absl::flat_hash_set<CopyableMovableInstance, InstanceTrackerHash>>; +TYPED_TEST_SUITE(InstanceTrackerTest, InstanceTrackerTableTypes); + +TYPED_TEST(InstanceTrackerTest, EraseIfAll) { + using Table = TypeParam; + InstanceTracker tracker; + for (int size = 0; size < 100; ++size) { + Table t; + for (int i = 0; i < size; ++i) { + t.emplace(i); + } + absl::erase_if(t, [](const auto&) { return true; }); + ASSERT_EQ(t.size(), 0); + } + EXPECT_EQ(tracker.live_instances(), 0); +} + +TYPED_TEST(InstanceTrackerTest, EraseIfNone) { + using Table = TypeParam; + InstanceTracker tracker; + { + Table t; + for (size_t size = 0; size < 100; ++size) { + absl::erase_if(t, [](const auto&) { return false; }); + ASSERT_EQ(t.size(), size); + t.emplace(size); + } + } + EXPECT_EQ(tracker.live_instances(), 0); +} + +TYPED_TEST(InstanceTrackerTest, EraseIfPartial) { + using Table = TypeParam; + InstanceTracker tracker; + for (int mod : {0, 1}) { + for (int size = 0; size < 100; ++size) { + SCOPED_TRACE(absl::StrCat(mod, " ", size)); + Table t; + std::vector<CopyableMovableInstance> expected; + for (int i = 0; i < size; ++i) { + t.emplace(i); + if (i % 2 != mod) { + expected.emplace_back(i); + } + } + absl::erase_if(t, [mod](const auto& x) { return x.value() % 2 == mod; }); + ASSERT_THAT(t, testing::UnorderedElementsAreArray(expected)); + } + } + EXPECT_EQ(tracker.live_instances(), 0); +} + +TYPED_TEST(SooTest, EraseIfAll) { + auto pred = [](const auto&) { return true; }; + for (int size = 0; size < 100; ++size) { + TypeParam t; + for (int i = 0; i < size; ++i) t.insert(i); + absl::container_internal::EraseIf(pred, &t); + ASSERT_EQ(t.size(), 0); + } +} + +TYPED_TEST(SooTest, EraseIfNone) { + auto pred = [](const auto&) { return false; }; + TypeParam t; + for (size_t size = 0; size < 100; ++size) { + absl::container_internal::EraseIf(pred, &t); + ASSERT_EQ(t.size(), size); + t.insert(size); + } +} + +TYPED_TEST(SooTest, EraseIfPartial) { + for (int mod : {0, 1}) { + auto pred = [&](const auto& x) { + return static_cast<int64_t>(x) % 2 == mod; + }; + for (int size = 0; size < 100; ++size) { + SCOPED_TRACE(absl::StrCat(mod, " ", size)); + TypeParam t; + std::vector<int64_t> expected; + for (int i = 0; i < size; ++i) { + t.insert(i); + if (i % 2 != mod) { + expected.push_back(i); + } + } + absl::container_internal::EraseIf(pred, &t); + ASSERT_THAT(t, testing::UnorderedElementsAreArray(expected)); + } + } +} + +TYPED_TEST(SooTest, ForEach) { + TypeParam t; + std::vector<int64_t> expected; + for (int size = 0; size < 100; ++size) { + SCOPED_TRACE(size); + { + SCOPED_TRACE("mutable iteration"); + std::vector<int64_t> actual; + auto f = [&](auto& x) { actual.push_back(static_cast<int64_t>(x)); }; + absl::container_internal::ForEach(f, &t); + ASSERT_THAT(actual, testing::UnorderedElementsAreArray(expected)); + } + { + SCOPED_TRACE("const iteration"); + std::vector<int64_t> actual; + auto f = [&](auto& x) { + static_assert( + std::is_const<std::remove_reference_t<decltype(x)>>::value, + "no mutable values should be passed to const ForEach"); + actual.push_back(static_cast<int64_t>(x)); + }; + const auto& ct = t; + absl::container_internal::ForEach(f, &ct); + ASSERT_THAT(actual, testing::UnorderedElementsAreArray(expected)); + } + t.insert(size); + expected.push_back(size); + } +} + +TEST(Table, ForEachMutate) { + StringTable t; + using ValueType = StringTable::value_type; + std::vector<ValueType> expected; + for (int size = 0; size < 100; ++size) { + SCOPED_TRACE(size); + std::vector<ValueType> actual; + auto f = [&](ValueType& x) { + actual.push_back(x); + x.second += "a"; + }; + absl::container_internal::ForEach(f, &t); + ASSERT_THAT(actual, testing::UnorderedElementsAreArray(expected)); + for (ValueType& v : expected) { + v.second += "a"; + } + ASSERT_THAT(t, testing::UnorderedElementsAreArray(expected)); + t.emplace(std::to_string(size), std::to_string(size)); + expected.emplace_back(std::to_string(size), std::to_string(size)); + } +} + +TYPED_TEST(SooTest, EraseIfReentryDeath) { + if (!IsAssertEnabled()) GTEST_SKIP() << "Assertions not enabled."; + + auto erase_if_with_removal_reentrance = [](size_t reserve_size) { + TypeParam t; + t.reserve(reserve_size); + int64_t first_value = -1; + t.insert(1024); + t.insert(5078); + auto pred = [&](const auto& x) { + if (first_value == -1) { + first_value = static_cast<int64_t>(x); + return false; + } + // We erase on second call to `pred` to reduce the chance that assertion + // will happen in IterateOverFullSlots. + t.erase(first_value); + return true; + }; + absl::container_internal::EraseIf(pred, &t); + }; + // Removal will likely happen in a different group. + EXPECT_DEATH_IF_SUPPORTED(erase_if_with_removal_reentrance(1024 * 16), + "hash table was modified unexpectedly"); + // Removal will happen in the same group. + EXPECT_DEATH_IF_SUPPORTED( + erase_if_with_removal_reentrance(CapacityToGrowth(Group::kWidth - 1)), + "hash table was modified unexpectedly"); +} + +// This test is useful to test soo branch. +TYPED_TEST(SooTest, EraseIfReentrySingleElementDeath) { + if (!IsAssertEnabled()) GTEST_SKIP() << "Assertions not enabled."; + + auto erase_if_with_removal_reentrance = []() { + TypeParam t; + t.insert(1024); + auto pred = [&](const auto& x) { + // We erase ourselves in order to confuse the erase_if. + t.erase(static_cast<int64_t>(x)); + return false; + }; + absl::container_internal::EraseIf(pred, &t); + }; + EXPECT_DEATH_IF_SUPPORTED(erase_if_with_removal_reentrance(), + "hash table was modified unexpectedly"); +} + TEST(Table, EraseBeginEndResetsReservedGrowth) { bool frozen = false; BadHashFreezableIntTable t{FreezableAlloc<int64_t>(&frozen)}; @@ -2534,7 +3231,8 @@ TEST(Table, EraseBeginEndResetsReservedGrowth) { for (int i = 0; i < 10; ++i) { // Create a long run (hash function returns constant). for (int j = 0; j < 100; ++j) t.insert(j); - // Erase elements from the middle of the long run, which creates tombstones. + // Erase elements from the middle of the long run, which creates + // tombstones. for (int j = 30; j < 60; ++j) t.erase(j); EXPECT_EQ(t.size(), 70); EXPECT_EQ(t.capacity(), cap); @@ -2552,7 +3250,7 @@ TEST(Table, GenerationInfoResetsOnClear) { if (!SwisstableGenerationsEnabled()) GTEST_SKIP() << "Generations disabled."; if (kMsvc) GTEST_SKIP() << "MSVC doesn't support | in regexp."; - IntTable t; + NonSooIntTable t; for (int i = 0; i < 1000; ++i) t.insert(i); t.reserve(t.size() + 100); @@ -2570,24 +3268,24 @@ TEST(Table, InvalidReferenceUseCrashesWithSanitizers) { GTEST_SKIP() << "MSan fails to detect some of these rehashes."; #endif - IntTable t; + NonSooIntTable t; t.insert(0); // Rehashing is guaranteed on every insertion while capacity is less than // RehashProbabilityConstant(). - int64_t i = 0; + int i = 0; while (t.capacity() <= RehashProbabilityConstant()) { // ptr will become invalidated on rehash. - const int64_t* ptr = &*t.begin(); + const auto* ptr = &*t.begin(); t.insert(++i); - EXPECT_DEATH_IF_SUPPORTED(std::cout << *ptr, "use-after-free") << i; + EXPECT_DEATH_IF_SUPPORTED(std::cout << **ptr, "use-after-free") << i; } } TEST(Iterator, InvalidComparisonDifferentTables) { if (!SwisstableGenerationsEnabled()) GTEST_SKIP() << "Generations disabled."; - IntTable t1, t2; - IntTable::iterator default_constructed_iter; + NonSooIntTable t1, t2; + NonSooIntTable::iterator default_constructed_iter; // We randomly use one of N empty generations for generations from empty // hashtables. In general, we won't always detect when iterators from // different empty hashtables are compared, but in this test case, we @@ -2616,7 +3314,7 @@ using RawHashSetAlloc = raw_hash_set<IntPolicy, hash_default_hash<int64_t>, TEST(Table, AllocatorPropagation) { TestAllocPropagation<RawHashSetAlloc>(); } struct CountedHash { - size_t operator()(int value) const { + size_t operator()(int64_t value) const { ++count; return static_cast<size_t>(value); } @@ -2678,6 +3376,224 @@ TEST(Table, CountedHash) { } } +// IterateOverFullSlots doesn't support SOO. +TEST(Table, IterateOverFullSlotsEmpty) { + NonSooIntTable t; + auto fail_if_any = [](const ctrl_t*, auto* i) { + FAIL() << "expected no slots " << **i; + }; + container_internal::IterateOverFullSlots( + RawHashSetTestOnlyAccess::GetCommon(t), + RawHashSetTestOnlyAccess::GetSlots(t), fail_if_any); + for (size_t i = 0; i < 256; ++i) { + t.reserve(i); + container_internal::IterateOverFullSlots( + RawHashSetTestOnlyAccess::GetCommon(t), + RawHashSetTestOnlyAccess::GetSlots(t), fail_if_any); + } +} + +TEST(Table, IterateOverFullSlotsFull) { + NonSooIntTable t; + + std::vector<int64_t> expected_slots; + for (int64_t idx = 0; idx < 128; ++idx) { + t.insert(idx); + expected_slots.push_back(idx); + + std::vector<int64_t> slots; + container_internal::IterateOverFullSlots( + RawHashSetTestOnlyAccess::GetCommon(t), + RawHashSetTestOnlyAccess::GetSlots(t), + [&t, &slots](const ctrl_t* ctrl, auto* i) { + ptrdiff_t ctrl_offset = + ctrl - RawHashSetTestOnlyAccess::GetCommon(t).control(); + ptrdiff_t slot_offset = i - RawHashSetTestOnlyAccess::GetSlots(t); + ASSERT_EQ(ctrl_offset, slot_offset); + slots.push_back(**i); + }); + EXPECT_THAT(slots, testing::UnorderedElementsAreArray(expected_slots)); + } +} + +TEST(Table, IterateOverFullSlotsDeathOnRemoval) { + if (!IsAssertEnabled()) GTEST_SKIP() << "Assertions not enabled."; + + auto iterate_with_reentrant_removal = [](int64_t size, + int64_t reserve_size = -1) { + if (reserve_size == -1) reserve_size = size; + for (int64_t idx = 0; idx < size; ++idx) { + NonSooIntTable t; + t.reserve(static_cast<size_t>(reserve_size)); + for (int val = 0; val <= idx; ++val) { + t.insert(val); + } + + container_internal::IterateOverFullSlots( + RawHashSetTestOnlyAccess::GetCommon(t), + RawHashSetTestOnlyAccess::GetSlots(t), + [&t](const ctrl_t*, auto* i) { + int64_t value = **i; + // Erase the other element from 2*k and 2*k+1 pair. + t.erase(value ^ 1); + }); + } + }; + + EXPECT_DEATH_IF_SUPPORTED(iterate_with_reentrant_removal(128), + "hash table was modified unexpectedly"); + // Removal will likely happen in a different group. + EXPECT_DEATH_IF_SUPPORTED(iterate_with_reentrant_removal(14, 1024 * 16), + "hash table was modified unexpectedly"); + // Removal will happen in the same group. + EXPECT_DEATH_IF_SUPPORTED(iterate_with_reentrant_removal(static_cast<int64_t>( + CapacityToGrowth(Group::kWidth - 1))), + "hash table was modified unexpectedly"); +} + +TEST(Table, IterateOverFullSlotsDeathOnInsert) { + if (!IsAssertEnabled()) GTEST_SKIP() << "Assertions not enabled."; + + auto iterate_with_reentrant_insert = [](int64_t reserve_size, + int64_t size_divisor = 2) { + int64_t size = reserve_size / size_divisor; + for (int64_t idx = 1; idx <= size; ++idx) { + NonSooIntTable t; + t.reserve(static_cast<size_t>(reserve_size)); + for (int val = 1; val <= idx; ++val) { + t.insert(val); + } + + container_internal::IterateOverFullSlots( + RawHashSetTestOnlyAccess::GetCommon(t), + RawHashSetTestOnlyAccess::GetSlots(t), + [&t](const ctrl_t*, auto* i) { + int64_t value = **i; + t.insert(-value); + }); + } + }; + + EXPECT_DEATH_IF_SUPPORTED(iterate_with_reentrant_insert(128), + "hash table was modified unexpectedly"); + // Insert will likely happen in a different group. + EXPECT_DEATH_IF_SUPPORTED(iterate_with_reentrant_insert(1024 * 16, 1024 * 2), + "hash table was modified unexpectedly"); + // Insert will happen in the same group. + EXPECT_DEATH_IF_SUPPORTED(iterate_with_reentrant_insert(static_cast<int64_t>( + CapacityToGrowth(Group::kWidth - 1))), + "hash table was modified unexpectedly"); +} + +template <typename T> +class SooTable : public testing::Test {}; +using FreezableSooTableTypes = + ::testing::Types<FreezableSizedValueSooTable<8>, + FreezableSizedValueSooTable<16>>; +TYPED_TEST_SUITE(SooTable, FreezableSooTableTypes); + +TYPED_TEST(SooTable, Basic) { + bool frozen = true; + TypeParam t{FreezableAlloc<typename TypeParam::value_type>(&frozen)}; + if (t.capacity() != SooCapacity()) { + CHECK_LT(sizeof(void*), 8) << "missing SOO coverage"; + GTEST_SKIP() << "not SOO on this platform"; + } + + t.insert(0); + EXPECT_EQ(t.capacity(), 1); + auto it = t.find(0); + EXPECT_EQ(it, t.begin()); + ASSERT_NE(it, t.end()); + EXPECT_EQ(*it, 0); + EXPECT_EQ(++it, t.end()); + EXPECT_EQ(t.find(1), t.end()); + EXPECT_EQ(t.size(), 1); + + t.erase(0); + EXPECT_EQ(t.size(), 0); + t.insert(1); + it = t.find(1); + EXPECT_EQ(it, t.begin()); + ASSERT_NE(it, t.end()); + EXPECT_EQ(*it, 1); + + t.clear(); + EXPECT_EQ(t.size(), 0); +} + +TEST(Table, RehashToSooUnsampled) { + SooIntTable t; + if (t.capacity() != SooCapacity()) { + CHECK_LT(sizeof(void*), 8) << "missing SOO coverage"; + GTEST_SKIP() << "not SOO on this platform"; + } + + // We disable hashtablez sampling for this test to ensure that the table isn't + // sampled. When the table is sampled, it won't rehash down to SOO. + SetHashtablezEnabled(false); + + t.reserve(100); + t.insert(0); + EXPECT_EQ(*t.begin(), 0); + + t.rehash(0); // Rehash back down to SOO table. + + EXPECT_EQ(t.capacity(), SooCapacity()); + EXPECT_EQ(t.size(), 1); + EXPECT_EQ(*t.begin(), 0); + EXPECT_EQ(t.find(0), t.begin()); + EXPECT_EQ(t.find(1), t.end()); +} + +TEST(Table, ReserveToNonSoo) { + for (int reserve_capacity : {8, 100000}) { + SooIntTable t; + t.insert(0); + + t.reserve(reserve_capacity); + + EXPECT_EQ(t.find(0), t.begin()); + EXPECT_EQ(t.size(), 1); + EXPECT_EQ(*t.begin(), 0); + EXPECT_EQ(t.find(1), t.end()); + } +} + +struct InconsistentHashEqType { + InconsistentHashEqType(int v1, int v2) : v1(v1), v2(v2) {} + template <typename H> + friend H AbslHashValue(H h, InconsistentHashEqType t) { + return H::combine(std::move(h), t.v1); + } + bool operator==(InconsistentHashEqType t) const { return v2 == t.v2; } + int v1, v2; +}; + +TEST(Iterator, InconsistentHashEqFunctorsValidation) { + if (!IsAssertEnabled()) GTEST_SKIP() << "Assertions not enabled."; + + ValueTable<InconsistentHashEqType> t; + for (int i = 0; i < 10; ++i) t.insert({i, i}); + // We need to find/insert multiple times to guarantee that we get the + // assertion because it's possible for the hash to collide with the inserted + // element that has v2==0. In those cases, the new element won't be inserted. + auto find_conflicting_elems = [&] { + for (int i = 100; i < 20000; ++i) { + EXPECT_EQ(t.find({i, 0}), t.end()); + } + }; + EXPECT_DEATH_IF_SUPPORTED(find_conflicting_elems(), + "hash/eq functors are inconsistent."); + auto insert_conflicting_elems = [&] { + for (int i = 100; i < 20000; ++i) { + EXPECT_EQ(t.insert({i, 0}).second, false); + } + }; + EXPECT_DEATH_IF_SUPPORTED(insert_conflicting_elems(), + "hash/eq functors are inconsistent."); +} + } // namespace } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/node_hash_map.h b/absl/container/node_hash_map.h index a396de2e..5615e496 100644 --- a/absl/container/node_hash_map.h +++ b/absl/container/node_hash_map.h @@ -32,21 +32,25 @@ // migration, because it guarantees pointer stability. Consider migrating to // `node_hash_map` and perhaps converting to a more efficient `flat_hash_map` // upon further review. +// +// `node_hash_map` is not exception-safe. #ifndef ABSL_CONTAINER_NODE_HASH_MAP_H_ #define ABSL_CONTAINER_NODE_HASH_MAP_H_ -#include <tuple> +#include <cstddef> +#include <memory> #include <type_traits> #include <utility> #include "absl/algorithm/container.h" -#include "absl/base/macros.h" +#include "absl/base/attributes.h" +#include "absl/container/hash_container_defaults.h" #include "absl/container/internal/container_memory.h" -#include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export #include "absl/container/internal/node_slot_policy.h" #include "absl/container/internal/raw_hash_map.h" // IWYU pragma: export #include "absl/memory/memory.h" +#include "absl/meta/type_traits.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -66,7 +70,7 @@ class NodeHashMapPolicy; // // * Supports heterogeneous lookup, through `find()`, `operator[]()` and // `insert()`, provided that the map is provided a compatible heterogeneous -// hashing function and equality operator. +// hashing function and equality operator. See below for details. // * Contains a `capacity()` member function indicating the number of element // slots (open, deleted, and empty) within the hash map. // * Returns `void` from the `erase(iterator)` overload. @@ -82,6 +86,19 @@ class NodeHashMapPolicy; // libraries (e.g. .dll, .so) is unsupported due to way `absl::Hash` values may // be randomized across dynamically loaded libraries. // +// To achieve heterogeneous lookup for custom types either `Hash` and `Eq` type +// parameters can be used or `T` should have public inner types +// `absl_container_hash` and (optionally) `absl_container_eq`. In either case, +// `typename Hash::is_transparent` and `typename Eq::is_transparent` should be +// well-formed. Both types are basically functors: +// * `Hash` should support `size_t operator()(U val) const` that returns a hash +// for the given `val`. +// * `Eq` should support `bool operator()(U lhs, V rhs) const` that returns true +// if `lhs` is equal to `rhs`. +// +// In most cases `T` needs only to provide the `absl_container_hash`. In this +// case `std::equal_to<void>` will be used instead of `eq` part. +// // Example: // // // Create a node hash map of three strings (that map to strings) @@ -100,11 +117,10 @@ class NodeHashMapPolicy; // if (result != ducks.end()) { // std::cout << "Result: " << result->second << std::endl; // } -template <class Key, class Value, - class Hash = absl::container_internal::hash_default_hash<Key>, - class Eq = absl::container_internal::hash_default_eq<Key>, +template <class Key, class Value, class Hash = DefaultHashContainerHash<Key>, + class Eq = DefaultHashContainerEq<Key>, class Alloc = std::allocator<std::pair<const Key, Value>>> -class node_hash_map +class ABSL_INTERNAL_ATTRIBUTE_OWNER node_hash_map : public absl::container_internal::raw_hash_map< absl::container_internal::NodeHashMapPolicy<Key, Value>, Hash, Eq, Alloc> { @@ -544,6 +560,38 @@ typename node_hash_map<K, V, H, E, A>::size_type erase_if( namespace container_internal { +// c_for_each_fast(node_hash_map<>, Function) +// +// Container-based version of the <algorithm> `std::for_each()` function to +// apply a function to a container's elements. +// There is no guarantees on the order of the function calls. +// Erasure and/or insertion of elements in the function is not allowed. +template <typename K, typename V, typename H, typename E, typename A, + typename Function> +decay_t<Function> c_for_each_fast(const node_hash_map<K, V, H, E, A>& c, + Function&& f) { + container_internal::ForEach(f, &c); + return f; +} +template <typename K, typename V, typename H, typename E, typename A, + typename Function> +decay_t<Function> c_for_each_fast(node_hash_map<K, V, H, E, A>& c, + Function&& f) { + container_internal::ForEach(f, &c); + return f; +} +template <typename K, typename V, typename H, typename E, typename A, + typename Function> +decay_t<Function> c_for_each_fast(node_hash_map<K, V, H, E, A>&& c, + Function&& f) { + container_internal::ForEach(f, &c); + return f; +} + +} // namespace container_internal + +namespace container_internal { + template <class Key, class Value> class NodeHashMapPolicy : public absl::container_internal::node_slot_policy< @@ -590,6 +638,13 @@ class NodeHashMapPolicy static Value& value(value_type* elem) { return elem->second; } static const Value& value(const value_type* elem) { return elem->second; } + + template <class Hash> + static constexpr HashSlotFn get_hash_slot_fn() { + return memory_internal::IsLayoutCompatible<Key, Value>::value + ? &TypeErasedDerefAndApplyToSlotFn<Hash, Key> + : nullptr; + } }; } // namespace container_internal diff --git a/absl/container/node_hash_map_test.cc b/absl/container/node_hash_map_test.cc index 9bcf470c..4ad5d0dc 100644 --- a/absl/container/node_hash_map_test.cc +++ b/absl/container/node_hash_map_test.cc @@ -14,6 +14,18 @@ #include "absl/container/node_hash_map.h" +#include <cstddef> +#include <new> +#include <string> +#include <tuple> +#include <type_traits> +#include <utility> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/container/internal/hash_policy_testing.h" #include "absl/container/internal/tracked.h" #include "absl/container/internal/unordered_map_constructor_test.h" #include "absl/container/internal/unordered_map_lookup_test.h" @@ -29,6 +41,7 @@ using ::testing::Field; using ::testing::IsEmpty; using ::testing::Pair; using ::testing::UnorderedElementsAre; +using ::testing::UnorderedElementsAreArray; using MapTypes = ::testing::Types< absl::node_hash_map<int, int, StatefulTestingHash, StatefulTestingEqual, @@ -257,6 +270,58 @@ TEST(NodeHashMap, EraseIf) { } } +TEST(NodeHashMap, CForEach) { + node_hash_map<int, int> m; + std::vector<std::pair<int, int>> expected; + for (int i = 0; i < 100; ++i) { + { + SCOPED_TRACE("mutable object iteration"); + std::vector<std::pair<int, int>> v; + absl::container_internal::c_for_each_fast( + m, [&v](std::pair<const int, int>& p) { v.push_back(p); }); + EXPECT_THAT(v, UnorderedElementsAreArray(expected)); + } + { + SCOPED_TRACE("const object iteration"); + std::vector<std::pair<int, int>> v; + const node_hash_map<int, int>& cm = m; + absl::container_internal::c_for_each_fast( + cm, [&v](const std::pair<const int, int>& p) { v.push_back(p); }); + EXPECT_THAT(v, UnorderedElementsAreArray(expected)); + } + { + SCOPED_TRACE("const object iteration"); + std::vector<std::pair<int, int>> v; + absl::container_internal::c_for_each_fast( + node_hash_map<int, int>(m), + [&v](std::pair<const int, int>& p) { v.push_back(p); }); + EXPECT_THAT(v, UnorderedElementsAreArray(expected)); + } + m[i] = i; + expected.emplace_back(i, i); + } +} + +TEST(NodeHashMap, CForEachMutate) { + node_hash_map<int, int> s; + std::vector<std::pair<int, int>> expected; + for (int i = 0; i < 100; ++i) { + std::vector<std::pair<int, int>> v; + absl::container_internal::c_for_each_fast( + s, [&v](std::pair<const int, int>& p) { + v.push_back(p); + p.second++; + }); + EXPECT_THAT(v, UnorderedElementsAreArray(expected)); + for (auto& p : expected) { + p.second++; + } + EXPECT_THAT(s, UnorderedElementsAreArray(expected)); + s[i] = i; + expected.emplace_back(i, i); + } +} + // This test requires std::launder for mutable key access in node handles. #if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606 TEST(NodeHashMap, NodeHandleMutableKeyAccess) { diff --git a/absl/container/node_hash_set.h b/absl/container/node_hash_set.h index 421ff460..53435ae6 100644 --- a/absl/container/node_hash_set.h +++ b/absl/container/node_hash_set.h @@ -31,18 +31,24 @@ // `node_hash_set` should be an easy migration. Consider migrating to // `node_hash_set` and perhaps converting to a more efficient `flat_hash_set` // upon further review. +// +// `node_hash_set` is not exception-safe. #ifndef ABSL_CONTAINER_NODE_HASH_SET_H_ #define ABSL_CONTAINER_NODE_HASH_SET_H_ +#include <cstddef> +#include <memory> #include <type_traits> #include "absl/algorithm/container.h" -#include "absl/base/macros.h" -#include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export +#include "absl/base/attributes.h" +#include "absl/container/hash_container_defaults.h" +#include "absl/container/internal/container_memory.h" #include "absl/container/internal/node_slot_policy.h" #include "absl/container/internal/raw_hash_set.h" // IWYU pragma: export #include "absl/memory/memory.h" +#include "absl/meta/type_traits.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -62,7 +68,7 @@ struct NodeHashSetPolicy; // // * Supports heterogeneous lookup, through `find()`, `operator[]()` and // `insert()`, provided that the set is provided a compatible heterogeneous -// hashing function and equality operator. +// hashing function and equality operator. See below for details. // * Contains a `capacity()` member function indicating the number of element // slots (open, deleted, and empty) within the hash set. // * Returns `void` from the `erase(iterator)` overload. @@ -78,6 +84,19 @@ struct NodeHashSetPolicy; // libraries (e.g. .dll, .so) is unsupported due to way `absl::Hash` values may // be randomized across dynamically loaded libraries. // +// To achieve heterogeneous lookup for custom types either `Hash` and `Eq` type +// parameters can be used or `T` should have public inner types +// `absl_container_hash` and (optionally) `absl_container_eq`. In either case, +// `typename Hash::is_transparent` and `typename Eq::is_transparent` should be +// well-formed. Both types are basically functors: +// * `Hash` should support `size_t operator()(U val) const` that returns a hash +// for the given `val`. +// * `Eq` should support `bool operator()(U lhs, V rhs) const` that returns true +// if `lhs` is equal to `rhs`. +// +// In most cases `T` needs only to provide the `absl_container_hash`. In this +// case `std::equal_to<void>` will be used instead of `eq` part. +// // Example: // // // Create a node hash set of three strings @@ -94,10 +113,9 @@ struct NodeHashSetPolicy; // if (ducks.contains("dewey")) { // std::cout << "We found dewey!" << std::endl; // } -template <class T, class Hash = absl::container_internal::hash_default_hash<T>, - class Eq = absl::container_internal::hash_default_eq<T>, - class Alloc = std::allocator<T>> -class node_hash_set +template <class T, class Hash = DefaultHashContainerHash<T>, + class Eq = DefaultHashContainerEq<T>, class Alloc = std::allocator<T>> +class ABSL_INTERNAL_ATTRIBUTE_OWNER node_hash_set : public absl::container_internal::raw_hash_set< absl::container_internal::NodeHashSetPolicy<T>, Hash, Eq, Alloc> { using Base = typename node_hash_set::raw_hash_set; @@ -451,6 +469,33 @@ typename node_hash_set<T, H, E, A>::size_type erase_if( namespace container_internal { +// c_for_each_fast(node_hash_set<>, Function) +// +// Container-based version of the <algorithm> `std::for_each()` function to +// apply a function to a container's elements. +// There is no guarantees on the order of the function calls. +// Erasure and/or insertion of elements in the function is not allowed. +template <typename T, typename H, typename E, typename A, typename Function> +decay_t<Function> c_for_each_fast(const node_hash_set<T, H, E, A>& c, + Function&& f) { + container_internal::ForEach(f, &c); + return f; +} +template <typename T, typename H, typename E, typename A, typename Function> +decay_t<Function> c_for_each_fast(node_hash_set<T, H, E, A>& c, Function&& f) { + container_internal::ForEach(f, &c); + return f; +} +template <typename T, typename H, typename E, typename A, typename Function> +decay_t<Function> c_for_each_fast(node_hash_set<T, H, E, A>&& c, Function&& f) { + container_internal::ForEach(f, &c); + return f; +} + +} // namespace container_internal + +namespace container_internal { + template <class T> struct NodeHashSetPolicy : absl::container_internal::node_slot_policy<T&, NodeHashSetPolicy<T>> { @@ -487,6 +532,11 @@ struct NodeHashSetPolicy } static size_t element_space_used(const T*) { return sizeof(T); } + + template <class Hash> + static constexpr HashSlotFn get_hash_slot_fn() { + return &TypeErasedDerefAndApplyToSlotFn<Hash, T>; + } }; } // namespace container_internal diff --git a/absl/container/node_hash_set_test.cc b/absl/container/node_hash_set_test.cc index 98a8dbdd..e616ac1e 100644 --- a/absl/container/node_hash_set_test.cc +++ b/absl/container/node_hash_set_test.cc @@ -14,10 +14,22 @@ #include "absl/container/node_hash_set.h" +#include <cstddef> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/container/internal/hash_generator_testing.h" +#include "absl/container/internal/hash_policy_testing.h" #include "absl/container/internal/unordered_set_constructor_test.h" #include "absl/container/internal/unordered_set_lookup_test.h" #include "absl/container/internal/unordered_set_members_test.h" #include "absl/container/internal/unordered_set_modifiers_test.h" +#include "absl/memory/memory.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -28,6 +40,7 @@ using ::absl::container_internal::hash_internal::EnumClass; using ::testing::IsEmpty; using ::testing::Pointee; using ::testing::UnorderedElementsAre; +using ::testing::UnorderedElementsAreArray; using SetTypes = ::testing::Types< node_hash_set<int, StatefulTestingHash, StatefulTestingEqual, Alloc<int>>, @@ -137,6 +150,39 @@ TEST(NodeHashSet, EraseIf) { } } +TEST(NodeHashSet, CForEach) { + using ValueType = std::pair<int, int>; + node_hash_set<ValueType> s; + std::vector<ValueType> expected; + for (int i = 0; i < 100; ++i) { + { + SCOPED_TRACE("mutable object iteration"); + std::vector<ValueType> v; + absl::container_internal::c_for_each_fast( + s, [&v](const ValueType& p) { v.push_back(p); }); + ASSERT_THAT(v, UnorderedElementsAreArray(expected)); + } + { + SCOPED_TRACE("const object iteration"); + std::vector<ValueType> v; + const node_hash_set<ValueType>& cs = s; + absl::container_internal::c_for_each_fast( + cs, [&v](const ValueType& p) { v.push_back(p); }); + ASSERT_THAT(v, UnorderedElementsAreArray(expected)); + } + { + SCOPED_TRACE("temporary object iteration"); + std::vector<ValueType> v; + absl::container_internal::c_for_each_fast( + node_hash_set<ValueType>(s), + [&v](const ValueType& p) { v.push_back(p); }); + ASSERT_THAT(v, UnorderedElementsAreArray(expected)); + } + s.emplace(i, i); + expected.emplace_back(i, i); + } +} + } // namespace } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/sample_element_size_test.cc b/absl/container/sample_element_size_test.cc index b23626b4..22470b49 100644 --- a/absl/container/sample_element_size_test.cc +++ b/absl/container/sample_element_size_test.cc @@ -12,6 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include <cstddef> +#include <unordered_set> +#include <utility> +#include <vector> + #include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/container/flat_hash_map.h" @@ -38,15 +43,16 @@ void TestInlineElementSize( // set cannot be flat_hash_set, however, since that would introduce a mutex // deadlock. std::unordered_set<const HashtablezInfo*>& preexisting_info, // NOLINT - std::vector<Table>& tables, const typename Table::value_type& elt, + std::vector<Table>& tables, + const std::vector<typename Table::value_type>& values, size_t expected_element_size) { for (int i = 0; i < 10; ++i) { // We create a new table and must store it somewhere so that when we store // a pointer to the resulting `HashtablezInfo` into `preexisting_info` // that we aren't storing a dangling pointer. tables.emplace_back(); - // We must insert an element to get a hashtablez to instantiate. - tables.back().insert(elt); + // We must insert elements to get a hashtablez to instantiate. + tables.back().insert(values.begin(), values.end()); } size_t new_count = 0; sampler.Iterate([&](const HashtablezInfo& info) { @@ -82,6 +88,9 @@ TEST(FlatHashMap, SampleElementSize) { std::vector<flat_hash_set<bigstruct>> flat_set_tables; std::vector<node_hash_map<int, bigstruct>> node_map_tables; std::vector<node_hash_set<bigstruct>> node_set_tables; + std::vector<bigstruct> set_values = {bigstruct{{0}}, bigstruct{{1}}}; + std::vector<std::pair<const int, bigstruct>> map_values = {{0, bigstruct{}}, + {1, bigstruct{}}}; // It takes thousands of new tables after changing the sampling parameters // before you actually get some instrumentation. And if you must actually @@ -97,14 +106,14 @@ TEST(FlatHashMap, SampleElementSize) { std::unordered_set<const HashtablezInfo*> preexisting_info; // NOLINT sampler.Iterate( [&](const HashtablezInfo& info) { preexisting_info.insert(&info); }); - TestInlineElementSize(sampler, preexisting_info, flat_map_tables, - {0, bigstruct{}}, sizeof(int) + sizeof(bigstruct)); - TestInlineElementSize(sampler, preexisting_info, node_map_tables, - {0, bigstruct{}}, sizeof(void*)); - TestInlineElementSize(sampler, preexisting_info, flat_set_tables, // - bigstruct{}, sizeof(bigstruct)); - TestInlineElementSize(sampler, preexisting_info, node_set_tables, // - bigstruct{}, sizeof(void*)); + TestInlineElementSize(sampler, preexisting_info, flat_map_tables, map_values, + sizeof(int) + sizeof(bigstruct)); + TestInlineElementSize(sampler, preexisting_info, node_map_tables, map_values, + sizeof(void*)); + TestInlineElementSize(sampler, preexisting_info, flat_set_tables, set_values, + sizeof(bigstruct)); + TestInlineElementSize(sampler, preexisting_info, node_set_tables, set_values, + sizeof(void*)); #endif } |