From b03cda5ec9b1f5aa3e2d0e5db4e436a11ed193bc Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Thu, 18 Jan 2024 09:55:51 -0800 Subject: Added benchmarks for smaller size copy constructors. PiperOrigin-RevId: 599538858 Change-Id: I9e92f4c9cfef1bfe6f8f925efe0ede3f309b6bf4 --- absl/container/internal/raw_hash_set_benchmark.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set_benchmark.cc b/absl/container/internal/raw_hash_set_benchmark.cc index 88b07373..05a06427 100644 --- a/absl/container/internal/raw_hash_set_benchmark.cc +++ b/absl/container/internal/raw_hash_set_benchmark.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -294,7 +295,7 @@ void BM_CopyCtorSparseInt(benchmark::State& state) { benchmark::DoNotOptimize(t2); } } -BENCHMARK(BM_CopyCtorSparseInt)->Range(128, 4096); +BENCHMARK(BM_CopyCtorSparseInt)->Range(1, 4096); void BM_CopyCtorInt(benchmark::State& state) { std::random_device rd; @@ -312,7 +313,7 @@ void BM_CopyCtorInt(benchmark::State& state) { benchmark::DoNotOptimize(t2); } } -BENCHMARK(BM_CopyCtorInt)->Range(128, 4096); +BENCHMARK(BM_CopyCtorInt)->Range(0, 4096); void BM_CopyCtorString(benchmark::State& state) { std::random_device rd; @@ -330,7 +331,7 @@ void BM_CopyCtorString(benchmark::State& state) { benchmark::DoNotOptimize(t2); } } -BENCHMARK(BM_CopyCtorString)->Range(128, 4096); +BENCHMARK(BM_CopyCtorString)->Range(0, 4096); void BM_CopyAssign(benchmark::State& state) { std::random_device rd; -- cgit v1.2.3 From f7d2b13ef2466a5a3b7bcc535c49d3962a5c89f9 Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Mon, 22 Jan 2024 09:09:23 -0800 Subject: Remove code pieces for no longer supported GCC versions. The minimum supported version today is GCC 7 (`__GNUC__ >= 7`). PiperOrigin-RevId: 600475215 Change-Id: I1aa46384f1e75f268649a48dbe2b42f3475bb07f --- absl/base/config.h | 8 ++------ absl/container/internal/compressed_tuple_test.cc | 2 -- absl/container/internal/hash_policy_testing.h | 3 +-- absl/flags/flag_test.cc | 8 -------- absl/types/optional_test.cc | 22 +--------------------- 5 files changed, 4 insertions(+), 39 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/base/config.h b/absl/base/config.h index 0fb66927..5fa9f0ef 100644 --- a/absl/base/config.h +++ b/absl/base/config.h @@ -379,9 +379,7 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || #define ABSL_HAVE_EXCEPTIONS 1 #endif // defined(__EXCEPTIONS) && ABSL_HAVE_FEATURE(cxx_exceptions) // Handle remaining special cases and default to exceptions being supported. -#elif !(defined(__GNUC__) && (__GNUC__ < 5) && !defined(__EXCEPTIONS)) && \ - !(ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(5, 0) && \ - !defined(__cpp_exceptions)) && \ +#elif !(defined(__GNUC__) && !defined(__cpp_exceptions)) && \ !(defined(_MSC_VER) && !defined(_CPPUNWIND)) #define ABSL_HAVE_EXCEPTIONS 1 #endif @@ -902,9 +900,7 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || #error ABSL_INTERNAL_HAS_CXA_DEMANGLE cannot be directly set #elif defined(OS_ANDROID) && (defined(__i386__) || defined(__x86_64__)) #define ABSL_INTERNAL_HAS_CXA_DEMANGLE 0 -#elif defined(__GNUC__) && defined(__GNUC_MINOR__) && \ - (__GNUC__ >= 4 || (__GNUC__ >= 3 && __GNUC_MINOR__ >= 4)) && \ - !defined(__mips__) +#elif defined(__GNUC__) && !defined(__mips__) #define ABSL_INTERNAL_HAS_CXA_DEMANGLE 1 #elif defined(__clang__) && !defined(_MSC_VER) #define ABSL_INTERNAL_HAS_CXA_DEMANGLE 1 diff --git a/absl/container/internal/compressed_tuple_test.cc b/absl/container/internal/compressed_tuple_test.cc index 74111f97..da07baab 100644 --- a/absl/container/internal/compressed_tuple_test.cc +++ b/absl/container/internal/compressed_tuple_test.cc @@ -358,7 +358,6 @@ TEST(CompressedTupleTest, Constexpr) { EXPECT_EQ(x2, 5); EXPECT_EQ(x3, CallType::kConstRef); -#if !defined(__GNUC__) || defined(__clang__) || __GNUC__ > 4 constexpr CompressedTuple, TrivialStruct, int> trivial = {}; constexpr CallType trivial0 = trivial.get<0>().value(); constexpr int trivial1 = trivial.get<1>().value(); @@ -367,7 +366,6 @@ TEST(CompressedTupleTest, Constexpr) { EXPECT_EQ(trivial0, CallType::kConstRef); EXPECT_EQ(trivial1, 0); EXPECT_EQ(trivial2, 0); -#endif constexpr CompressedTuple, NonTrivialStruct, absl::optional> non_trivial = {}; diff --git a/absl/container/internal/hash_policy_testing.h b/absl/container/internal/hash_policy_testing.h index 01c40d2e..66bb12ec 100644 --- a/absl/container/internal/hash_policy_testing.h +++ b/absl/container/internal/hash_policy_testing.h @@ -174,8 +174,7 @@ ABSL_NAMESPACE_END // From GCC-4.9 Changelog: (src: https://gcc.gnu.org/gcc-4.9/changes.html) // "the unordered associative containers in and // meet the allocator-aware container requirements;" -#if (defined(__GLIBCXX__) && __GLIBCXX__ <= 20140425 ) || \ -( __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 9 )) +#if defined(__GLIBCXX__) && __GLIBCXX__ <= 20140425 #define ABSL_UNORDERED_SUPPORTS_ALLOC_CTORS 0 #else #define ABSL_UNORDERED_SUPPORTS_ALLOC_CTORS 1 diff --git a/absl/flags/flag_test.cc b/absl/flags/flag_test.cc index 8d14ba8d..53ad4635 100644 --- a/absl/flags/flag_test.cc +++ b/absl/flags/flag_test.cc @@ -1044,13 +1044,7 @@ TEST_F(FlagTest, MacroWithinAbslFlag) { // -------------------------------------------------------------------- -#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ <= 5 -#define ABSL_SKIP_OPTIONAL_BOOL_TEST_DUE_TO_GCC_BUG -#endif - -#ifndef ABSL_SKIP_OPTIONAL_BOOL_TEST_DUE_TO_GCC_BUG ABSL_FLAG(absl::optional, optional_bool, absl::nullopt, "help"); -#endif ABSL_FLAG(absl::optional, optional_int, {}, "help"); ABSL_FLAG(absl::optional, optional_double, 9.3, "help"); ABSL_FLAG(absl::optional, optional_string, absl::nullopt, "help"); @@ -1064,7 +1058,6 @@ ABSL_FLAG(std::optional, std_optional_int64, std::nullopt, "help"); namespace { -#ifndef ABSL_SKIP_OPTIONAL_BOOL_TEST_DUE_TO_GCC_BUG TEST_F(FlagTest, TestOptionalBool) { EXPECT_FALSE(absl::GetFlag(FLAGS_optional_bool).has_value()); EXPECT_EQ(absl::GetFlag(FLAGS_optional_bool), absl::nullopt); @@ -1083,7 +1076,6 @@ TEST_F(FlagTest, TestOptionalBool) { } // -------------------------------------------------------------------- -#endif TEST_F(FlagTest, TestOptionalInt) { EXPECT_FALSE(absl::GetFlag(FLAGS_optional_int).has_value()); diff --git a/absl/types/optional_test.cc b/absl/types/optional_test.cc index a4daa799..115e20ce 100644 --- a/absl/types/optional_test.cc +++ b/absl/types/optional_test.cc @@ -982,18 +982,6 @@ TEST(optionalTest, PointerStuff) { static_assert((*opt1).x == ConstexprType::kCtorInt, ""); } -// gcc has a bug pre 4.9.1 where it doesn't do correct overload resolution -// when overloads are const-qualified and *this is an raluve. -// Skip that test to make the build green again when using the old compiler. -// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59296 is fixed in 4.9.1. -#if defined(__GNUC__) && !defined(__clang__) -#define GCC_VERSION \ - (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) -#if GCC_VERSION < 40901 -#define ABSL_SKIP_OVERLOAD_TEST_DUE_TO_GCC_BUG -#endif -#endif - TEST(optionalTest, Value) { using O = absl::optional; using CO = const absl::optional; @@ -1006,16 +994,12 @@ TEST(optionalTest, Value) { EXPECT_EQ("lvalue_c", lvalue_c.value()); EXPECT_EQ("xvalue", O(absl::in_place, "xvalue").value()); EXPECT_EQ("xvalue_c", OC(absl::in_place, "xvalue_c").value()); -#ifndef ABSL_SKIP_OVERLOAD_TEST_DUE_TO_GCC_BUG EXPECT_EQ("cxvalue", CO(absl::in_place, "cxvalue").value()); -#endif EXPECT_EQ("&", TypeQuals(lvalue.value())); EXPECT_EQ("c&", TypeQuals(clvalue.value())); EXPECT_EQ("c&", TypeQuals(lvalue_c.value())); EXPECT_EQ("&&", TypeQuals(O(absl::in_place, "xvalue").value())); -#ifndef ABSL_SKIP_OVERLOAD_TEST_DUE_TO_GCC_BUG EXPECT_EQ("c&&", TypeQuals(CO(absl::in_place, "cxvalue").value())); -#endif EXPECT_EQ("c&&", TypeQuals(OC(absl::in_place, "xvalue_c").value())); #if !defined(ABSL_VOLATILE_RETURN_TYPES_DEPRECATED) @@ -1039,7 +1023,7 @@ TEST(optionalTest, Value) { // test constexpr value() constexpr absl::optional o1(1); static_assert(1 == o1.value(), ""); // const & -#if !defined(_MSC_VER) && !defined(ABSL_SKIP_OVERLOAD_TEST_DUE_TO_GCC_BUG) +#ifndef _MSC_VER using COI = const absl::optional; static_assert(2 == COI(2).value(), ""); // const && #endif @@ -1057,15 +1041,11 @@ TEST(optionalTest, DerefOperator) { EXPECT_EQ("lvalue_c", *lvalue_c); EXPECT_EQ("xvalue", *O(absl::in_place, "xvalue")); EXPECT_EQ("xvalue_c", *OC(absl::in_place, "xvalue_c")); -#ifndef ABSL_SKIP_OVERLOAD_TEST_DUE_TO_GCC_BUG EXPECT_EQ("cxvalue", *CO(absl::in_place, "cxvalue")); -#endif EXPECT_EQ("&", TypeQuals(*lvalue)); EXPECT_EQ("c&", TypeQuals(*clvalue)); EXPECT_EQ("&&", TypeQuals(*O(absl::in_place, "xvalue"))); -#ifndef ABSL_SKIP_OVERLOAD_TEST_DUE_TO_GCC_BUG EXPECT_EQ("c&&", TypeQuals(*CO(absl::in_place, "cxvalue"))); -#endif EXPECT_EQ("c&&", TypeQuals(*OC(absl::in_place, "xvalue_c"))); #if !defined(ABSL_VOLATILE_RETURN_TYPES_DEPRECATED) -- cgit v1.2.3 From 27f15a052ba54c984faa1eb6044de59b237e6f37 Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Wed, 24 Jan 2024 10:08:22 -0800 Subject: Use absl::NoDestructor for global HashtablezSampler. PiperOrigin-RevId: 601156448 Change-Id: Id6d19bda7eb3acee11cfab3a95e611996d6ef4cc --- absl/container/BUILD.bazel | 1 + absl/container/CMakeLists.txt | 1 + absl/container/internal/hashtablez_sampler.cc | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) (limited to 'absl/container/internal') diff --git a/absl/container/BUILD.bazel b/absl/container/BUILD.bazel index 0ba2fa76..fda866a5 100644 --- a/absl/container/BUILD.bazel +++ b/absl/container/BUILD.bazel @@ -563,6 +563,7 @@ cc_library( "//absl/base", "//absl/base:config", "//absl/base:core_headers", + "//absl/base:no_destructor", "//absl/base:raw_logging_internal", "//absl/debugging:stacktrace", "//absl/memory", diff --git a/absl/container/CMakeLists.txt b/absl/container/CMakeLists.txt index 128cc0e9..449a2cad 100644 --- a/absl/container/CMakeLists.txt +++ b/absl/container/CMakeLists.txt @@ -602,6 +602,7 @@ absl_cc_library( absl::base absl::config absl::exponential_biased + absl::no_destructor absl::raw_logging_internal absl::sample_recorder absl::synchronization diff --git a/absl/container/internal/hashtablez_sampler.cc b/absl/container/internal/hashtablez_sampler.cc index 79a0973a..b21beef0 100644 --- a/absl/container/internal/hashtablez_sampler.cc +++ b/absl/container/internal/hashtablez_sampler.cc @@ -24,6 +24,7 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/internal/raw_logging.h" +#include "absl/base/no_destructor.h" #include "absl/debugging/stacktrace.h" #include "absl/memory/memory.h" #include "absl/profiling/internal/exponential_biased.h" @@ -64,7 +65,7 @@ ABSL_PER_THREAD_TLS_KEYWORD SamplingState global_next_sample = {0, 0}; #endif // defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) HashtablezSampler& GlobalHashtablezSampler() { - static auto* sampler = new HashtablezSampler(); + static absl::NoDestructor sampler; return *sampler; } -- cgit v1.2.3 From d5eb503257f23d41f3e67933ffd85fb989141acb Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Mon, 29 Jan 2024 13:33:24 -0800 Subject: Introduce `RawHashSetLayout` helper class. PiperOrigin-RevId: 602485199 Change-Id: I5cc10eb8dcfe5988cf939080a224522e02ad8607 --- absl/container/internal/raw_hash_set.h | 104 ++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 41 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 3518bc34..ba7df607 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -80,7 +80,7 @@ // slot_type slots[capacity]; // }; // -// The length of this array is computed by `AllocSize()` below. +// The length of this array is computed by `RawHashSetLayout::alloc_size` below. // // Control bytes (`ctrl_t`) are bytes (collected into groups of a // platform-specific size) that define the state of the corresponding slot in @@ -983,12 +983,6 @@ using HashSetIteratorGenerationInfo = HashSetIteratorGenerationInfoDisabled; // A valid capacity is a non-zero integer `2^m - 1`. inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; } -// Computes the offset from the start of the backing allocation of control. -// infoz and growth_left are stored at the beginning of the backing array. -inline size_t ControlOffset(bool has_infoz) { - return (has_infoz ? sizeof(HashtablezInfoHandle) : 0) + sizeof(size_t); -} - // Returns the number of "cloned control bytes". // // This is the number of control bytes that are present both at the beginning @@ -996,29 +990,57 @@ inline size_t ControlOffset(bool has_infoz) { // `Group::kWidth`-width probe window starting from any control byte. constexpr size_t NumClonedBytes() { return Group::kWidth - 1; } -// Given the capacity of a table, computes the offset (from the start of the -// backing allocation) of the generation counter (if it exists). -inline size_t GenerationOffset(size_t capacity, bool has_infoz) { - assert(IsValidCapacity(capacity)); - const size_t num_control_bytes = capacity + 1 + NumClonedBytes(); - return ControlOffset(has_infoz) + num_control_bytes; +// Returns the number of control bytes including cloned. +inline size_t NumControlBytes(size_t capacity) { + return capacity + 1 + NumClonedBytes(); } -// Given the capacity of a table, computes the offset (from the start of the -// backing allocation) at which the slots begin. -inline size_t SlotOffset(size_t capacity, size_t slot_align, bool has_infoz) { - assert(IsValidCapacity(capacity)); - return (GenerationOffset(capacity, has_infoz) + NumGenerationBytes() + - slot_align - 1) & - (~slot_align + 1); +// Computes the offset from the start of the backing allocation of control. +// infoz and growth_left are stored at the beginning of the backing array. +inline static size_t ControlOffset(bool has_infoz) { + return (has_infoz ? sizeof(HashtablezInfoHandle) : 0) + sizeof(size_t); } -// Given the capacity of a table, computes the total size of the backing -// array. -inline size_t AllocSize(size_t capacity, size_t slot_size, size_t slot_align, - bool has_infoz) { - return SlotOffset(capacity, slot_align, has_infoz) + capacity * slot_size; -} +// Helper class for computing offsets and allocation size of hash set fields. +class RawHashSetLayout { + public: + explicit RawHashSetLayout(size_t capacity, size_t slot_align, bool has_infoz) + : capacity_(capacity), + control_offset_(ControlOffset(has_infoz)), + generation_offset_(control_offset_ + NumControlBytes(capacity)), + slot_offset_( + (generation_offset_ + NumGenerationBytes() + slot_align - 1) & + (~slot_align + 1)) { + assert(IsValidCapacity(capacity)); + } + + // Returns the capacity of a table. + size_t capacity() const { return capacity_; } + + // Returns precomputed offset from the start of the backing allocation of + // control. + size_t control_offset() const { return control_offset_; } + + // Given the capacity of a table, computes the offset (from the start of the + // backing allocation) of the generation counter (if it exists). + size_t generation_offset() const { return generation_offset_; } + + // Given the capacity of a table, computes the offset (from the start of the + // backing allocation) at which the slots begin. + size_t slot_offset() const { return slot_offset_; } + + // Given the capacity of a table, computes the total size of the backing + // array. + size_t alloc_size(size_t slot_size) const { + return slot_offset_ + capacity_ * slot_size; + } + + private: + size_t capacity_; + size_t control_offset_; + size_t generation_offset_; + size_t slot_offset_; +}; // CommonFields hold the fields in raw_hash_set that do not depend // on template parameters. This allows us to conveniently pass all @@ -1116,7 +1138,8 @@ class CommonFields : public CommonFieldsGenerationInfo { // The size of the backing array allocation. size_t alloc_size(size_t slot_size, size_t slot_align) const { - return AllocSize(capacity(), slot_size, slot_align, has_infoz()); + return RawHashSetLayout(capacity(), slot_align, has_infoz()) + .alloc_size(slot_size); } // Returns the number of control bytes set to kDeleted. For testing only. @@ -1608,27 +1631,25 @@ class HashSetResizeHelper { sample_size > 0 ? Sample(sample_size) : c.infoz(); const bool has_infoz = infoz.IsSampled(); - const size_t cap = c.capacity(); - const size_t alloc_size = - AllocSize(cap, SizeOfSlot, AlignOfSlot, has_infoz); - char* mem = static_cast( - Allocate(&alloc, alloc_size)); + RawHashSetLayout layout(c.capacity(), AlignOfSlot, has_infoz); + char* mem = static_cast(Allocate( + &alloc, layout.alloc_size(SizeOfSlot))); const GenerationType old_generation = c.generation(); - c.set_generation_ptr(reinterpret_cast( - mem + GenerationOffset(cap, has_infoz))); + c.set_generation_ptr( + reinterpret_cast(mem + layout.generation_offset())); c.set_generation(NextGeneration(old_generation)); - c.set_control(reinterpret_cast(mem + ControlOffset(has_infoz))); - c.set_slots(mem + SlotOffset(cap, AlignOfSlot, has_infoz)); + c.set_control(reinterpret_cast(mem + layout.control_offset())); + c.set_slots(mem + layout.slot_offset()); ResetGrowthLeft(c); const bool grow_single_group = - IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity()); + IsGrowingIntoSingleGroupApplicable(old_capacity_, layout.capacity()); if (old_capacity_ != 0 && grow_single_group) { if (TransferUsesMemcpy) { GrowSizeIntoSingleGroupTransferable(c, old_slots, SizeOfSlot); DeallocateOld(alloc, SizeOfSlot, old_slots); } else { - GrowIntoSingleGroupShuffleControlBytes(c.control(), c.capacity()); + GrowIntoSingleGroupShuffleControlBytes(c.control(), layout.capacity()); } } else { ResetCtrl(c, SizeOfSlot); @@ -1636,7 +1657,7 @@ class HashSetResizeHelper { c.set_has_infoz(has_infoz); if (has_infoz) { - infoz.RecordStorageChanged(c.size(), cap); + infoz.RecordStorageChanged(c.size(), layout.capacity()); if (grow_single_group || old_capacity_ == 0) { infoz.RecordRehash(0); } @@ -1675,9 +1696,10 @@ class HashSetResizeHelper { template void DeallocateOld(CharAlloc alloc_ref, size_t slot_size, void* old_slots) { SanitizerUnpoisonMemoryRegion(old_slots, slot_size * old_capacity_); + auto layout = RawHashSetLayout(old_capacity_, AlignOfSlot, had_infoz_); Deallocate( - &alloc_ref, old_ctrl_ - ControlOffset(had_infoz_), - AllocSize(old_capacity_, slot_size, AlignOfSlot, had_infoz_)); + &alloc_ref, old_ctrl_ - layout.control_offset(), + layout.alloc_size(slot_size)); } private: -- cgit v1.2.3 From c44dd5acd7848a175d1fb939cdb81a4cf8d4c912 Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Tue, 30 Jan 2024 12:52:43 -0800 Subject: Early return from destroy_slots for trivially destructible types in flat_hash_{*}. PiperOrigin-RevId: 602813933 Change-Id: I744fe438281755a141b2fd47e54ab9c6c0fad5a3 --- absl/container/BUILD.bazel | 2 + absl/container/CMakeLists.txt | 2 + absl/container/flat_hash_map.h | 5 +- absl/container/flat_hash_map_test.cc | 12 ++++ absl/container/flat_hash_set.h | 5 +- absl/container/flat_hash_set_test.cc | 12 ++++ absl/container/internal/common_policy_traits.h | 12 +++- .../internal/common_policy_traits_test.cc | 65 +++++++++++++++------- absl/container/internal/container_memory.h | 15 ++++- absl/container/internal/container_memory_test.cc | 18 ++++++ absl/container/internal/raw_hash_set.h | 1 + 11 files changed, 122 insertions(+), 27 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/BUILD.bazel b/absl/container/BUILD.bazel index fda866a5..bc5b2201 100644 --- a/absl/container/BUILD.bazel +++ b/absl/container/BUILD.bazel @@ -264,6 +264,7 @@ cc_test( deps = [ ":flat_hash_map", ":hash_generator_testing", + ":test_allocator", ":unordered_map_constructor_test", ":unordered_map_lookup_test", ":unordered_map_members_test", @@ -301,6 +302,7 @@ cc_test( ":container_memory", ":flat_hash_set", ":hash_generator_testing", + ":test_allocator", ":unordered_set_constructor_test", ":unordered_set_lookup_test", ":unordered_set_members_test", diff --git a/absl/container/CMakeLists.txt b/absl/container/CMakeLists.txt index ee9ca9c3..3f80d29b 100644 --- a/absl/container/CMakeLists.txt +++ b/absl/container/CMakeLists.txt @@ -307,6 +307,7 @@ absl_cc_test( absl::check absl::flat_hash_map absl::hash_generator_testing + absl::test_allocator absl::type_traits absl::unordered_map_constructor_test absl::unordered_map_lookup_test @@ -348,6 +349,7 @@ absl_cc_test( absl::hash_generator_testing absl::memory absl::strings + absl::test_allocator absl::unordered_set_constructor_test absl::unordered_set_lookup_test absl::unordered_set_members_test diff --git a/absl/container/flat_hash_map.h b/absl/container/flat_hash_map.h index acd013b0..808a62ba 100644 --- a/absl/container/flat_hash_map.h +++ b/absl/container/flat_hash_map.h @@ -573,9 +573,10 @@ struct FlatHashMapPolicy { slot_policy::construct(alloc, slot, std::forward(args)...); } + // Returns std::true_type in case destroy is trivial. template - static void destroy(Allocator* alloc, slot_type* slot) { - slot_policy::destroy(alloc, slot); + static auto destroy(Allocator* alloc, slot_type* slot) { + return slot_policy::destroy(alloc, slot); } template diff --git a/absl/container/flat_hash_map_test.cc b/absl/container/flat_hash_map_test.cc index d90fe9d5..8ef1a62b 100644 --- a/absl/container/flat_hash_map_test.cc +++ b/absl/container/flat_hash_map_test.cc @@ -22,6 +22,7 @@ #include "gtest/gtest.h" #include "absl/container/internal/hash_generator_testing.h" +#include "absl/container/internal/test_allocator.h" #include "absl/container/internal/unordered_map_constructor_test.h" #include "absl/container/internal/unordered_map_lookup_test.h" #include "absl/container/internal/unordered_map_members_test.h" @@ -351,6 +352,17 @@ TEST(FlatHashMap, RecursiveTypeCompiles) { t.m[0] = RecursiveType{}; } +TEST(FlatHashMap, FlatHashMapPolicyDestroyReturnsTrue) { + EXPECT_TRUE( + (decltype(FlatHashMapPolicy::destroy>( + nullptr, nullptr))())); + EXPECT_FALSE( + (decltype(FlatHashMapPolicy::destroy>( + nullptr, nullptr))())); + EXPECT_FALSE((decltype(FlatHashMapPolicy>::destroy< + std::allocator>(nullptr, nullptr))())); +} + } // namespace } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/flat_hash_set.h b/absl/container/flat_hash_set.h index a94a82a0..cd74923c 100644 --- a/absl/container/flat_hash_set.h +++ b/absl/container/flat_hash_set.h @@ -29,6 +29,7 @@ #ifndef ABSL_CONTAINER_FLAT_HASH_SET_H_ #define ABSL_CONTAINER_FLAT_HASH_SET_H_ +#include #include #include @@ -473,9 +474,11 @@ struct FlatHashSetPolicy { std::forward(args)...); } + // Return std::true_type in case destroy is trivial. template - static void destroy(Allocator* alloc, slot_type* slot) { + static auto destroy(Allocator* alloc, slot_type* slot) { absl::allocator_traits::destroy(*alloc, slot); + return IsDestructionTrivial(); } static T& element(slot_type* slot) { return *slot; } diff --git a/absl/container/flat_hash_set_test.cc b/absl/container/flat_hash_set_test.cc index a60b4bf5..9ce9267e 100644 --- a/absl/container/flat_hash_set_test.cc +++ b/absl/container/flat_hash_set_test.cc @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -24,6 +25,7 @@ #include "absl/base/config.h" #include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_generator_testing.h" +#include "absl/container/internal/test_allocator.h" #include "absl/container/internal/unordered_set_constructor_test.h" #include "absl/container/internal/unordered_set_lookup_test.h" #include "absl/container/internal/unordered_set_members_test.h" @@ -237,6 +239,16 @@ TEST(FlatHashSet, PoisonInline) { } } +TEST(FlatHashSet, FlatHashSetPolicyDestroyReturnsTrue) { + EXPECT_TRUE((decltype(FlatHashSetPolicy::destroy>( + nullptr, nullptr))())); + EXPECT_FALSE( + (decltype(FlatHashSetPolicy::destroy>( + nullptr, nullptr))())); + EXPECT_FALSE((decltype(FlatHashSetPolicy>::destroy< + std::allocator>(nullptr, nullptr))())); +} + } // namespace } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/internal/common_policy_traits.h b/absl/container/internal/common_policy_traits.h index 57eac678..77df4790 100644 --- a/absl/container/internal/common_policy_traits.h +++ b/absl/container/internal/common_policy_traits.h @@ -45,9 +45,10 @@ struct common_policy_traits { // PRECONDITION: `slot` is INITIALIZED // POSTCONDITION: `slot` is UNINITIALIZED + // Returns std::true_type in case destroy is trivial. template - static void destroy(Alloc* alloc, slot_type* slot) { - Policy::destroy(alloc, slot); + static auto destroy(Alloc* alloc, slot_type* slot) { + return Policy::destroy(alloc, slot); } // Transfers the `old_slot` to `new_slot`. Any memory allocated by the @@ -86,6 +87,13 @@ struct common_policy_traits { std::true_type>::value; } + // Returns true if destroy is trivial and can be omitted. + template + static constexpr bool destroy_is_trivial() { + return std::is_same(nullptr, nullptr)), + std::true_type>::value; + } + private: // To rank the overloads below for overload resolution. Rank0 is preferred. struct Rank2 {}; diff --git a/absl/container/internal/common_policy_traits_test.cc b/absl/container/internal/common_policy_traits_test.cc index faee3e7a..8d8f8baa 100644 --- a/absl/container/internal/common_policy_traits_test.cc +++ b/absl/container/internal/common_policy_traits_test.cc @@ -39,44 +39,59 @@ struct PolicyWithoutOptionalOps { using key_type = Slot; using init_type = Slot; - static std::function construct; - static std::function destroy; + struct PolicyFunctions { + std::function construct; + std::function destroy; + std::function element; + }; + + static PolicyFunctions* functions() { + static PolicyFunctions* functions = new PolicyFunctions(); + return functions; + } - static std::function element; + static void construct(void* a, Slot* b, Slot c) { + functions()->construct(a, b, c); + } + static void destroy(void* a, Slot* b) { functions()->destroy(a, b); } + static Slot& element(Slot* b) { return functions()->element(b); } }; -std::function PolicyWithoutOptionalOps::construct; -std::function PolicyWithoutOptionalOps::destroy; - -std::function PolicyWithoutOptionalOps::element; - struct PolicyWithOptionalOps : PolicyWithoutOptionalOps { - static std::function transfer; + struct TransferFunctions { + std::function transfer; + }; + + static TransferFunctions* transfer_fn() { + static TransferFunctions* transfer_fn = new TransferFunctions(); + return transfer_fn; + } + static void transfer(void* a, Slot* b, Slot* c) { + transfer_fn()->transfer(a, b, c); + } }; -std::function PolicyWithOptionalOps::transfer; -struct PolicyWithMemcpyTransfer : PolicyWithoutOptionalOps { - static std::function transfer; +struct PolicyWithMemcpyTransferAndTrivialDestroy : PolicyWithoutOptionalOps { + static std::true_type transfer(void*, Slot*, Slot*) { return {}; } + static std::true_type destroy(void*, Slot*) { return {}; } }; -std::function - PolicyWithMemcpyTransfer::transfer; struct Test : ::testing::Test { Test() { - PolicyWithoutOptionalOps::construct = [&](void* a1, Slot* a2, Slot a3) { + PolicyWithoutOptionalOps::functions()->construct = [&](void* a1, Slot* a2, + Slot a3) { construct.Call(a1, a2, std::move(a3)); }; - PolicyWithoutOptionalOps::destroy = [&](void* a1, Slot* a2) { + PolicyWithoutOptionalOps::functions()->destroy = [&](void* a1, Slot* a2) { destroy.Call(a1, a2); }; - PolicyWithoutOptionalOps::element = [&](Slot* a1) -> Slot& { + PolicyWithoutOptionalOps::functions()->element = [&](Slot* a1) -> Slot& { return element.Call(a1); }; - PolicyWithOptionalOps::transfer = [&](void* a1, Slot* a2, Slot* a3) { - return transfer.Call(a1, a2, a3); - }; + PolicyWithOptionalOps::transfer_fn()->transfer = + [&](void* a1, Slot* a2, Slot* a3) { return transfer.Call(a1, a2, a3); }; } std::allocator alloc; @@ -125,7 +140,15 @@ TEST(TransferUsesMemcpy, Basic) { EXPECT_FALSE( common_policy_traits::transfer_uses_memcpy()); EXPECT_TRUE( - common_policy_traits::transfer_uses_memcpy()); + common_policy_traits< + PolicyWithMemcpyTransferAndTrivialDestroy>::transfer_uses_memcpy()); +} + +TEST(DestroyIsTrivial, Basic) { + EXPECT_FALSE(common_policy_traits::destroy_is_trivial< + std::allocator>()); + EXPECT_TRUE(common_policy_traits:: + destroy_is_trivial>()); } } // namespace diff --git a/absl/container/internal/container_memory.h b/absl/container/internal/container_memory.h index 3262d4eb..d86d7b80 100644 --- a/absl/container/internal/container_memory.h +++ b/absl/container/internal/container_memory.h @@ -68,6 +68,18 @@ void* Allocate(Alloc* alloc, size_t n) { return p; } +// Returns true if the destruction of the value with given Allocator will be +// trivial. +template +constexpr auto IsDestructionTrivial() { + constexpr bool result = + std::is_trivially_destructible::value && + std::is_same::template rebind_alloc, + std::allocator>::value; + return std::integral_constant(); +} + // The pointer must have been previously obtained by calling // Allocate(alloc, n). template @@ -414,12 +426,13 @@ struct map_slot_policy { } template - static void destroy(Allocator* alloc, slot_type* slot) { + static auto destroy(Allocator* alloc, slot_type* slot) { if (kMutableKeys::value) { absl::allocator_traits::destroy(*alloc, &slot->mutable_value); } else { absl::allocator_traits::destroy(*alloc, &slot->value); } + return IsDestructionTrivial(); } template diff --git a/absl/container/internal/container_memory_test.cc b/absl/container/internal/container_memory_test.cc index 90d64bf5..c973ac25 100644 --- a/absl/container/internal/container_memory_test.cc +++ b/absl/container/internal/container_memory_test.cc @@ -280,6 +280,24 @@ TEST(MapSlotPolicy, TransferReturnsTrue) { } } +TEST(MapSlotPolicy, DestroyReturnsTrue) { + { + using slot_policy = map_slot_policy; + EXPECT_TRUE( + (std::is_same>( + nullptr, nullptr)), + std::true_type>::value)); + } + { + EXPECT_FALSE(std::is_trivially_destructible>::value); + using slot_policy = map_slot_policy>; + EXPECT_TRUE( + (std::is_same>( + nullptr, nullptr)), + std::false_type>::value)); + } +} + } // namespace } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index ba7df607..9f16a815 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -2881,6 +2881,7 @@ class raw_hash_set { } inline void destroy_slots() { + if (PolicyTraits::template destroy_is_trivial()) return; const size_t cap = capacity(); const ctrl_t* ctrl = control(); slot_type* slot = slot_array(); -- cgit v1.2.3 From 2812af9184eaa2bfd18d1545c57bcf8cbee88a9d Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Wed, 31 Jan 2024 01:37:11 -0800 Subject: Avoid extra `& msbs` on every iteration over the mask for GroupPortableImpl. PiperOrigin-RevId: 602974812 Change-Id: Ic35b41e321b9456a8ddd83470ee2eb07c51e3180 --- absl/container/internal/raw_hash_set.h | 49 ++++++++++++++----------- absl/container/internal/raw_hash_set_test.cc | 54 ++++++++++++++++++++++++++-- 2 files changed, 79 insertions(+), 24 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 9f16a815..c464de6a 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -374,6 +374,9 @@ uint32_t TrailingZeros(T x) { return static_cast(countr_zero(x)); } +// 8 bytes bitmask with most significant bit set for every byte. +constexpr uint64_t kMsbs8Bytes = 0x8080808080808080ULL; + // An abstract bitmask, such as that emitted by a SIMD instruction. // // Specifically, this type implements a simple bitset whose representation is @@ -423,27 +426,35 @@ class NonIterableBitMask { // an ordinary 16-bit bitset occupying the low 16 bits of `mask`. When // `SignificantBits` is 8 and `Shift` is 3, abstract bits are represented as // the bytes `0x00` and `0x80`, and it occupies all 64 bits of the bitmask. +// If NullifyBitsOnIteration is true (only allowed for Shift == 3), +// non zero abstract bit is allowed to have additional bits +// (e.g., `0xff`, `0x83` and `0x9c` are ok, but `0x6f` is not). // // For example: // for (int i : BitMask(0b101)) -> yields 0, 2 // for (int i : BitMask(0x0000000080800000)) -> yields 2, 3 -template +template class BitMask : public NonIterableBitMask { using Base = NonIterableBitMask; static_assert(std::is_unsigned::value, ""); static_assert(Shift == 0 || Shift == 3, ""); + static_assert(!NullifyBitsOnIteration || Shift == 3, ""); public: - explicit BitMask(T mask) : Base(mask) {} + explicit BitMask(T mask) : Base(mask) { + if (Shift == 3 && !NullifyBitsOnIteration) { + assert(this->mask_ == (this->mask_ & kMsbs8Bytes)); + } + } // BitMask is an iterator over the indices of its abstract bits. using value_type = int; using iterator = BitMask; using const_iterator = BitMask; BitMask& operator++() { - if (Shift == 3) { - constexpr uint64_t msbs = 0x8080808080808080ULL; - this->mask_ &= msbs; + if (Shift == 3 && NullifyBitsOnIteration) { + this->mask_ &= kMsbs8Bytes; } this->mask_ &= (this->mask_ - 1); return *this; @@ -685,10 +696,11 @@ struct GroupAArch64Impl { ctrl = vld1_u8(reinterpret_cast(pos)); } - BitMask Match(h2_t hash) const { + auto Match(h2_t hash) const { uint8x8_t dup = vdup_n_u8(hash); auto mask = vceq_u8(ctrl, dup); - return BitMask( + return BitMask( vget_lane_u64(vreinterpret_u64_u8(mask), 0)); } @@ -704,12 +716,13 @@ struct GroupAArch64Impl { // Returns a bitmask representing the positions of full slots. // Note: for `is_small()` tables group may contain the "same" slot twice: // original and mirrored. - BitMask MaskFull() const { + auto MaskFull() const { uint64_t mask = vget_lane_u64( vreinterpret_u64_u8(vcge_s8(vreinterpret_s8_u8(ctrl), vdup_n_s8(static_cast(0)))), 0); - return BitMask(mask); + return BitMask(mask); } NonIterableBitMask MaskEmptyOrDeleted() const { @@ -736,11 +749,10 @@ struct GroupAArch64Impl { void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(ctrl), 0); - constexpr uint64_t msbs = 0x8080808080808080ULL; constexpr uint64_t slsbs = 0x0202020202020202ULL; constexpr uint64_t midbs = 0x7e7e7e7e7e7e7e7eULL; auto x = slsbs & (mask >> 6); - auto res = (x + midbs) | msbs; + auto res = (x + midbs) | kMsbs8Bytes; little_endian::Store64(dst, res); } @@ -768,30 +780,26 @@ struct GroupPortableImpl { // v = 0x1716151413121110 // hash = 0x12 // retval = (v - lsbs) & ~v & msbs = 0x0000000080800000 - constexpr uint64_t msbs = 0x8080808080808080ULL; constexpr uint64_t lsbs = 0x0101010101010101ULL; auto x = ctrl ^ (lsbs * hash); - return BitMask((x - lsbs) & ~x & msbs); + return BitMask((x - lsbs) & ~x & kMsbs8Bytes); } NonIterableBitMask MaskEmpty() const { - constexpr uint64_t msbs = 0x8080808080808080ULL; return NonIterableBitMask((ctrl & ~(ctrl << 6)) & - msbs); + kMsbs8Bytes); } // Returns a bitmask representing the positions of full slots. // Note: for `is_small()` tables group may contain the "same" slot twice: // original and mirrored. BitMask MaskFull() const { - constexpr uint64_t msbs = 0x8080808080808080ULL; - return BitMask((ctrl ^ msbs) & msbs); + return BitMask((ctrl ^ kMsbs8Bytes) & kMsbs8Bytes); } NonIterableBitMask MaskEmptyOrDeleted() const { - constexpr uint64_t msbs = 0x8080808080808080ULL; return NonIterableBitMask((ctrl & ~(ctrl << 7)) & - msbs); + kMsbs8Bytes); } uint32_t CountLeadingEmptyOrDeleted() const { @@ -803,9 +811,8 @@ struct GroupPortableImpl { } void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { - constexpr uint64_t msbs = 0x8080808080808080ULL; constexpr uint64_t lsbs = 0x0101010101010101ULL; - auto x = ctrl & msbs; + auto x = ctrl & kMsbs8Bytes; auto res = (~x + (x >> 7)) & ~lsbs; little_endian::Store64(dst, res); } diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index f9797f56..6baaa060 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -15,6 +15,7 @@ #include "absl/container/internal/raw_hash_set.h" #include +#include #include #include #include @@ -75,6 +76,7 @@ struct RawHashSetTestOnlyAccess { namespace { using ::testing::ElementsAre; +using ::testing::ElementsAreArray; using ::testing::Eq; using ::testing::Ge; using ::testing::Lt; @@ -156,20 +158,66 @@ TEST(BitMask, Smoke) { EXPECT_THAT((BitMask(0xAA)), ElementsAre(1, 3, 5, 7)); } -TEST(BitMask, WithShift) { +TEST(BitMask, WithShift_MatchPortable) { // See the non-SSE version of Group for details on what this math is for. uint64_t ctrl = 0x1716151413121110; uint64_t hash = 0x12; - constexpr uint64_t msbs = 0x8080808080808080ULL; constexpr uint64_t lsbs = 0x0101010101010101ULL; auto x = ctrl ^ (lsbs * hash); - uint64_t mask = (x - lsbs) & ~x & msbs; + uint64_t mask = (x - lsbs) & ~x & kMsbs8Bytes; EXPECT_EQ(0x0000000080800000, mask); BitMask b(mask); EXPECT_EQ(*b, 2); } +constexpr uint64_t kSome8BytesMask = /* */ 0x8000808080008000ULL; +constexpr uint64_t kSome8BytesMaskAllOnes = 0xff00ffffff00ff00ULL; +constexpr auto kSome8BytesMaskBits = std::array{1, 3, 4, 5, 7}; + + +TEST(BitMask, WithShift_FullMask) { + EXPECT_THAT((BitMask(kMsbs8Bytes)), + ElementsAre(0, 1, 2, 3, 4, 5, 6, 7)); + EXPECT_THAT( + (BitMask(kMsbs8Bytes)), + ElementsAre(0, 1, 2, 3, 4, 5, 6, 7)); + EXPECT_THAT( + (BitMask(~uint64_t{0})), + ElementsAre(0, 1, 2, 3, 4, 5, 6, 7)); +} + +TEST(BitMask, WithShift_EmptyMask) { + EXPECT_THAT((BitMask(0)), ElementsAre()); + EXPECT_THAT((BitMask(0)), + ElementsAre()); +} + +TEST(BitMask, WithShift_SomeMask) { + EXPECT_THAT((BitMask(kSome8BytesMask)), + ElementsAreArray(kSome8BytesMaskBits)); + EXPECT_THAT((BitMask( + kSome8BytesMask)), + ElementsAreArray(kSome8BytesMaskBits)); + EXPECT_THAT((BitMask( + kSome8BytesMaskAllOnes)), + ElementsAreArray(kSome8BytesMaskBits)); +} + +TEST(BitMask, WithShift_SomeMaskExtraBitsForNullify) { + // Verify that adding extra bits into non zero bytes is fine. + uint64_t extra_bits = 77; + for (int i = 0; i < 100; ++i) { + // Add extra bits, but keep zero bytes untouched. + uint64_t extra_mask = extra_bits & kSome8BytesMaskAllOnes; + EXPECT_THAT((BitMask( + kSome8BytesMask | extra_mask)), + ElementsAreArray(kSome8BytesMaskBits)) + << i << " " << extra_mask; + extra_bits = (extra_bits + 1) * 3; + } +} + TEST(BitMask, LeadingTrailing) { EXPECT_EQ((BitMask(0x00001a40).LeadingZeros()), 3); EXPECT_EQ((BitMask(0x00001a40).TrailingZeros()), 6); -- cgit v1.2.3 From 4c7e7c7d94eaaa3bff3142c257d880a468a35934 Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Wed, 31 Jan 2024 13:45:52 -0800 Subject: Type erased hash_slot_fn that depends only on key types (and hash function). PiperOrigin-RevId: 603148301 Change-Id: Ie2e5702995c9e1ef4d5aaab23bc89a1eb5007a86 --- absl/container/BUILD.bazel | 4 ++ absl/container/CMakeLists.txt | 3 + absl/container/flat_hash_map.h | 7 +++ absl/container/flat_hash_set.h | 6 ++ absl/container/internal/container_memory.h | 20 +++++++ absl/container/internal/container_memory_test.cc | 14 +++++ absl/container/internal/hash_policy_traits.h | 35 ++++++++++++ absl/container/internal/hash_policy_traits_test.cc | 64 ++++++++++++++++++++++ absl/container/internal/raw_hash_set.cc | 4 +- absl/container/internal/raw_hash_set.h | 15 ++--- .../internal/raw_hash_set_allocator_test.cc | 8 ++- absl/container/internal/raw_hash_set_benchmark.cc | 11 ++++ .../internal/raw_hash_set_probe_benchmark.cc | 5 ++ absl/container/internal/raw_hash_set_test.cc | 19 ++++++- absl/container/node_hash_map.h | 8 +++ absl/container/node_hash_set.h | 7 +++ 16 files changed, 214 insertions(+), 16 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/BUILD.bazel b/absl/container/BUILD.bazel index 91633948..5160ccea 100644 --- a/absl/container/BUILD.bazel +++ b/absl/container/BUILD.bazel @@ -357,6 +357,7 @@ cc_library( copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ + ":container_memory", ":hash_function_defaults", ":node_slot_policy", ":raw_hash_set", @@ -504,6 +505,7 @@ cc_test( copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ + ":container_memory", ":hash_policy_traits", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", @@ -714,6 +716,7 @@ cc_binary( tags = ["benchmark"], visibility = ["//visibility:private"], deps = [ + ":container_memory", ":hash_function_defaults", ":raw_hash_set", "//absl/base:raw_logging_internal", @@ -753,6 +756,7 @@ cc_test( copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ + ":container_memory", ":raw_hash_set", ":tracked", "//absl/base:config", diff --git a/absl/container/CMakeLists.txt b/absl/container/CMakeLists.txt index 3f80d29b..85af0bd7 100644 --- a/absl/container/CMakeLists.txt +++ b/absl/container/CMakeLists.txt @@ -401,6 +401,7 @@ absl_cc_library( COPTS ${ABSL_DEFAULT_COPTS} DEPS + absl::container_memory absl::core_headers absl::hash_function_defaults absl::node_slot_policy @@ -560,6 +561,7 @@ absl_cc_test( COPTS ${ABSL_TEST_COPTS} DEPS + absl::container_memory absl::hash_policy_traits GTest::gmock_main ) @@ -776,6 +778,7 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::config + absl::container_memory absl::raw_hash_set absl::tracked GTest::gmock_main diff --git a/absl/container/flat_hash_map.h b/absl/container/flat_hash_map.h index 808a62ba..2a1ad0fb 100644 --- a/absl/container/flat_hash_map.h +++ b/absl/container/flat_hash_map.h @@ -593,6 +593,13 @@ struct FlatHashMapPolicy { std::forward(args)...); } + template + static constexpr HashSlotFn get_hash_slot_fn() { + return memory_internal::IsLayoutCompatible::value + ? &TypeErasedApplyToSlotFn + : nullptr; + } + static size_t space_used(const slot_type*) { return 0; } static std::pair& element(slot_type* slot) { return slot->value; } diff --git a/absl/container/flat_hash_set.h b/absl/container/flat_hash_set.h index cd74923c..1be00195 100644 --- a/absl/container/flat_hash_set.h +++ b/absl/container/flat_hash_set.h @@ -29,6 +29,7 @@ #ifndef ABSL_CONTAINER_FLAT_HASH_SET_H_ #define ABSL_CONTAINER_FLAT_HASH_SET_H_ +#include #include #include #include @@ -492,6 +493,11 @@ struct FlatHashSetPolicy { } static size_t space_used(const T*) { return 0; } + + template + static constexpr HashSlotFn get_hash_slot_fn() { + return &TypeErasedApplyToSlotFn; + } }; } // namespace container_internal diff --git a/absl/container/internal/container_memory.h b/absl/container/internal/container_memory.h index d86d7b80..ba8e08a2 100644 --- a/absl/container/internal/container_memory.h +++ b/absl/container/internal/container_memory.h @@ -464,6 +464,26 @@ struct map_slot_policy { } }; +// Type erased function for computing hash of the slot. +using HashSlotFn = size_t (*)(const void* hash_fn, void* slot); + +// Type erased function to apply `Fn` to data inside of the `slot`. +// The data is expected to have type `T`. +template +size_t TypeErasedApplyToSlotFn(const void* fn, void* slot) { + const auto* f = static_cast(fn); + return (*f)(*static_cast(slot)); +} + +// Type erased function to apply `Fn` to data inside of the `*slot_ptr`. +// The data is expected to have type `T`. +template +size_t TypeErasedDerefAndApplyToSlotFn(const void* fn, void* slot_ptr) { + const auto* f = static_cast(fn); + const T* slot = *static_cast(slot_ptr); + return (*f)(*slot); +} + } // namespace container_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/container/internal/container_memory_test.cc b/absl/container/internal/container_memory_test.cc index c973ac25..7e4357d5 100644 --- a/absl/container/internal/container_memory_test.cc +++ b/absl/container/internal/container_memory_test.cc @@ -298,6 +298,20 @@ TEST(MapSlotPolicy, DestroyReturnsTrue) { } } +TEST(ApplyTest, TypeErasedApplyToSlotFn) { + size_t x = 7; + auto fn = [](size_t v) { return v * 2; }; + EXPECT_EQ((TypeErasedApplyToSlotFn(&fn, &x)), 14); +} + +TEST(ApplyTest, TypeErasedDerefAndApplyToSlotFn) { + size_t x = 7; + auto fn = [](size_t v) { return v * 2; }; + size_t* x_ptr = &x; + EXPECT_EQ( + (TypeErasedDerefAndApplyToSlotFn(&fn, &x_ptr)), 14); +} + } // namespace } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/internal/hash_policy_traits.h b/absl/container/internal/hash_policy_traits.h index 164ec123..86ffd1be 100644 --- a/absl/container/internal/hash_policy_traits.h +++ b/absl/container/internal/hash_policy_traits.h @@ -148,6 +148,41 @@ struct hash_policy_traits : common_policy_traits { static auto value(T* elem) -> decltype(P::value(elem)) { return P::value(elem); } + + using HashSlotFn = size_t (*)(const void* hash_fn, void* slot); + + template + static constexpr HashSlotFn get_hash_slot_fn() { +// get_hash_slot_fn may return nullptr to signal that non type erased function +// should be used. GCC warns against comparing function address with nullptr. +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +// silent error: the address of * will never be NULL [-Werror=address] +#pragma GCC diagnostic ignored "-Waddress" +#endif + return Policy::template get_hash_slot_fn() == nullptr + ? &hash_slot_fn_non_type_erased + : Policy::template get_hash_slot_fn(); +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif + } + + private: + template + struct HashElement { + template + size_t operator()(const K& key, Args&&...) const { + return h(key); + } + const Hash& h; + }; + + template + static size_t hash_slot_fn_non_type_erased(const void* hash_fn, void* slot) { + return Policy::apply(HashElement{*static_cast(hash_fn)}, + Policy::element(static_cast(slot))); + } }; } // namespace container_internal diff --git a/absl/container/internal/hash_policy_traits_test.cc b/absl/container/internal/hash_policy_traits_test.cc index 82d7cc3a..2d2c7c2c 100644 --- a/absl/container/internal/hash_policy_traits_test.cc +++ b/absl/container/internal/hash_policy_traits_test.cc @@ -14,12 +14,14 @@ #include "absl/container/internal/hash_policy_traits.h" +#include #include #include #include #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "absl/container/internal/container_memory.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -42,6 +44,11 @@ struct PolicyWithoutOptionalOps { static int apply(int v) { return apply_impl(v); } static std::function apply_impl; static std::function value; + + template + static constexpr HashSlotFn get_hash_slot_fn() { + return nullptr; + } }; std::function PolicyWithoutOptionalOps::apply_impl; @@ -74,6 +81,63 @@ TEST_F(Test, value) { EXPECT_EQ(&b, &hash_policy_traits::value(&a)); } +struct Hash { + size_t operator()(Slot a) const { return static_cast(a) * 5; } +}; + +struct PolicyNoHashFn { + using slot_type = Slot; + using key_type = Slot; + using init_type = Slot; + + static size_t* apply_called_count; + + static Slot& element(Slot* slot) { return *slot; } + template + static size_t apply(const Fn& fn, int v) { + ++(*apply_called_count); + return fn(v); + } + + template + static constexpr HashSlotFn get_hash_slot_fn() { + return nullptr; + } +}; + +size_t* PolicyNoHashFn::apply_called_count; + +struct PolicyCustomHashFn : PolicyNoHashFn { + template + static constexpr HashSlotFn get_hash_slot_fn() { + return &TypeErasedApplyToSlotFn; + } +}; + +TEST(HashTest, PolicyNoHashFn_get_hash_slot_fn) { + size_t apply_called_count = 0; + PolicyNoHashFn::apply_called_count = &apply_called_count; + + Hash hasher; + Slot value = 7; + auto* fn = hash_policy_traits::get_hash_slot_fn(); + EXPECT_NE(fn, nullptr); + EXPECT_EQ(fn(&hasher, &value), hasher(value)); + EXPECT_EQ(apply_called_count, 1); +} + +TEST(HashTest, PolicyCustomHashFn_get_hash_slot_fn) { + size_t apply_called_count = 0; + PolicyNoHashFn::apply_called_count = &apply_called_count; + + Hash hasher; + Slot value = 7; + auto* fn = hash_policy_traits::get_hash_slot_fn(); + EXPECT_EQ(fn, PolicyCustomHashFn::get_hash_slot_fn()); + EXPECT_EQ(fn(&hasher, &value), hasher(value)); + EXPECT_EQ(apply_called_count, 0); +} + } // namespace } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/internal/raw_hash_set.cc b/absl/container/internal/raw_hash_set.cc index 9f8ea519..02301e19 100644 --- a/absl/container/internal/raw_hash_set.cc +++ b/absl/container/internal/raw_hash_set.cc @@ -140,7 +140,7 @@ static inline void* PrevSlot(void* slot, size_t slot_size) { return reinterpret_cast(reinterpret_cast(slot) - slot_size); } -void DropDeletesWithoutResize(CommonFields& common, +void DropDeletesWithoutResize(CommonFields& common, const void* hash_fn, const PolicyFunctions& policy, void* tmp_space) { void* set = &common; void* slot_array = common.slot_array(); @@ -175,7 +175,7 @@ void DropDeletesWithoutResize(CommonFields& common, ++i, slot_ptr = NextSlot(slot_ptr, slot_size)) { assert(slot_ptr == SlotAddress(slot_array, i, slot_size)); if (!IsDeleted(ctrl[i])) continue; - const size_t hash = (*hasher)(set, slot_ptr); + const size_t hash = (*hasher)(hash_fn, slot_ptr); const FindInfo target = find_first_non_full(common, hash); const size_t new_i = target.offset; total_probe_length += target.probe_length; diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index c464de6a..3b793825 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -1802,7 +1802,7 @@ struct PolicyFunctions { size_t slot_size; // Returns the hash of the pointed-to slot. - size_t (*hash_slot)(void* set, void* slot); + size_t (*hash_slot)(const void* hash_fn, void* slot); // Transfer the contents of src_slot to dst_slot. void (*transfer)(void* set, void* dst_slot, void* src_slot); @@ -1847,7 +1847,7 @@ ABSL_ATTRIBUTE_NOINLINE void TransferRelocatable(void*, void* dst, void* src) { } // Type-erased version of raw_hash_set::drop_deletes_without_resize. -void DropDeletesWithoutResize(CommonFields& common, +void DropDeletesWithoutResize(CommonFields& common, const void* hash_fn, const PolicyFunctions& policy, void* tmp_space); // A SwissTable. @@ -2989,7 +2989,7 @@ class raw_hash_set { inline void drop_deletes_without_resize() { // Stack-allocate space for swapping elements. alignas(slot_type) unsigned char tmp[sizeof(slot_type)]; - DropDeletesWithoutResize(common(), GetPolicyFunctions(), tmp); + DropDeletesWithoutResize(common(), &hash_ref(), GetPolicyFunctions(), tmp); } // Called whenever the table *might* need to conditionally grow. @@ -3238,13 +3238,6 @@ class raw_hash_set { return settings_.template get<3>(); } - // Make type-specific functions for this type's PolicyFunctions struct. - static size_t hash_slot_fn(void* set, void* slot) { - auto* h = static_cast(set); - return PolicyTraits::apply( - HashElement{h->hash_ref()}, - PolicyTraits::element(static_cast(slot))); - } static void transfer_slot_fn(void* set, void* dst, void* src) { auto* h = static_cast(set); h->transfer(static_cast(dst), static_cast(src)); @@ -3266,7 +3259,7 @@ class raw_hash_set { static const PolicyFunctions& GetPolicyFunctions() { static constexpr PolicyFunctions value = { sizeof(slot_type), - &raw_hash_set::hash_slot_fn, + PolicyTraits::template get_hash_slot_fn(), PolicyTraits::transfer_uses_memcpy() ? TransferRelocatable : &raw_hash_set::transfer_slot_fn, diff --git a/absl/container/internal/raw_hash_set_allocator_test.cc b/absl/container/internal/raw_hash_set_allocator_test.cc index 05dcfaa6..7e7a5063 100644 --- a/absl/container/internal/raw_hash_set_allocator_test.cc +++ b/absl/container/internal/raw_hash_set_allocator_test.cc @@ -25,6 +25,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/base/config.h" +#include "absl/container/internal/container_memory.h" #include "absl/container/internal/raw_hash_set.h" #include "absl/container/internal/tracked.h" @@ -133,7 +134,7 @@ class CheckedAlloc { }; struct Identity { - int32_t operator()(int32_t v) const { return v; } + size_t operator()(int32_t v) const { return static_cast(v); } }; struct Policy { @@ -178,6 +179,11 @@ struct Policy { } static slot_type& element(slot_type* slot) { return *slot; } + + template + static constexpr HashSlotFn get_hash_slot_fn() { + return nullptr; + } }; template diff --git a/absl/container/internal/raw_hash_set_benchmark.cc b/absl/container/internal/raw_hash_set_benchmark.cc index 05a06427..bc8184d4 100644 --- a/absl/container/internal/raw_hash_set_benchmark.cc +++ b/absl/container/internal/raw_hash_set_benchmark.cc @@ -24,6 +24,7 @@ #include #include "absl/base/internal/raw_logging.h" +#include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_function_defaults.h" #include "absl/container/internal/raw_hash_set.h" #include "absl/strings/str_format.h" @@ -59,6 +60,11 @@ struct IntPolicy { static auto apply(F&& f, int64_t x) -> decltype(std::forward(f)(x, x)) { return std::forward(f)(x, x); } + + template + static constexpr HashSlotFn get_hash_slot_fn() { + return nullptr; + } }; class StringPolicy { @@ -117,6 +123,11 @@ class StringPolicy { return apply_impl(std::forward(f), PairArgs(std::forward(args)...)); } + + template + static constexpr HashSlotFn get_hash_slot_fn() { + return nullptr; + } }; struct StringHash : container_internal::hash_default_hash { diff --git a/absl/container/internal/raw_hash_set_probe_benchmark.cc b/absl/container/internal/raw_hash_set_probe_benchmark.cc index 5d4184b2..8f36305d 100644 --- a/absl/container/internal/raw_hash_set_probe_benchmark.cc +++ b/absl/container/internal/raw_hash_set_probe_benchmark.cc @@ -70,6 +70,11 @@ struct Policy { -> decltype(std::forward(f)(arg, arg)) { return std::forward(f)(arg, arg); } + + template + static constexpr auto get_hash_slot_fn() { + return nullptr; + } }; absl::BitGen& GlobalBitGen() { diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 6baaa060..7ec72b22 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -405,6 +405,11 @@ struct ValuePolicy { return absl::container_internal::DecomposeValue( std::forward(f), std::forward(args)...); } + + template + static constexpr HashSlotFn get_hash_slot_fn() { + return nullptr; + } }; using IntPolicy = ValuePolicy; @@ -468,6 +473,11 @@ class StringPolicy { return apply_impl(std::forward(f), PairArgs(std::forward(args)...)); } + + template + static constexpr HashSlotFn get_hash_slot_fn() { + return nullptr; + } }; struct StringHash : absl::Hash { @@ -923,6 +933,11 @@ struct DecomposePolicy { static auto apply(F&& f, const T& x) -> decltype(std::forward(f)(x, x)) { return std::forward(f)(x, x); } + + template + static constexpr HashSlotFn get_hash_slot_fn() { + return nullptr; + } }; template @@ -1083,7 +1098,7 @@ size_t MaxDensitySize(size_t n) { } struct Modulo1000Hash { - size_t operator()(int x) const { return x % 1000; } + size_t operator()(int64_t x) const { return static_cast(x) % 1000; } }; struct Modulo1000HashTable @@ -2664,7 +2679,7 @@ using RawHashSetAlloc = raw_hash_set, TEST(Table, AllocatorPropagation) { TestAllocPropagation(); } struct CountedHash { - size_t operator()(int value) const { + size_t operator()(int64_t value) const { ++count; return static_cast(value); } diff --git a/absl/container/node_hash_map.h b/absl/container/node_hash_map.h index a396de2e..72e78958 100644 --- a/absl/container/node_hash_map.h +++ b/absl/container/node_hash_map.h @@ -36,6 +36,7 @@ #ifndef ABSL_CONTAINER_NODE_HASH_MAP_H_ #define ABSL_CONTAINER_NODE_HASH_MAP_H_ +#include #include #include #include @@ -590,6 +591,13 @@ class NodeHashMapPolicy static Value& value(value_type* elem) { return elem->second; } static const Value& value(const value_type* elem) { return elem->second; } + + template + static constexpr HashSlotFn get_hash_slot_fn() { + return memory_internal::IsLayoutCompatible::value + ? &TypeErasedDerefAndApplyToSlotFn + : nullptr; + } }; } // namespace container_internal diff --git a/absl/container/node_hash_set.h b/absl/container/node_hash_set.h index 421ff460..ec9ab519 100644 --- a/absl/container/node_hash_set.h +++ b/absl/container/node_hash_set.h @@ -35,10 +35,12 @@ #ifndef ABSL_CONTAINER_NODE_HASH_SET_H_ #define ABSL_CONTAINER_NODE_HASH_SET_H_ +#include #include #include "absl/algorithm/container.h" #include "absl/base/macros.h" +#include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export #include "absl/container/internal/node_slot_policy.h" #include "absl/container/internal/raw_hash_set.h" // IWYU pragma: export @@ -487,6 +489,11 @@ struct NodeHashSetPolicy } static size_t element_space_used(const T*) { return sizeof(T); } + + template + static constexpr HashSlotFn get_hash_slot_fn() { + return &TypeErasedDerefAndApplyToSlotFn; + } }; } // namespace container_internal -- cgit v1.2.3 From 7339447a7f457f1d8efa6322c971e71afc304d32 Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Thu, 1 Feb 2024 11:43:33 -0800 Subject: Optimize raw_hash_set destructor. There are three optimizations here: 1. Early exit in case all slots were destroyed. especially useful for empty tables. 2. MatchFull is used in order to iterate over all full slots. 3. Portable group is used for `MatchFull` in the case of small table. PiperOrigin-RevId: 603434899 Change-Id: I40bc90d17331d579cfbb1b4e0693f0913e5c38e4 --- absl/container/internal/raw_hash_set.h | 70 +++++++++++++++++++++------- absl/container/internal/raw_hash_set_test.cc | 35 ++++++++++++++ 2 files changed, 87 insertions(+), 18 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 3b793825..0bb77bda 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -822,21 +822,21 @@ struct GroupPortableImpl { #ifdef ABSL_INTERNAL_HAVE_SSE2 using Group = GroupSse2Impl; -using GroupEmptyOrDeleted = GroupSse2Impl; +using GroupFullEmptyOrDeleted = GroupSse2Impl; #elif defined(ABSL_INTERNAL_HAVE_ARM_NEON) && defined(ABSL_IS_LITTLE_ENDIAN) using Group = GroupAArch64Impl; // For Aarch64, we use the portable implementation for counting and masking -// empty or deleted group elements. This is to avoid the latency of moving +// full, empty or deleted group elements. This is to avoid the latency of moving // between data GPRs and Neon registers when it does not provide a benefit. // Using Neon is profitable when we call Match(), but is not when we don't, -// which is the case when we do *EmptyOrDeleted operations. It is difficult to -// make a similar approach beneficial on other architectures such as x86 since -// they have much lower GPR <-> vector register transfer latency and 16-wide -// Groups. -using GroupEmptyOrDeleted = GroupPortableImpl; +// which is the case when we do *EmptyOrDeleted and MaskFull operations. +// It is difficult to make a similar approach beneficial on other architectures +// such as x86 since they have much lower GPR <-> vector register transfer +// latency and 16-wide Groups. +using GroupFullEmptyOrDeleted = GroupPortableImpl; #else using Group = GroupPortableImpl; -using GroupEmptyOrDeleted = GroupPortableImpl; +using GroupFullEmptyOrDeleted = GroupPortableImpl; #endif // When there is an insertion with no reserved growth, we rehash with @@ -1463,7 +1463,7 @@ inline FindInfo find_first_non_full(const CommonFields& common, size_t hash) { auto seq = probe(common, hash); const ctrl_t* ctrl = common.control(); while (true) { - GroupEmptyOrDeleted g{ctrl + seq.offset()}; + GroupFullEmptyOrDeleted g{ctrl + seq.offset()}; auto mask = g.MaskEmptyOrDeleted(); if (mask) { #if !defined(NDEBUG) @@ -1545,6 +1545,44 @@ inline void* SlotAddress(void* slot_array, size_t slot, size_t slot_size) { (slot * slot_size)); } +// Iterates over all full slots and calls `cb(SlotType*)`. +// NOTE: no erasure from this table allowed during Callback call. +template +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots( + const CommonFields& c, SlotType* slot, Callback cb) { + const size_t cap = c.capacity(); + const ctrl_t* ctrl = c.control(); + if (is_small(cap)) { + // Mirrored/cloned control bytes in small table are also located in the + // first group (starting from position 0). We are taking group from position + // `capacity` in order to avoid duplicates. + + // Small tables capacity fits into portable group, where + // GroupPortableImpl::MaskFull is more efficient for the + // capacity <= GroupPortableImpl::kWidth. + assert(cap <= GroupPortableImpl::kWidth && + "unexpectedly large small capacity"); + static_assert(Group::kWidth >= GroupPortableImpl::kWidth, + "unexpected group width"); + // Group starts from kSentinel slot, so indices in the mask will + // be increased by 1. + --slot; + for (uint32_t i : GroupPortableImpl(ctrl + cap).MaskFull()) { + cb(slot + i); + } + return; + } + size_t remaining = c.size(); + while (remaining != 0) { + for (uint32_t i : GroupFullEmptyOrDeleted(ctrl).MaskFull()) { + cb(slot + i); + --remaining; + } + slot += Group::kWidth; + ctrl += Group::kWidth; + } +} + // Helper class to perform resize of the hash set. // // It contains special optimizations for small group resizes. @@ -2028,7 +2066,7 @@ class raw_hash_set { void skip_empty_or_deleted() { while (IsEmptyOrDeleted(*ctrl_)) { uint32_t shift = - GroupEmptyOrDeleted{ctrl_}.CountLeadingEmptyOrDeleted(); + GroupFullEmptyOrDeleted{ctrl_}.CountLeadingEmptyOrDeleted(); ctrl_ += shift; slot_ += shift; } @@ -2889,14 +2927,10 @@ class raw_hash_set { inline void destroy_slots() { if (PolicyTraits::template destroy_is_trivial()) return; - const size_t cap = capacity(); - const ctrl_t* ctrl = control(); - slot_type* slot = slot_array(); - for (size_t i = 0; i != cap; ++i) { - if (IsFull(ctrl[i])) { - destroy(slot + i); - } - } + IterateOverFullSlots(common(), slot_array(), + [&](slot_type* slot) ABSL_ATTRIBUTE_ALWAYS_INLINE { + this->destroy(slot); + }); } inline void dealloc() { diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 7ec72b22..5852904f 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -63,6 +63,10 @@ ABSL_NAMESPACE_BEGIN namespace container_internal { struct RawHashSetTestOnlyAccess { + template + static auto GetCommon(const C& c) -> decltype(c.common()) { + return c.common(); + } template static auto GetSlots(const C& c) -> decltype(c.slot_array()) { return c.slot_array(); @@ -2741,6 +2745,37 @@ TEST(Table, CountedHash) { } } +TEST(Table, IterateOverFullSlotsEmpty) { + IntTable t; + auto fail_if_any = [](int64_t* i) { FAIL() << "expected no slots " << i; }; + container_internal::IterateOverFullSlots( + RawHashSetTestOnlyAccess::GetCommon(t), + RawHashSetTestOnlyAccess::GetSlots(t), fail_if_any); + for (size_t i = 0; i < 256; ++i) { + t.reserve(i); + container_internal::IterateOverFullSlots( + RawHashSetTestOnlyAccess::GetCommon(t), + RawHashSetTestOnlyAccess::GetSlots(t), fail_if_any); + } +} + +TEST(Table, IterateOverFullSlotsFull) { + IntTable t; + + std::vector expected_slots; + for (int64_t i = 0; i < 128; ++i) { + t.insert(i); + expected_slots.push_back(i); + + std::vector slots; + container_internal::IterateOverFullSlots( + RawHashSetTestOnlyAccess::GetCommon(t), + RawHashSetTestOnlyAccess::GetSlots(t), + [&slots](int64_t* i) { slots.push_back(*i); }); + EXPECT_THAT(slots, testing::UnorderedElementsAreArray(expected_slots)); + } +} + } // namespace } // namespace container_internal ABSL_NAMESPACE_END -- cgit v1.2.3 From ddcf8be90575d494e40fcd4e0c408f0237efe0da Mon Sep 17 00:00:00 2001 From: Dennis Kormalev Date: Mon, 5 Feb 2024 10:56:44 -0800 Subject: Enable StringLikeTest in hash_function_defaults_test PiperOrigin-RevId: 604369517 Change-Id: I6024a8828563c5a2487ba85ede91a88d7059f9c8 --- absl/container/internal/hash_function_defaults_test.cc | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/hash_function_defaults_test.cc b/absl/container/internal/hash_function_defaults_test.cc index c31af3be..7da0c86a 100644 --- a/absl/container/internal/hash_function_defaults_test.cc +++ b/absl/container/internal/hash_function_defaults_test.cc @@ -476,27 +476,25 @@ struct StringLikeTest : public ::testing::Test { hash_default_hash hash; }; -TYPED_TEST_SUITE_P(StringLikeTest); +TYPED_TEST_SUITE(StringLikeTest, StringTypesCartesianProduct); -TYPED_TEST_P(StringLikeTest, Eq) { +TYPED_TEST(StringLikeTest, Eq) { EXPECT_TRUE(this->eq(this->a1, this->b1)); EXPECT_TRUE(this->eq(this->b1, this->a1)); } -TYPED_TEST_P(StringLikeTest, NotEq) { +TYPED_TEST(StringLikeTest, NotEq) { EXPECT_FALSE(this->eq(this->a1, this->b2)); EXPECT_FALSE(this->eq(this->b2, this->a1)); } -TYPED_TEST_P(StringLikeTest, HashEq) { +TYPED_TEST(StringLikeTest, HashEq) { EXPECT_EQ(this->hash(this->a1), this->hash(this->b1)); EXPECT_EQ(this->hash(this->a2), this->hash(this->b2)); // It would be a poor hash function which collides on these strings. EXPECT_NE(this->hash(this->a1), this->hash(this->b2)); } -TYPED_TEST_SUITE(StringLikeTest, StringTypesCartesianProduct); - } // namespace } // namespace container_internal ABSL_NAMESPACE_END -- cgit v1.2.3 From 643b48a3b4362a932c0e41afce62deb55adf825b Mon Sep 17 00:00:00 2001 From: Dennis Kormalev Date: Wed, 7 Feb 2024 08:39:23 -0800 Subject: Add absl_container_hash-based HashEq specialization SwissTable provides support for heterogeneous lookup in associative containers through transparent Hash and Eq types. However, it is not possible for user types to provide additional specializations to allow their types to use this functionality. This CL brings ability for user types to specify their own transparent absl_container_hash and (optionally) absl_container_eq inner types to achieve the same functionality. PiperOrigin-RevId: 604994859 Change-Id: I302486d292c9a18b7d4c77033227008f5539e354 --- absl/container/BUILD.bazel | 4 + absl/container/CMakeLists.txt | 4 + absl/container/flat_hash_map.h | 15 ++- absl/container/flat_hash_set.h | 15 ++- absl/container/internal/hash_function_defaults.h | 69 +++++++++- .../internal/hash_function_defaults_test.cc | 139 ++++++++++++++++++++- absl/container/node_hash_map.h | 15 ++- absl/container/node_hash_set.h | 15 ++- 8 files changed, 270 insertions(+), 6 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/BUILD.bazel b/absl/container/BUILD.bazel index 5160ccea..0de45263 100644 --- a/absl/container/BUILD.bazel +++ b/absl/container/BUILD.bazel @@ -423,8 +423,10 @@ cc_library( "//visibility:private", ], deps = [ + ":common", "//absl/base:config", "//absl/hash", + "//absl/meta:type_traits", "//absl/strings", "//absl/strings:cord", ], @@ -437,6 +439,8 @@ cc_test( linkopts = ABSL_DEFAULT_LINKOPTS, tags = NOTEST_TAGS_MOBILE + ["no_test_loonix"], deps = [ + ":flat_hash_map", + ":flat_hash_set", ":hash_function_defaults", "//absl/hash", "//absl/random", diff --git a/absl/container/CMakeLists.txt b/absl/container/CMakeLists.txt index 85af0bd7..4b08e6a3 100644 --- a/absl/container/CMakeLists.txt +++ b/absl/container/CMakeLists.txt @@ -471,9 +471,11 @@ absl_cc_library( ${ABSL_DEFAULT_COPTS} DEPS absl::config + absl::container_common absl::cord absl::hash absl::strings + absl::type_traits PUBLIC ) @@ -487,6 +489,8 @@ absl_cc_test( DEPS absl::cord absl::cord_test_helpers + absl::flat_hash_map + absl::flat_hash_set absl::hash_function_defaults absl::hash absl::random_random diff --git a/absl/container/flat_hash_map.h b/absl/container/flat_hash_map.h index 2a1ad0fb..a33c794f 100644 --- a/absl/container/flat_hash_map.h +++ b/absl/container/flat_hash_map.h @@ -62,7 +62,7 @@ struct FlatHashMapPolicy; // * Requires values that are MoveConstructible // * Supports heterogeneous lookup, through `find()`, `operator[]()` and // `insert()`, provided that the map is provided a compatible heterogeneous -// hashing function and equality operator. +// hashing function and equality operator. See below for details. // * Invalidates any references and pointers to elements within the table after // `rehash()` and when the table is moved. // * Contains a `capacity()` member function indicating the number of element @@ -80,6 +80,19 @@ struct FlatHashMapPolicy; // libraries (e.g. .dll, .so) is unsupported due to way `absl::Hash` values may // be randomized across dynamically loaded libraries. // +// To achieve heterogeneous lookup for custom types either `Hash` and `Eq` type +// parameters can be used or `T` should have public inner types +// `absl_container_hash` and (optionally) `absl_container_eq`. In either case, +// `typename Hash::is_transparent` and `typename Eq::is_transparent` should be +// well-formed. Both types are basically functors: +// * `Hash` should support `size_t operator()(U val) const` that returns a hash +// for the given `val`. +// * `Eq` should support `bool operator()(U lhs, V rhs) const` that returns true +// if `lhs` is equal to `rhs`. +// +// In most cases `T` needs only to provide the `absl_container_hash`. In this +// case `std::equal_to` will be used instead of `eq` part. +// // NOTE: A `flat_hash_map` stores its value types directly inside its // implementation array to avoid memory indirection. Because a `flat_hash_map` // is designed to move data when rehashed, map values will not retain pointer diff --git a/absl/container/flat_hash_set.h b/absl/container/flat_hash_set.h index 1be00195..5f72f954 100644 --- a/absl/container/flat_hash_set.h +++ b/absl/container/flat_hash_set.h @@ -60,7 +60,7 @@ struct FlatHashSetPolicy; // * Requires keys that are CopyConstructible // * Supports heterogeneous lookup, through `find()` and `insert()`, provided // that the set is provided a compatible heterogeneous hashing function and -// equality operator. +// equality operator. See below for details. // * Invalidates any references and pointers to elements within the table after // `rehash()` and when the table is moved. // * Contains a `capacity()` member function indicating the number of element @@ -78,6 +78,19 @@ struct FlatHashSetPolicy; // libraries (e.g. .dll, .so) is unsupported due to way `absl::Hash` values may // be randomized across dynamically loaded libraries. // +// To achieve heterogeneous lookup for custom types either `Hash` and `Eq` type +// parameters can be used or `T` should have public inner types +// `absl_container_hash` and (optionally) `absl_container_eq`. In either case, +// `typename Hash::is_transparent` and `typename Eq::is_transparent` should be +// well-formed. Both types are basically functors: +// * `Hash` should support `size_t operator()(U val) const` that returns a hash +// for the given `val`. +// * `Eq` should support `bool operator()(U lhs, V rhs) const` that returns true +// if `lhs` is equal to `rhs`. +// +// In most cases `T` needs only to provide the `absl_container_hash`. In this +// case `std::equal_to` will be used instead of `eq` part. +// // NOTE: A `flat_hash_set` stores its keys directly inside its implementation // array to avoid memory indirection. Because a `flat_hash_set` is designed to // move data when rehashed, set keys will not retain pointer stability. If you diff --git a/absl/container/internal/hash_function_defaults.h b/absl/container/internal/hash_function_defaults.h index a3613b4b..0f07bcfe 100644 --- a/absl/container/internal/hash_function_defaults.h +++ b/absl/container/internal/hash_function_defaults.h @@ -45,14 +45,16 @@ #ifndef ABSL_CONTAINER_INTERNAL_HASH_FUNCTION_DEFAULTS_H_ #define ABSL_CONTAINER_INTERNAL_HASH_FUNCTION_DEFAULTS_H_ -#include #include +#include #include #include #include #include "absl/base/config.h" +#include "absl/container/internal/common.h" #include "absl/hash/hash.h" +#include "absl/meta/type_traits.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" @@ -188,6 +190,71 @@ struct HashEq> : HashEq {}; template struct HashEq> : HashEq {}; +template +struct HasAbslContainerHash : std::false_type {}; + +template +struct HasAbslContainerHash> + : std::true_type {}; + +template +struct HasAbslContainerEq : std::false_type {}; + +template +struct HasAbslContainerEq> + : std::true_type {}; + +template +struct AbslContainerEq { + using type = std::equal_to<>; +}; + +template +struct AbslContainerEq< + T, typename std::enable_if_t::value>> { + using type = typename T::absl_container_eq; +}; + +template +struct AbslContainerHash { + using type = void; +}; + +template +struct AbslContainerHash< + T, typename std::enable_if_t::value>> { + using type = typename T::absl_container_hash; +}; + +// HashEq specialization for user types that provide `absl_container_hash` and +// (optionally) `absl_container_eq`. This specialization allows user types to +// provide heterogeneous lookup without requiring to explicitly specify Hash/Eq +// type arguments in unordered Abseil containers. +// +// Both `absl_container_hash` and `absl_container_eq` should be transparent +// (have inner is_transparent type). While there is no technical reason to +// restrict to transparent-only types, there is also no feasible use case when +// it shouldn't be transparent - it is easier to relax the requirement later if +// such a case arises rather than restricting it. +// +// If type provides only `absl_container_hash` then `eq` part will be +// `std::equal_to`. +// +// User types are not allowed to provide only a `Eq` part as there is no +// feasible use case for this behavior - if Hash should be a default one then Eq +// should be an equivalent to the `std::equal_to`. +template +struct HashEq::value>> { + using Hash = typename AbslContainerHash::type; + using Eq = typename AbslContainerEq::type; + static_assert(IsTransparent::value, + "absl_container_hash must be transparent. To achieve it add a " + "`using is_transparent = void;` clause to this type."); + static_assert(IsTransparent::value, + "absl_container_eq must be transparent. To achieve it add a " + "`using is_transparent = void;` clause to this type."); +}; + // This header's visibility is restricted. If you need to access the default // hasher please use the container's ::hasher alias instead. // diff --git a/absl/container/internal/hash_function_defaults_test.cc b/absl/container/internal/hash_function_defaults_test.cc index 7da0c86a..912d1190 100644 --- a/absl/container/internal/hash_function_defaults_test.cc +++ b/absl/container/internal/hash_function_defaults_test.cc @@ -14,11 +14,15 @@ #include "absl/container/internal/hash_function_defaults.h" +#include #include #include #include #include "gtest/gtest.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/hash/hash.h" #include "absl/random/random.h" #include "absl/strings/cord.h" #include "absl/strings/cord_test_helpers.h" @@ -495,13 +499,146 @@ TYPED_TEST(StringLikeTest, HashEq) { EXPECT_NE(this->hash(this->a1), this->hash(this->b2)); } +struct TypeWithAbslContainerHash { + struct absl_container_hash { + using is_transparent = void; + + size_t operator()(const TypeWithAbslContainerHash& foo) const { + return absl::HashOf(foo.value); + } + + // Extra overload to test that heterogeneity works for this hasher. + size_t operator()(int value) const { return absl::HashOf(value); } + }; + + friend bool operator==(const TypeWithAbslContainerHash& lhs, + const TypeWithAbslContainerHash& rhs) { + return lhs.value == rhs.value; + } + + friend bool operator==(const TypeWithAbslContainerHash& lhs, int rhs) { + return lhs.value == rhs; + } + + int value; + int noise; +}; + +struct TypeWithAbslContainerHashAndEq { + struct absl_container_hash { + using is_transparent = void; + + size_t operator()(const TypeWithAbslContainerHashAndEq& foo) const { + return absl::HashOf(foo.value); + } + + // Extra overload to test that heterogeneity works for this hasher. + size_t operator()(int value) const { return absl::HashOf(value); } + }; + + struct absl_container_eq { + using is_transparent = void; + + bool operator()(const TypeWithAbslContainerHashAndEq& lhs, + const TypeWithAbslContainerHashAndEq& rhs) const { + return lhs.value == rhs.value; + } + + // Extra overload to test that heterogeneity works for this eq. + bool operator()(const TypeWithAbslContainerHashAndEq& lhs, int rhs) const { + return lhs.value == rhs; + } + }; + + template + bool operator==(T&& other) const = delete; + + int value; + int noise; +}; + +using AbslContainerHashTypes = + Types; + +template +using AbslContainerHashTest = ::testing::Test; + +TYPED_TEST_SUITE(AbslContainerHashTest, AbslContainerHashTypes); + +TYPED_TEST(AbslContainerHashTest, HasherWorks) { + hash_default_hash hasher; + + TypeParam foo1{/*value=*/1, /*noise=*/100}; + TypeParam foo1_copy{/*value=*/1, /*noise=*/20}; + TypeParam foo2{/*value=*/2, /*noise=*/100}; + + EXPECT_EQ(hasher(foo1), absl::HashOf(1)); + EXPECT_EQ(hasher(foo2), absl::HashOf(2)); + EXPECT_EQ(hasher(foo1), hasher(foo1_copy)); + + // Heterogeneity works. + EXPECT_EQ(hasher(foo1), hasher(1)); + EXPECT_EQ(hasher(foo2), hasher(2)); +} + +TYPED_TEST(AbslContainerHashTest, EqWorks) { + hash_default_eq eq; + + TypeParam foo1{/*value=*/1, /*noise=*/100}; + TypeParam foo1_copy{/*value=*/1, /*noise=*/20}; + TypeParam foo2{/*value=*/2, /*noise=*/100}; + + EXPECT_TRUE(eq(foo1, foo1_copy)); + EXPECT_FALSE(eq(foo1, foo2)); + + // Heterogeneity works. + EXPECT_TRUE(eq(foo1, 1)); + EXPECT_FALSE(eq(foo1, 2)); +} + +TYPED_TEST(AbslContainerHashTest, HeterogeneityInMapWorks) { + absl::flat_hash_map map; + + TypeParam foo1{/*value=*/1, /*noise=*/100}; + TypeParam foo1_copy{/*value=*/1, /*noise=*/20}; + TypeParam foo2{/*value=*/2, /*noise=*/100}; + TypeParam foo3{/*value=*/3, /*noise=*/100}; + + map[foo1] = 1; + map[foo2] = 2; + + EXPECT_TRUE(map.contains(foo1_copy)); + EXPECT_EQ(map.at(foo1_copy), 1); + EXPECT_TRUE(map.contains(1)); + EXPECT_EQ(map.at(1), 1); + EXPECT_TRUE(map.contains(2)); + EXPECT_EQ(map.at(2), 2); + EXPECT_FALSE(map.contains(foo3)); + EXPECT_FALSE(map.contains(3)); +} + +TYPED_TEST(AbslContainerHashTest, HeterogeneityInSetWorks) { + absl::flat_hash_set set; + + TypeParam foo1{/*value=*/1, /*noise=*/100}; + TypeParam foo1_copy{/*value=*/1, /*noise=*/20}; + TypeParam foo2{/*value=*/2, /*noise=*/100}; + + set.insert(foo1); + + EXPECT_TRUE(set.contains(foo1_copy)); + EXPECT_TRUE(set.contains(1)); + EXPECT_FALSE(set.contains(foo2)); + EXPECT_FALSE(set.contains(2)); +} + } // namespace } // namespace container_internal ABSL_NAMESPACE_END } // namespace absl enum Hash : size_t { - kStd = 0x1, // std::hash + kStd = 0x1, // std::hash #ifdef _MSC_VER kExtension = kStd, // In MSVC, std::hash == ::hash #else // _MSC_VER diff --git a/absl/container/node_hash_map.h b/absl/container/node_hash_map.h index 72e78958..cb41543c 100644 --- a/absl/container/node_hash_map.h +++ b/absl/container/node_hash_map.h @@ -67,7 +67,7 @@ class NodeHashMapPolicy; // // * Supports heterogeneous lookup, through `find()`, `operator[]()` and // `insert()`, provided that the map is provided a compatible heterogeneous -// hashing function and equality operator. +// hashing function and equality operator. See below for details. // * Contains a `capacity()` member function indicating the number of element // slots (open, deleted, and empty) within the hash map. // * Returns `void` from the `erase(iterator)` overload. @@ -83,6 +83,19 @@ class NodeHashMapPolicy; // libraries (e.g. .dll, .so) is unsupported due to way `absl::Hash` values may // be randomized across dynamically loaded libraries. // +// To achieve heterogeneous lookup for custom types either `Hash` and `Eq` type +// parameters can be used or `T` should have public inner types +// `absl_container_hash` and (optionally) `absl_container_eq`. In either case, +// `typename Hash::is_transparent` and `typename Eq::is_transparent` should be +// well-formed. Both types are basically functors: +// * `Hash` should support `size_t operator()(U val) const` that returns a hash +// for the given `val`. +// * `Eq` should support `bool operator()(U lhs, V rhs) const` that returns true +// if `lhs` is equal to `rhs`. +// +// In most cases `T` needs only to provide the `absl_container_hash`. In this +// case `std::equal_to` will be used instead of `eq` part. +// // Example: // // // Create a node hash map of three strings (that map to strings) diff --git a/absl/container/node_hash_set.h b/absl/container/node_hash_set.h index ec9ab519..8cc4b624 100644 --- a/absl/container/node_hash_set.h +++ b/absl/container/node_hash_set.h @@ -64,7 +64,7 @@ struct NodeHashSetPolicy; // // * Supports heterogeneous lookup, through `find()`, `operator[]()` and // `insert()`, provided that the set is provided a compatible heterogeneous -// hashing function and equality operator. +// hashing function and equality operator. See below for details. // * Contains a `capacity()` member function indicating the number of element // slots (open, deleted, and empty) within the hash set. // * Returns `void` from the `erase(iterator)` overload. @@ -80,6 +80,19 @@ struct NodeHashSetPolicy; // libraries (e.g. .dll, .so) is unsupported due to way `absl::Hash` values may // be randomized across dynamically loaded libraries. // +// To achieve heterogeneous lookup for custom types either `Hash` and `Eq` type +// parameters can be used or `T` should have public inner types +// `absl_container_hash` and (optionally) `absl_container_eq`. In either case, +// `typename Hash::is_transparent` and `typename Eq::is_transparent` should be +// well-formed. Both types are basically functors: +// * `Hash` should support `size_t operator()(U val) const` that returns a hash +// for the given `val`. +// * `Eq` should support `bool operator()(U lhs, V rhs) const` that returns true +// if `lhs` is equal to `rhs`. +// +// In most cases `T` needs only to provide the `absl_container_hash`. In this +// case `std::equal_to` will be used instead of `eq` part. +// // Example: // // // Create a node hash set of three strings -- cgit v1.2.3 From 0be9f99723eba44462245013d6a433c1ad9157ee Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Wed, 7 Feb 2024 15:26:28 -0800 Subject: Avoid hash computation and `Group::Match` in small tables copy and use `IterateOverFullSlots` for iterating for all tables. PiperOrigin-RevId: 605116090 Change-Id: Ia65c664421f7630225b00f1c45c636b4681121ce --- absl/container/internal/raw_hash_set.h | 78 +++++++++++++++++++++------- absl/container/internal/raw_hash_set_test.cc | 46 +++++++++++++++- 2 files changed, 102 insertions(+), 22 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 0bb77bda..dced5b2b 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -1545,7 +1545,7 @@ inline void* SlotAddress(void* slot_array, size_t slot, size_t slot_size) { (slot * slot_size)); } -// Iterates over all full slots and calls `cb(SlotType*)`. +// Iterates over all full slots and calls `cb(const ctrl_t*, SlotType*)`. // NOTE: no erasure from this table allowed during Callback call. template ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots( @@ -1566,16 +1566,18 @@ ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots( "unexpected group width"); // Group starts from kSentinel slot, so indices in the mask will // be increased by 1. + const auto mask = GroupPortableImpl(ctrl + cap).MaskFull(); + --ctrl; --slot; - for (uint32_t i : GroupPortableImpl(ctrl + cap).MaskFull()) { - cb(slot + i); + for (uint32_t i : mask) { + cb(ctrl + i, slot + i); } return; } size_t remaining = c.size(); while (remaining != 0) { for (uint32_t i : GroupFullEmptyOrDeleted(ctrl).MaskFull()) { - cb(slot + i); + cb(ctrl + i, slot + i); --remaining; } slot += Group::kWidth; @@ -2260,19 +2262,55 @@ class raw_hash_set { that.alloc_ref())) {} raw_hash_set(const raw_hash_set& that, const allocator_type& a) - : raw_hash_set(0, that.hash_ref(), that.eq_ref(), a) { + : raw_hash_set(GrowthToLowerboundCapacity(that.size()), that.hash_ref(), + that.eq_ref(), a) { const size_t size = that.size(); - if (size == 0) return; - reserve(size); - // Because the table is guaranteed to be empty, we can do something faster - // than a full `insert`. - for (const auto& v : that) { - const size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, v); - auto target = find_first_non_full_outofline(common(), hash); - SetCtrl(common(), target.offset, H2(hash), sizeof(slot_type)); - emplace_at(target.offset, v); - common().maybe_increment_generation_on_insert(); - infoz().RecordInsert(hash, target.probe_length); + if (size == 0) { + return; + } + const size_t cap = capacity(); + // Note about single group tables: + // 1. It is correct to have any order of elements. + // 2. Order has to be non deterministic. + // 3. We are assigning elements with arbitrary `shift` starting from + // `capacity + shift` position. + // 4. `shift` must be coprime with `capacity + 1` in order to be able to use + // modular arithmetic to traverse all positions, instead if cycling + // through a subset of positions. Odd numbers are coprime with any + // `capacity + 1` (2^N). + size_t offset = cap; + const size_t shift = + is_single_group(cap) ? (PerTableSalt(control()) | 1) : 0; + IterateOverFullSlots( + that.common(), that.slot_array(), + [&](const ctrl_t* that_ctrl, + slot_type* that_slot) ABSL_ATTRIBUTE_ALWAYS_INLINE { + if (shift == 0) { + // Big tables case. Position must be searched via probing. + // The table is guaranteed to be empty, so we can do faster than + // a full `insert`. + const size_t hash = PolicyTraits::apply( + HashElement{hash_ref()}, PolicyTraits::element(that_slot)); + FindInfo target = find_first_non_full_outofline(common(), hash); + infoz().RecordInsert(hash, target.probe_length); + offset = target.offset; + } else { + // Small tables case. Next position is computed via shift. + offset = (offset + shift) & cap; + } + const h2_t h2 = static_cast(*that_ctrl); + assert( // We rely that hash is not changed for small tables. + H2(PolicyTraits::apply(HashElement{hash_ref()}, + PolicyTraits::element(that_slot))) == h2 && + "hash function value changed unexpectedly during the copy"); + SetCtrl(common(), offset, h2, sizeof(slot_type)); + emplace_at(offset, PolicyTraits::element(that_slot)); + common().maybe_increment_generation_on_insert(); + }); + if (shift != 0) { + // On small table copy we do not record individual inserts. + // RecordInsert requires hash, but it is unknown for small tables. + infoz().RecordStorageChanged(size, cap); } common().set_size(size); set_growth_left(growth_left() - size); @@ -2927,10 +2965,10 @@ class raw_hash_set { inline void destroy_slots() { if (PolicyTraits::template destroy_is_trivial()) return; - IterateOverFullSlots(common(), slot_array(), - [&](slot_type* slot) ABSL_ATTRIBUTE_ALWAYS_INLINE { - this->destroy(slot); - }); + IterateOverFullSlots( + common(), slot_array(), + [&](const ctrl_t*, slot_type* slot) + ABSL_ATTRIBUTE_ALWAYS_INLINE { this->destroy(slot); }); } inline void dealloc() { diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 5852904f..1c428ea7 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -1765,6 +1765,40 @@ TEST(Table, CopyConstruct) { } } +TEST(Table, CopyDifferentSizes) { + IntTable t; + + for (int i = 0; i < 100; ++i) { + t.emplace(i); + IntTable c = t; + for (int j = 0; j <= i; ++j) { + ASSERT_TRUE(c.find(j) != c.end()) << "i=" << i << " j=" << j; + } + // Testing find miss to verify that table is not full. + ASSERT_TRUE(c.find(-1) == c.end()); + } +} + +TEST(Table, CopyDifferentCapacities) { + for (int cap = 1; cap < 100; cap = cap * 2 + 1) { + IntTable t; + t.reserve(static_cast(cap)); + for (int i = 0; i <= cap; ++i) { + t.emplace(i); + if (i != cap && i % 5 != 0) { + continue; + } + IntTable c = t; + for (int j = 0; j <= i; ++j) { + ASSERT_TRUE(c.find(j) != c.end()) + << "cap=" << cap << " i=" << i << " j=" << j; + } + // Testing find miss to verify that table is not full. + ASSERT_TRUE(c.find(-1) == c.end()); + } + } +} + TEST(Table, CopyConstructWithAlloc) { StringTable t; t.emplace("a", "b"); @@ -2747,7 +2781,9 @@ TEST(Table, CountedHash) { TEST(Table, IterateOverFullSlotsEmpty) { IntTable t; - auto fail_if_any = [](int64_t* i) { FAIL() << "expected no slots " << i; }; + auto fail_if_any = [](const ctrl_t*, int64_t* i) { + FAIL() << "expected no slots " << i; + }; container_internal::IterateOverFullSlots( RawHashSetTestOnlyAccess::GetCommon(t), RawHashSetTestOnlyAccess::GetSlots(t), fail_if_any); @@ -2771,7 +2807,13 @@ TEST(Table, IterateOverFullSlotsFull) { container_internal::IterateOverFullSlots( RawHashSetTestOnlyAccess::GetCommon(t), RawHashSetTestOnlyAccess::GetSlots(t), - [&slots](int64_t* i) { slots.push_back(*i); }); + [&t, &slots](const ctrl_t* ctrl, int64_t* i) { + ptrdiff_t ctrl_offset = + ctrl - RawHashSetTestOnlyAccess::GetCommon(t).control(); + ptrdiff_t slot_offset = i - RawHashSetTestOnlyAccess::GetSlots(t); + ASSERT_EQ(ctrl_offset, slot_offset); + slots.push_back(*i); + }); EXPECT_THAT(slots, testing::UnorderedElementsAreArray(expected_slots)); } } -- cgit v1.2.3 From 99f0b6d16f0e9ab6027379c31ce6c99dc1db5e9c Mon Sep 17 00:00:00 2001 From: Matt Kulukundis Date: Wed, 7 Feb 2024 16:02:47 -0800 Subject: Switch rank structs to be consistent with written guidance in go/ranked-overloads PiperOrigin-RevId: 605125821 Change-Id: I2ee260eaf283acafd80abfd2b7419a0e9f597a78 --- absl/container/internal/common_policy_traits.h | 19 ++++++++++--------- absl/strings/cord.h | 2 +- absl/strings/internal/cord_internal.h | 11 ++++++----- 3 files changed, 17 insertions(+), 15 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/common_policy_traits.h b/absl/container/internal/common_policy_traits.h index 77df4790..c521f612 100644 --- a/absl/container/internal/common_policy_traits.h +++ b/absl/container/internal/common_policy_traits.h @@ -64,7 +64,7 @@ struct common_policy_traits { // UNINITIALIZED template static void transfer(Alloc* alloc, slot_type* new_slot, slot_type* old_slot) { - transfer_impl(alloc, new_slot, old_slot, Rank0{}); + transfer_impl(alloc, new_slot, old_slot, Rank2{}); } // PRECONDITION: `slot` is INITIALIZED @@ -83,7 +83,7 @@ struct common_policy_traits { static constexpr bool transfer_uses_memcpy() { return std::is_same>( - nullptr, nullptr, nullptr, Rank0{})), + nullptr, nullptr, nullptr, Rank2{})), std::true_type>::value; } @@ -95,18 +95,19 @@ struct common_policy_traits { } private: - // To rank the overloads below for overload resolution. Rank0 is preferred. - struct Rank2 {}; - struct Rank1 : Rank2 {}; - struct Rank0 : Rank1 {}; + // Use go/ranked-overloads for dispatching. + struct Rank0 {}; + struct Rank1 : Rank0 {}; + struct Rank2 : Rank1 {}; // Use auto -> decltype as an enabler. // P::transfer returns std::true_type if transfer uses memcpy (e.g. in // node_slot_policy). template static auto transfer_impl(Alloc* alloc, slot_type* new_slot, - slot_type* old_slot, Rank0) - -> decltype(P::transfer(alloc, new_slot, old_slot)) { + slot_type* old_slot, + Rank2) -> decltype(P::transfer(alloc, new_slot, + old_slot)) { return P::transfer(alloc, new_slot, old_slot); } #if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606 @@ -129,7 +130,7 @@ struct common_policy_traits { template static void transfer_impl(Alloc* alloc, slot_type* new_slot, - slot_type* old_slot, Rank2) { + slot_type* old_slot, Rank0) { construct(alloc, new_slot, std::move(element(old_slot))); destroy(alloc, old_slot); } diff --git a/absl/strings/cord.h b/absl/strings/cord.h index d2ba9673..2583aa8a 100644 --- a/absl/strings/cord.h +++ b/absl/strings/cord.h @@ -1120,7 +1120,7 @@ Cord MakeCordFromExternal(absl::string_view data, Releaser&& releaser) { } else { using ReleaserType = absl::decay_t; cord_internal::InvokeReleaser( - cord_internal::Rank0{}, ReleaserType(std::forward(releaser)), + cord_internal::Rank1{}, ReleaserType(std::forward(releaser)), data); } return cord; diff --git a/absl/strings/internal/cord_internal.h b/absl/strings/internal/cord_internal.h index 549f9175..a52deea1 100644 --- a/absl/strings/internal/cord_internal.h +++ b/absl/strings/internal/cord_internal.h @@ -352,18 +352,19 @@ struct CordRepExternal : public CordRep { static void Delete(CordRep* rep); }; -struct Rank1 {}; -struct Rank0 : Rank1 {}; +// Use go/ranked-overloads for dispatching. +struct Rank0 {}; +struct Rank1 : Rank0 {}; template > -void InvokeReleaser(Rank0, Releaser&& releaser, absl::string_view data) { +void InvokeReleaser(Rank1, Releaser&& releaser, absl::string_view data) { ::absl::base_internal::invoke(std::forward(releaser), data); } template > -void InvokeReleaser(Rank1, Releaser&& releaser, absl::string_view) { +void InvokeReleaser(Rank0, Releaser&& releaser, absl::string_view) { ::absl::base_internal::invoke(std::forward(releaser)); } @@ -381,7 +382,7 @@ struct CordRepExternalImpl } ~CordRepExternalImpl() { - InvokeReleaser(Rank0{}, std::move(this->template get<0>()), + InvokeReleaser(Rank1{}, std::move(this->template get<0>()), absl::string_view(base, length)); } -- cgit v1.2.3 From 760b21530ffc22ef4859356e543022b18de7d2ce Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Thu, 8 Feb 2024 16:30:05 -0800 Subject: Make `begin()` to return `end()` on empty tables. PiperOrigin-RevId: 605460827 Change-Id: I57007a7ad18829e7bfed27ba65871afbd227d012 --- absl/container/internal/raw_hash_set.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index dced5b2b..d6950e61 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -2027,6 +2027,7 @@ class raw_hash_set { ++ctrl_; ++slot_; skip_empty_or_deleted(); + if (ABSL_PREDICT_FALSE(*ctrl_ == ctrl_t::kSentinel)) ctrl_ = nullptr; return *this; } // PRECONDITION: not an end() iterator. @@ -2061,10 +2062,8 @@ class raw_hash_set { explicit iterator(const GenerationType* generation_ptr) : HashSetIteratorGenerationInfo(generation_ptr), ctrl_(nullptr) {} - // Fixes up `ctrl_` to point to a full by advancing it and `slot_` until - // they reach one. - // - // If a sentinel is reached, we null `ctrl_` out instead. + // Fixes up `ctrl_` to point to a full or sentinel by advancing `ctrl_` and + // `slot_` until they reach one. void skip_empty_or_deleted() { while (IsEmptyOrDeleted(*ctrl_)) { uint32_t shift = @@ -2072,7 +2071,6 @@ class raw_hash_set { ctrl_ += shift; slot_ += shift; } - if (ABSL_PREDICT_FALSE(*ctrl_ == ctrl_t::kSentinel)) ctrl_ = nullptr; } ctrl_t* control() const { return ctrl_; } @@ -2369,8 +2367,11 @@ class raw_hash_set { ~raw_hash_set() { destructor_impl(); } iterator begin() ABSL_ATTRIBUTE_LIFETIME_BOUND { + // TODO(b/324478958): Consider reverting if no impact. + if (ABSL_PREDICT_FALSE(empty())) return end(); auto it = iterator_at(0); it.skip_empty_or_deleted(); + assert(IsFull(*it.control())); return it; } iterator end() ABSL_ATTRIBUTE_LIFETIME_BOUND { -- cgit v1.2.3 From 8a3caf7dea955b513a6c1b572a2423c6b4213402 Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Thu, 15 Feb 2024 23:54:17 -0800 Subject: Introduce `Group::MaskNonFull` without usage. It can be used instead of `MaskEmptyOrDeleted` in case of inserting to empty table. `MaskNonFull` requires less operations, in particular it eliminates `_mm_set1_epi8` and `_mm_cmpgt_epi8` operations. PiperOrigin-RevId: 607587394 Change-Id: Ia48f922d1ca6de38cc91e7ab0d608c45f8f2c446 --- absl/container/internal/raw_hash_set.h | 27 +++++++++++++++++++++++ absl/container/internal/raw_hash_set_benchmark.cc | 22 ++++++++++++++++++ absl/container/internal/raw_hash_set_test.cc | 19 ++++++++++++++++ 3 files changed, 68 insertions(+) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index d6950e61..cef042e7 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -657,6 +657,14 @@ struct GroupSse2Impl { static_cast(_mm_movemask_epi8(ctrl) ^ 0xffff)); } + // Returns a bitmask representing the positions of non full slots. + // Note: this includes: kEmpty, kDeleted, kSentinel. + // It is useful in contexts when kSentinel is not present. + auto MaskNonFull() const { + return BitMask( + static_cast(_mm_movemask_epi8(ctrl))); + } + // Returns a bitmask representing the positions of empty or deleted slots. NonIterableBitMask MaskEmptyOrDeleted() const { auto special = _mm_set1_epi8(static_cast(ctrl_t::kSentinel)); @@ -725,6 +733,18 @@ struct GroupAArch64Impl { /*NullifyBitsOnIteration=*/true>(mask); } + // Returns a bitmask representing the positions of non full slots. + // Note: this includes: kEmpty, kDeleted, kSentinel. + // It is useful in contexts when kSentinel is not present. + auto MaskNonFull() const { + uint64_t mask = vget_lane_u64( + vreinterpret_u64_u8(vclt_s8(vreinterpret_s8_u8(ctrl), + vdup_n_s8(static_cast(0)))), + 0); + return BitMask(mask); + } + NonIterableBitMask MaskEmptyOrDeleted() const { uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(vcgt_s8( @@ -797,6 +817,13 @@ struct GroupPortableImpl { return BitMask((ctrl ^ kMsbs8Bytes) & kMsbs8Bytes); } + // Returns a bitmask representing the positions of non full slots. + // Note: this includes: kEmpty, kDeleted, kSentinel. + // It is useful in contexts when kSentinel is not present. + auto MaskNonFull() const { + return BitMask(ctrl & kMsbs8Bytes); + } + NonIterableBitMask MaskEmptyOrDeleted() const { return NonIterableBitMask((ctrl & ~(ctrl << 7)) & kMsbs8Bytes); diff --git a/absl/container/internal/raw_hash_set_benchmark.cc b/absl/container/internal/raw_hash_set_benchmark.cc index bc8184d4..8cd43fb5 100644 --- a/absl/container/internal/raw_hash_set_benchmark.cc +++ b/absl/container/internal/raw_hash_set_benchmark.cc @@ -479,6 +479,17 @@ void BM_Group_MaskEmptyOrDeleted(benchmark::State& state) { } BENCHMARK(BM_Group_MaskEmptyOrDeleted); +void BM_Group_MaskNonFull(benchmark::State& state) { + std::array group; + Iota(group.begin(), group.end(), -4); + Group g{group.data()}; + for (auto _ : state) { + ::benchmark::DoNotOptimize(g); + ::benchmark::DoNotOptimize(g.MaskNonFull()); + } +} +BENCHMARK(BM_Group_MaskNonFull); + void BM_Group_CountLeadingEmptyOrDeleted(benchmark::State& state) { std::array group; Iota(group.begin(), group.end(), -2); @@ -501,6 +512,17 @@ void BM_Group_MatchFirstEmptyOrDeleted(benchmark::State& state) { } BENCHMARK(BM_Group_MatchFirstEmptyOrDeleted); +void BM_Group_MatchFirstNonFull(benchmark::State& state) { + std::array group; + Iota(group.begin(), group.end(), -2); + Group g{group.data()}; + for (auto _ : state) { + ::benchmark::DoNotOptimize(g); + ::benchmark::DoNotOptimize(g.MaskNonFull().LowestBitSet()); + } +} +BENCHMARK(BM_Group_MatchFirstNonFull); + void BM_DropDeletes(benchmark::State& state) { constexpr size_t capacity = (1 << 20) - 1; std::vector ctrl(capacity + 1 + Group::kWidth); diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 1c428ea7..8e196fdf 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -307,6 +307,25 @@ TEST(Group, MaskFull) { } } +TEST(Group, MaskNonFull) { + if (Group::kWidth == 16) { + ctrl_t group[] = { + ctrl_t::kEmpty, CtrlT(1), ctrl_t::kDeleted, CtrlT(3), + ctrl_t::kEmpty, CtrlT(5), ctrl_t::kSentinel, CtrlT(7), + CtrlT(7), CtrlT(5), ctrl_t::kDeleted, CtrlT(1), + CtrlT(1), ctrl_t::kSentinel, ctrl_t::kEmpty, CtrlT(1)}; + EXPECT_THAT(Group{group}.MaskNonFull(), + ElementsAre(0, 2, 4, 6, 10, 13, 14)); + } else if (Group::kWidth == 8) { + ctrl_t group[] = {ctrl_t::kEmpty, CtrlT(1), ctrl_t::kEmpty, + ctrl_t::kDeleted, CtrlT(2), ctrl_t::kSentinel, + ctrl_t::kSentinel, CtrlT(1)}; + EXPECT_THAT(Group{group}.MaskNonFull(), ElementsAre(0, 2, 3, 5, 6)); + } else { + FAIL() << "No test coverage for Group::kWidth==" << Group::kWidth; + } +} + TEST(Group, MaskEmptyOrDeleted) { if (Group::kWidth == 16) { ctrl_t group[] = {ctrl_t::kEmpty, CtrlT(1), ctrl_t::kEmpty, CtrlT(3), -- cgit v1.2.3 From c28f689cd0c05fd73c9eacda7f3ceb193093304d Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Tue, 20 Feb 2024 14:34:43 -0800 Subject: Use const_cast to avoid duplicating the implementation of raw_hash_set::find(key). Motivation: the implementation becomes more complicated with small object optimization. PiperOrigin-RevId: 608742838 Change-Id: I55fc42321b1967f9c7bbee49817a2f2d4ee44b56 --- absl/container/internal/raw_hash_set.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index cef042e7..7eacfa8c 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -2849,8 +2849,7 @@ class raw_hash_set { template const_iterator find(const key_arg& key) const ABSL_ATTRIBUTE_LIFETIME_BOUND { - prefetch_heap_block(); - return find(key, hash_ref()(key)); + return const_cast(this)->find(key); } template -- cgit v1.2.3 From b0f85e2355b173d3f89dee29a7f817b52f8e72a2 Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Wed, 21 Feb 2024 04:41:13 -0800 Subject: Improve raw_hash_set tests. PiperOrigin-RevId: 608947694 Change-Id: Ie53a91c4d78dcb80c57227616b488ec64b23c588 --- absl/container/internal/raw_hash_set_test.cc | 43 +++++++++++++++++++--------- 1 file changed, 29 insertions(+), 14 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 8e196fdf..109340c7 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -800,10 +800,10 @@ TYPED_TEST_P(SmallTableResizeTest, InsertIntoSmallTable) { } TYPED_TEST_P(SmallTableResizeTest, ResizeGrowSmallTables) { - TypeParam t; for (size_t source_size = 0; source_size < 32; ++source_size) { for (size_t target_size = source_size; target_size < 32; ++target_size) { for (bool rehash : {false, true}) { + TypeParam t; for (size_t i = 0; i < source_size; ++i) { t.insert(static_cast(i)); } @@ -822,9 +822,10 @@ TYPED_TEST_P(SmallTableResizeTest, ResizeGrowSmallTables) { } TYPED_TEST_P(SmallTableResizeTest, ResizeReduceSmallTables) { - TypeParam t; for (size_t source_size = 0; source_size < 32; ++source_size) { for (size_t target_size = 0; target_size <= source_size; ++target_size) { + TypeParam t; + t.reserve(source_size); size_t inserted_count = std::min(source_size, 5); for (size_t i = 0; i < inserted_count; ++i) { t.insert(static_cast(i)); @@ -2282,20 +2283,34 @@ TEST(Table, IterationOrderChangesByInstance) { } TEST(Table, IterationOrderChangesOnRehash) { - std::vector garbage; - for (int i = 0; i < 5000; ++i) { - auto t = MakeSimpleTable(20); - const auto reference = OrderOfIteration(t); - // Force rehash to the same size. - t.rehash(0); - auto trial = OrderOfIteration(t); - if (trial != reference) { - // We are done. - return; + // We test different sizes with many small numbers, because small table + // resize has a different codepath. + // Note: iteration order for size() <= 1 is always the same. + for (size_t size : std::vector{2, 3, 6, 7, 12, 15, 20, 50}) { + for (size_t rehash_size : { + size_t{0}, // Force rehash is guaranteed. + size * 10 // Rehash to the larger capacity is guaranteed. + }) { + std::vector garbage; + bool ok = false; + for (int i = 0; i < 5000; ++i) { + auto t = MakeSimpleTable(size); + const auto reference = OrderOfIteration(t); + // Force rehash. + t.rehash(rehash_size); + auto trial = OrderOfIteration(t); + if (trial != reference) { + // We are done. + ok = true; + break; + } + garbage.push_back(std::move(t)); + } + EXPECT_TRUE(ok) + << "Iteration order remained the same across many attempts " << size + << "->" << rehash_size << "."; } - garbage.push_back(std::move(t)); } - FAIL() << "Iteration order remained the same across many attempts."; } // Verify that pointers are invalidated as soon as a second element is inserted. -- cgit v1.2.3 From 92c8575d24cb5c27718e1cd1ed21a9847351a11b Mon Sep 17 00:00:00 2001 From: Arthur O'Dwyer Date: Wed, 21 Feb 2024 09:17:54 -0800 Subject: PR #1618: inlined_vector: Use trivial relocation for `SwapInlinedElements` Imported from GitHub PR https://github.com/abseil/abseil-cpp/pull/1618 I noticed while working on #1615 that `inlined_vector` could use the trivial relocatability trait here, too. Here the memcpy codepath already exists; we just have to opt in to using it. Merge 567a1dd9b6b3352f649e900b24834b59e39cfa14 into a7012a5bfcf26a41b9dd32d4c429004773503dd6 Merging this change closes #1618 COPYBARA_INTEGRATE_REVIEW=https://github.com/abseil/abseil-cpp/pull/1618 from Quuxplusone:trivial-swap 567a1dd9b6b3352f649e900b24834b59e39cfa14 PiperOrigin-RevId: 609019296 Change-Id: I4055ab790245752179e405b490fcd479e7389726 --- absl/container/inlined_vector_test.cc | 33 +++++++++++++++++++++++++++++++- absl/container/internal/inlined_vector.h | 31 +++++++++++++++--------------- 2 files changed, 47 insertions(+), 17 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/inlined_vector_test.cc b/absl/container/inlined_vector_test.cc index 241389ae..5ecf88a9 100644 --- a/absl/container/inlined_vector_test.cc +++ b/absl/container/inlined_vector_test.cc @@ -304,6 +304,35 @@ TEST(UniquePtr, MoveAssign) { } } +// Swapping containers of unique pointers should work fine, with no +// leaks, despite the fact that unique pointers are trivially relocatable but +// not trivially destructible. +// TODO(absl-team): Using unique_ptr here is technically correct, but +// a trivially relocatable struct would be less semantically confusing. +TEST(UniquePtr, Swap) { + for (size_t size1 = 0; size1 < 5; ++size1) { + for (size_t size2 = 0; size2 < 5; ++size2) { + absl::InlinedVector, 2> a; + absl::InlinedVector, 2> b; + for (size_t i = 0; i < size1; ++i) { + a.push_back(std::make_unique(i + 10)); + } + for (size_t i = 0; i < size2; ++i) { + b.push_back(std::make_unique(i + 20)); + } + a.swap(b); + ASSERT_THAT(a, SizeIs(size2)); + ASSERT_THAT(b, SizeIs(size1)); + for (size_t i = 0; i < a.size(); ++i) { + ASSERT_THAT(a[i], Pointee(i + 20)); + } + for (size_t i = 0; i < b.size(); ++i) { + ASSERT_THAT(b[i], Pointee(i + 10)); + } + } + } +} + // At the end of this test loop, the elements between [erase_begin, erase_end) // should have reference counts == 0, and all others elements should have // reference counts == 1. @@ -783,7 +812,9 @@ TEST(OverheadTest, Storage) { // The union should be absorbing some of the allocation bookkeeping overhead // in the larger vectors, leaving only the size_ field as overhead. - struct T { void* val; }; + struct T { + void* val; + }; size_t expected_overhead = sizeof(T); EXPECT_EQ((2 * expected_overhead), diff --git a/absl/container/internal/inlined_vector.h b/absl/container/internal/inlined_vector.h index 0eb9c34d..90a74dc7 100644 --- a/absl/container/internal/inlined_vector.h +++ b/absl/container/internal/inlined_vector.h @@ -322,14 +322,13 @@ class Storage { // The policy to be used specifically when swapping inlined elements. using SwapInlinedElementsPolicy = absl::conditional_t< - // Fast path: if the value type can be trivially move constructed/assigned - // and destroyed, and we know the allocator doesn't do anything fancy, - // then it's safe for us to simply swap the bytes in the inline storage. - // It's as if we had move-constructed a temporary vector, move-assigned - // one to the other, then move-assigned the first from the temporary. - absl::conjunction>, - absl::is_trivially_move_assignable>, - absl::is_trivially_destructible>, + // Fast path: if the value type can be trivially relocated, and we + // know the allocator doesn't do anything fancy, then it's safe for us + // to simply swap the bytes in the inline storage. It's as if we had + // relocated the first vector's elements into temporary storage, + // relocated the second's elements into the (now-empty) first's, + // and then relocated from temporary storage into the second. + absl::conjunction>, std::is_same>>>::value, MemcpyPolicy, absl::conditional_t::value, ElementwiseSwapPolicy, @@ -624,8 +623,8 @@ void Storage::InitFrom(const Storage& other) { template template -auto Storage::Initialize(ValueAdapter values, SizeType new_size) - -> void { +auto Storage::Initialize(ValueAdapter values, + SizeType new_size) -> void { // Only callable from constructors! ABSL_HARDENING_ASSERT(!GetIsAllocated()); ABSL_HARDENING_ASSERT(GetSize() == 0); @@ -656,8 +655,8 @@ auto Storage::Initialize(ValueAdapter values, SizeType new_size) template template -auto Storage::Assign(ValueAdapter values, SizeType new_size) - -> void { +auto Storage::Assign(ValueAdapter values, + SizeType new_size) -> void { StorageView storage_view = MakeStorageView(); AllocationTransaction allocation_tx(GetAllocator()); @@ -699,8 +698,8 @@ auto Storage::Assign(ValueAdapter values, SizeType new_size) template template -auto Storage::Resize(ValueAdapter values, SizeType new_size) - -> void { +auto Storage::Resize(ValueAdapter values, + SizeType new_size) -> void { StorageView storage_view = MakeStorageView(); Pointer const base = storage_view.data; const SizeType size = storage_view.size; @@ -885,8 +884,8 @@ auto Storage::EmplaceBackSlow(Args&&... args) -> Reference { } template -auto Storage::Erase(ConstIterator from, ConstIterator to) - -> Iterator { +auto Storage::Erase(ConstIterator from, + ConstIterator to) -> Iterator { StorageView storage_view = MakeStorageView(); auto erase_size = static_cast>(std::distance(from, to)); -- cgit v1.2.3 From 831e57a483cb80100888e9f9722710c0b6afe6d7 Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Wed, 21 Feb 2024 11:06:19 -0800 Subject: Change find_or_prepare_insert to return std::pair to match return type of insert. PiperOrigin-RevId: 609058024 Change-Id: I2f7cc2daf862e7e2d23acd6dd3fe85cb1945d5f0 --- absl/container/internal/raw_hash_map.h | 6 +++--- absl/container/internal/raw_hash_set.h | 37 +++++++++++++++++----------------- 2 files changed, 21 insertions(+), 22 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_map.h b/absl/container/internal/raw_hash_map.h index 97182bc7..64dcd3d4 100644 --- a/absl/container/internal/raw_hash_map.h +++ b/absl/container/internal/raw_hash_map.h @@ -201,8 +201,8 @@ class raw_hash_map : public raw_hash_set { if (res.second) this->emplace_at(res.first, std::forward(k), std::forward(v)); else - Policy::value(&*this->iterator_at(res.first)) = std::forward(v); - return {this->iterator_at(res.first), res.second}; + Policy::value(&*res.first) = std::forward(v); + return res; } template @@ -213,7 +213,7 @@ class raw_hash_map : public raw_hash_set { this->emplace_at(res.first, std::piecewise_construct, std::forward_as_tuple(std::forward(k)), std::forward_as_tuple(std::forward(args)...)); - return {this->iterator_at(res.first), res.second}; + return res; } }; diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 7eacfa8c..e07d5460 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -2329,7 +2329,7 @@ class raw_hash_set { PolicyTraits::element(that_slot))) == h2 && "hash function value changed unexpectedly during the copy"); SetCtrl(common(), offset, h2, sizeof(slot_type)); - emplace_at(offset, PolicyTraits::element(that_slot)); + emplace_at(iterator_at(offset), PolicyTraits::element(that_slot)); common().maybe_increment_generation_on_insert(); }); if (shift != 0) { @@ -2629,11 +2629,11 @@ class raw_hash_set { F&& f) ABSL_ATTRIBUTE_LIFETIME_BOUND { auto res = find_or_prepare_insert(key); if (res.second) { - slot_type* slot = slot_array() + res.first; + slot_type* slot = res.first.slot(); std::forward(f)(constructor(&alloc_ref(), &slot)); assert(!slot); } - return iterator_at(res.first); + return res.first; } // Extension API: support for heterogeneous keys. @@ -2956,7 +2956,7 @@ class raw_hash_set { if (res.second) { s.emplace_at(res.first, std::forward(args)...); } - return {s.iterator_at(res.first), res.second}; + return res; } raw_hash_set& s; }; @@ -2967,11 +2967,11 @@ class raw_hash_set { std::pair operator()(const K& key, Args&&...) && { auto res = s.find_or_prepare_insert(key); if (res.second) { - s.transfer(s.slot_array() + res.first, &slot); + s.transfer(res.first.slot(), &slot); } else if (do_destroy) { s.destroy(&slot); } - return {s.iterator_at(res.first), res.second}; + return res; } raw_hash_set& s; // Constructed slot. Either moved into place or destroyed. @@ -3208,11 +3208,11 @@ class raw_hash_set { } protected: - // Attempts to find `key` in the table; if it isn't found, returns a slot that - // the value can be inserted into, with the control byte already set to - // `key`'s H2. + // Attempts to find `key` in the table; if it isn't found, returns an iterator + // where the value can be inserted into, with the control byte already set to + // `key`'s H2. Returns a bool indicating whether an insertion can take place. template - std::pair find_or_prepare_insert(const K& key) { + std::pair find_or_prepare_insert(const K& key) { prefetch_heap_block(); auto hash = hash_ref()(key); auto seq = probe(common(), hash); @@ -3223,13 +3223,13 @@ class raw_hash_set { if (ABSL_PREDICT_TRUE(PolicyTraits::apply( EqualElement{key, eq_ref()}, PolicyTraits::element(slot_array() + seq.offset(i))))) - return {seq.offset(i), false}; + return {iterator_at(seq.offset(i)), false}; } if (ABSL_PREDICT_TRUE(g.MaskEmpty())) break; seq.next(); assert(seq.index() <= capacity() && "full table!"); } - return {prepare_insert(hash), true}; + return {iterator_at(prepare_insert(hash)), true}; } // Given the hash of a value not currently in the table, finds the next @@ -3272,16 +3272,15 @@ class raw_hash_set { // after an unsuccessful find_or_prepare_insert() and before any other // modifications happen in the raw_hash_set. // - // PRECONDITION: i is an index returned from find_or_prepare_insert(k), where - // k is the key decomposed from `forward(args)...`, and the bool - // returned by find_or_prepare_insert(k) was true. + // PRECONDITION: iter was returned from find_or_prepare_insert(k), where k is + // the key decomposed from `forward(args)...`, and the bool returned by + // find_or_prepare_insert(k) was true. // POSTCONDITION: *m.iterator_at(i) == value_type(forward(args)...). template - void emplace_at(size_t i, Args&&... args) { - construct(slot_array() + i, std::forward(args)...); + void emplace_at(iterator iter, Args&&... args) { + construct(iter.slot(), std::forward(args)...); - assert(PolicyTraits::apply(FindElement{*this}, *iterator_at(i)) == - iterator_at(i) && + assert(PolicyTraits::apply(FindElement{*this}, *iter) == iter && "constructed value does not match the lookup key"); } -- cgit v1.2.3 From d87dc03cee90a0cac2dbf254217b346ca693bb83 Mon Sep 17 00:00:00 2001 From: Vitaly Goldshteyn Date: Thu, 22 Feb 2024 17:06:14 -0800 Subject: Optimize `prepare_insert`, when resize happens. It removes single unnecessary probing before resize that is beneficial for small tables the most. PiperOrigin-RevId: 609547787 Change-Id: If6584919b4c93945ea078b1c1a9f57b355dce924 --- absl/container/internal/raw_hash_set.h | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index e07d5460..7b33de63 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -1623,8 +1623,10 @@ class HashSetResizeHelper { old_capacity_(c.capacity()), had_infoz_(c.has_infoz()) {} - // Optimized for small groups version of `find_first_non_full` applicable - // only right after calling `raw_hash_set::resize`. + // Optimized for small groups version of `find_first_non_full`. + // Beneficial only right after calling `raw_hash_set::resize`. + // It is safe to call in case capacity is big or was not changed, but there + // will be no performance benefit. // It has implicit assumption that `resize` will call // `GrowSizeIntoSingleGroup*` in case `IsGrowingIntoSingleGroupApplicable`. // Falls back to `find_first_non_full` in case of big groups, so it is @@ -3244,21 +3246,21 @@ class raw_hash_set { const size_t cap = capacity(); resize(growth_left() > 0 ? cap : NextCapacity(cap)); } - auto target = find_first_non_full(common(), hash); - if (!rehash_for_bug_detection && - ABSL_PREDICT_FALSE(growth_left() == 0 && - !IsDeleted(control()[target.offset]))) { - size_t old_capacity = capacity(); + FindInfo target; + if (!rehash_for_bug_detection && ABSL_PREDICT_FALSE(growth_left() == 0)) { + const size_t old_capacity = capacity(); rehash_and_grow_if_necessary(); - // NOTE: It is safe to use `FindFirstNonFullAfterResize`. - // `FindFirstNonFullAfterResize` must be called right after resize. + // NOTE: It is safe to use `FindFirstNonFullAfterResize` after + // `rehash_and_grow_if_necessary`, whether capacity changes or not. // `rehash_and_grow_if_necessary` may *not* call `resize` // and perform `drop_deletes_without_resize` instead. But this - // could happen only on big tables. + // could happen only on big tables and will not change capacity. // For big tables `FindFirstNonFullAfterResize` will always - // fallback to normal `find_first_non_full`, so it is safe to use it. + // fallback to normal `find_first_non_full`. target = HashSetResizeHelper::FindFirstNonFullAfterResize( common(), old_capacity, hash); + } else { + target = find_first_non_full(common(), hash); } common().increment_size(); set_growth_left(growth_left() - IsEmpty(control()[target.offset])); -- cgit v1.2.3 From eef325b1d102aa4cbf1dbd865493ea0757222f3f Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Fri, 23 Feb 2024 11:33:08 -0800 Subject: Add braces for conditional statements in raw_hash_map functions. The current version violates the Google C++ style guide - see https://google.github.io/styleguide/cppguide.html#Formatting_Looping_Branching. This code does not fit into the historical exception that "the curly braces for the controlled statement or the line breaks inside the curly braces may be omitted if as a result the entire statement appears on either a single line (in which case there is a space between the closing parenthesis and the controlled statement) or on two lines (in which case there is a line break after the closing parenthesis and there are no braces)." PiperOrigin-RevId: 609789188 Change-Id: Id7ae9596e454dac5581d19939564c07670077f92 --- absl/container/internal/raw_hash_map.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_map.h b/absl/container/internal/raw_hash_map.h index 64dcd3d4..464bf23b 100644 --- a/absl/container/internal/raw_hash_map.h +++ b/absl/container/internal/raw_hash_map.h @@ -198,10 +198,11 @@ class raw_hash_map : public raw_hash_set { std::pair insert_or_assign_impl(K&& k, V&& v) ABSL_ATTRIBUTE_LIFETIME_BOUND { auto res = this->find_or_prepare_insert(k); - if (res.second) + if (res.second) { this->emplace_at(res.first, std::forward(k), std::forward(v)); - else + } else { Policy::value(&*res.first) = std::forward(v); + } return res; } @@ -209,10 +210,11 @@ class raw_hash_map : public raw_hash_set { std::pair try_emplace_impl(K&& k, Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND { auto res = this->find_or_prepare_insert(k); - if (res.second) + if (res.second) { this->emplace_at(res.first, std::piecewise_construct, std::forward_as_tuple(std::forward(k)), std::forward_as_tuple(std::forward(args)...)); + } return res; } }; -- cgit v1.2.3 From b7372748dcdc35f40c12742cb57947a409e7f0b7 Mon Sep 17 00:00:00 2001 From: Paul Rigge Date: Wed, 28 Feb 2024 11:38:24 -0800 Subject: Rework casting in raw_hash_set's `IsFull()`. Instead of casting an int to the enum type where the int does not have an associated enum value, cast the enum to its underlying type. This should be no functional change but make some linters happier. PiperOrigin-RevId: 611172311 Change-Id: I9ae10f8fa2029014236f60a90ee2ab2273c66fa5 --- absl/container/internal/raw_hash_set.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 7b33de63..07ff79af 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -571,7 +571,9 @@ inline h2_t H2(size_t hash) { return hash & 0x7F; } // Helpers for checking the state of a control byte. inline bool IsEmpty(ctrl_t c) { return c == ctrl_t::kEmpty; } -inline bool IsFull(ctrl_t c) { return c >= static_cast(0); } +inline bool IsFull(ctrl_t c) { + return static_cast>(c) >= 0; +} inline bool IsDeleted(ctrl_t c) { return c == ctrl_t::kDeleted; } inline bool IsEmptyOrDeleted(ctrl_t c) { return c < ctrl_t::kSentinel; } -- cgit v1.2.3 From 7a4344511816e82234700795e7f2aaa80e85a119 Mon Sep 17 00:00:00 2001 From: Vitaly Goldshteyn Date: Sun, 3 Mar 2024 09:17:56 -0800 Subject: Create `BM_GroupPortable_Match`. PiperOrigin-RevId: 612201313 Change-Id: Ia9e7f146f5e1ecaffcb15de694049b716db38d02 --- absl/container/internal/raw_hash_set_benchmark.cc | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set_benchmark.cc b/absl/container/internal/raw_hash_set_benchmark.cc index 8cd43fb5..0fa9c712 100644 --- a/absl/container/internal/raw_hash_set_benchmark.cc +++ b/absl/container/internal/raw_hash_set_benchmark.cc @@ -457,6 +457,19 @@ void BM_Group_Match(benchmark::State& state) { } BENCHMARK(BM_Group_Match); +void BM_GroupPortable_Match(benchmark::State& state) { + std::array group; + Iota(group.begin(), group.end(), -4); + GroupPortableImpl g{group.data()}; + h2_t h = 1; + for (auto _ : state) { + ::benchmark::DoNotOptimize(h); + ::benchmark::DoNotOptimize(g); + ::benchmark::DoNotOptimize(g.Match(h)); + } +} +BENCHMARK(BM_GroupPortable_Match); + void BM_Group_MaskEmpty(benchmark::State& state) { std::array group; Iota(group.begin(), group.end(), -4); -- cgit v1.2.3 From 7bd9ff910d489658da58251de1317eb3f790a2c6 Mon Sep 17 00:00:00 2001 From: Arthur O'Dwyer Date: Sun, 3 Mar 2024 18:47:41 -0800 Subject: PR #1632: inlined_vector: Use trivial relocation for `erase` Imported from GitHub PR https://github.com/abseil/abseil-cpp/pull/1632 Prior art for the `vector::erase` optimization: https://github.com/AmadeusITGroup/amc/blob/efcb7be/include/amc/vectorcommon.hpp#L176-L180 https://github.com/bloomberg/bde/blob/e15f05be6/groups/bsl/bslalg/bslalg_arrayprimitives.h#L3787-L3799 https://github.com/facebook/folly/blob/d24bf04/folly/FBVector.h#L1254-L1262 https://github.com/qt/qtbase/blob/fbfee2d/src/corelib/tools/qarraydataops.h#L856-L861 Merge 6ce011079ccf945ae95434ce45ea6c5e3a088af8 into 55d28d4b3b82f9a47b3fa9b811b675a032820621 Merging this change closes #1632 COPYBARA_INTEGRATE_REVIEW=https://github.com/abseil/abseil-cpp/pull/1632 from Quuxplusone:trivial-erase 6ce011079ccf945ae95434ce45ea6c5e3a088af8 PiperOrigin-RevId: 612278964 Change-Id: I327ace8a38292b4610c6be031cc334e77c76fd35 --- absl/container/inlined_vector.h | 13 ++++++++ absl/container/inlined_vector_test.cc | 51 ++++++++++++++++++++++++++++++++ absl/container/internal/inlined_vector.h | 30 ++++++++++++++----- 3 files changed, 86 insertions(+), 8 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/inlined_vector.h b/absl/container/inlined_vector.h index 04e2c385..974b6521 100644 --- a/absl/container/inlined_vector.h +++ b/absl/container/inlined_vector.h @@ -775,7 +775,20 @@ class InlinedVector { ABSL_HARDENING_ASSERT(pos >= begin()); ABSL_HARDENING_ASSERT(pos < end()); + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102329#c2 + // It appears that GCC thinks that since `pos` is a const pointer and may + // point to uninitialized memory at this point, a warning should be + // issued. But `pos` is actually only used to compute an array index to + // write to. +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#pragma GCC diagnostic ignored "-Wuninitialized" +#endif return storage_.Erase(pos, pos + 1); +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic pop +#endif } // Overload of `InlinedVector::erase(...)` that erases every element in the diff --git a/absl/container/inlined_vector_test.cc b/absl/container/inlined_vector_test.cc index 5ecf88a9..6954262e 100644 --- a/absl/container/inlined_vector_test.cc +++ b/absl/container/inlined_vector_test.cc @@ -333,6 +333,57 @@ TEST(UniquePtr, Swap) { } } +// Erasing from a container of unique pointers should work fine, with no +// leaks, despite the fact that unique pointers are trivially relocatable but +// not trivially destructible. +// TODO(absl-team): Using unique_ptr here is technically correct, but +// a trivially relocatable struct would be less semantically confusing. +TEST(UniquePtr, EraseSingle) { + for (size_t size = 4; size < 16; ++size) { + absl::InlinedVector, 8> a; + for (size_t i = 0; i < size; ++i) { + a.push_back(std::make_unique(i)); + } + a.erase(a.begin()); + ASSERT_THAT(a, SizeIs(size - 1)); + for (size_t i = 0; i < size - 1; ++i) { + ASSERT_THAT(a[i], Pointee(i + 1)); + } + a.erase(a.begin() + 2); + ASSERT_THAT(a, SizeIs(size - 2)); + ASSERT_THAT(a[0], Pointee(1)); + ASSERT_THAT(a[1], Pointee(2)); + for (size_t i = 2; i < size - 2; ++i) { + ASSERT_THAT(a[i], Pointee(i + 2)); + } + } +} + +// Erasing from a container of unique pointers should work fine, with no +// leaks, despite the fact that unique pointers are trivially relocatable but +// not trivially destructible. +// TODO(absl-team): Using unique_ptr here is technically correct, but +// a trivially relocatable struct would be less semantically confusing. +TEST(UniquePtr, EraseMulti) { + for (size_t size = 5; size < 16; ++size) { + absl::InlinedVector, 8> a; + for (size_t i = 0; i < size; ++i) { + a.push_back(std::make_unique(i)); + } + a.erase(a.begin(), a.begin() + 2); + ASSERT_THAT(a, SizeIs(size - 2)); + for (size_t i = 0; i < size - 2; ++i) { + ASSERT_THAT(a[i], Pointee(i + 2)); + } + a.erase(a.begin() + 1, a.begin() + 3); + ASSERT_THAT(a, SizeIs(size - 4)); + ASSERT_THAT(a[0], Pointee(2)); + for (size_t i = 1; i < size - 4; ++i) { + ASSERT_THAT(a[i], Pointee(i + 4)); + } + } +} + // At the end of this test loop, the elements between [erase_begin, erase_end) // should have reference counts == 0, and all others elements should have // reference counts == 1. diff --git a/absl/container/internal/inlined_vector.h b/absl/container/internal/inlined_vector.h index 90a74dc7..a1575328 100644 --- a/absl/container/internal/inlined_vector.h +++ b/absl/container/internal/inlined_vector.h @@ -893,16 +893,30 @@ auto Storage::Erase(ConstIterator from, std::distance(ConstIterator(storage_view.data), from)); SizeType erase_end_index = erase_index + erase_size; - IteratorValueAdapter> move_values( - MoveIterator(storage_view.data + erase_end_index)); - - AssignElements(storage_view.data + erase_index, move_values, - storage_view.size - erase_end_index); + // Fast path: if the value type is trivially relocatable and we know + // the allocator doesn't do anything fancy, then we know it is legal for us to + // simply destroy the elements in the "erasure window" (which cannot throw) + // and then memcpy downward to close the window. + if (absl::is_trivially_relocatable>::value && + std::is_nothrow_destructible>::value && + std::is_same>>::value) { + DestroyAdapter::DestroyElements( + GetAllocator(), storage_view.data + erase_index, erase_size); + std::memmove( + reinterpret_cast(storage_view.data + erase_index), + reinterpret_cast(storage_view.data + erase_end_index), + (storage_view.size - erase_end_index) * sizeof(ValueType)); + } else { + IteratorValueAdapter> move_values( + MoveIterator(storage_view.data + erase_end_index)); - DestroyAdapter::DestroyElements( - GetAllocator(), storage_view.data + (storage_view.size - erase_size), - erase_size); + AssignElements(storage_view.data + erase_index, move_values, + storage_view.size - erase_end_index); + DestroyAdapter::DestroyElements( + GetAllocator(), storage_view.data + (storage_view.size - erase_size), + erase_size); + } SubtractSize(erase_size); return Iterator(storage_view.data + erase_index); } -- cgit v1.2.3 From 8dc90ff07402cd027daec520bb77f46e51855889 Mon Sep 17 00:00:00 2001 From: Vitaly Goldshteyn Date: Mon, 4 Mar 2024 13:12:48 -0800 Subject: Extract `InsertPosition` function to be able to reuse it. PiperOrigin-RevId: 612560213 Change-Id: Id75dfd1222a0bed8ec72ce21e4a97b1d09fc9eaa --- absl/container/internal/raw_hash_set.cc | 5 +++-- absl/container/internal/raw_hash_set.h | 35 ++++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 13 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.cc b/absl/container/internal/raw_hash_set.cc index 02301e19..efb3d6f3 100644 --- a/absl/container/internal/raw_hash_set.cc +++ b/absl/container/internal/raw_hash_set.cc @@ -104,10 +104,11 @@ bool CommonFieldsGenerationInfoEnabled::should_rehash_for_bug_detection_on_move( return ShouldRehashForBugDetection(ctrl, capacity); } -bool ShouldInsertBackwards(size_t hash, const ctrl_t* ctrl) { +bool ShouldInsertBackwardsForDebug(size_t capacity, size_t hash, + const ctrl_t* ctrl) { // To avoid problems with weak hashes and single bit tests, we use % 13. // TODO(kfm,sbenza): revisit after we do unconditional mixing - return (H1(hash, ctrl) ^ RandomSeed()) % 13 > 6; + return !is_small(capacity) && (H1(hash, ctrl) ^ RandomSeed()) % 13 > 6; } void ConvertDeletedToEmptyAndFullToDeleted(ctrl_t* ctrl, size_t capacity) { diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 07ff79af..514f21ae 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -546,7 +546,27 @@ inline bool IsEmptyGeneration(const GenerationType* generation) { // Mixes a randomly generated per-process seed with `hash` and `ctrl` to // randomize insertion order within groups. -bool ShouldInsertBackwards(size_t hash, const ctrl_t* ctrl); +bool ShouldInsertBackwardsForDebug(size_t capacity, size_t hash, + const ctrl_t* ctrl); + +// Returns insert position for the given mask. +// We want to add entropy even when ASLR is not enabled. +// In debug build we will randomly insert in either the front or back of +// the group. +// TODO(kfm,sbenza): revisit after we do unconditional mixing +template +ABSL_ATTRIBUTE_ALWAYS_INLINE inline auto GetInsertionOffset( + Mask mask, ABSL_ATTRIBUTE_UNUSED size_t capacity, + ABSL_ATTRIBUTE_UNUSED size_t hash, + ABSL_ATTRIBUTE_UNUSED const ctrl_t* ctrl) { +#if defined(NDEBUG) + return mask.LowestBitSet(); +#else + return ShouldInsertBackwardsForDebug(capacity, hash, ctrl) + ? mask.HighestBitSet() + : mask.LowestBitSet(); +#endif +} // Returns a per-table, hash salt, which changes on resize. This gets mixed into // H1 to randomize iteration order per-table. @@ -1495,16 +1515,9 @@ inline FindInfo find_first_non_full(const CommonFields& common, size_t hash) { GroupFullEmptyOrDeleted g{ctrl + seq.offset()}; auto mask = g.MaskEmptyOrDeleted(); if (mask) { -#if !defined(NDEBUG) - // We want to add entropy even when ASLR is not enabled. - // In debug build we will randomly insert in either the front or back of - // the group. - // TODO(kfm,sbenza): revisit after we do unconditional mixing - if (!is_small(common.capacity()) && ShouldInsertBackwards(hash, ctrl)) { - return {seq.offset(mask.HighestBitSet()), seq.index()}; - } -#endif - return {seq.offset(mask.LowestBitSet()), seq.index()}; + return { + seq.offset(GetInsertionOffset(mask, common.capacity(), hash, ctrl)), + seq.index()}; } seq.next(); assert(seq.index() <= common.capacity() && "full table!"); -- cgit v1.2.3 From 1449c9a106b090f61441ba245c781d7d2f89000c Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Wed, 6 Mar 2024 10:00:52 -0800 Subject: Implement small object optimization in swisstable - disabled for now. Details: - We use the space for control/slots pointers as the inline buffer. - We use a max inline capacity of 1 to make the implementation much simpler and to avoid having to randomize the iteration order for inline tables. - For iteration of inline tables, we introduce the kSooControl buffer which just has 1 full control byte followed by 1 sentinel control byte so that incrementing yields an end() iterator. We don't access kSooControl during lookups - only iteration. PiperOrigin-RevId: 613253492 Change-Id: Id98ff11842f8bef27ac7ed88138dc03b46ce4fa6 --- absl/container/flat_hash_set_test.cc | 91 +-- absl/container/internal/hash_policy_traits.h | 15 + absl/container/internal/raw_hash_set.cc | 79 ++- absl/container/internal/raw_hash_set.h | 796 +++++++++++++++++++++------ absl/container/internal/raw_hash_set_test.cc | 450 ++++++++++----- absl/container/sample_element_size_test.cc | 31 +- 6 files changed, 1091 insertions(+), 371 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/flat_hash_set_test.cc b/absl/container/flat_hash_set_test.cc index 9ce9267e..b425bc50 100644 --- a/absl/container/flat_hash_set_test.cc +++ b/absl/container/flat_hash_set_test.cc @@ -181,15 +181,13 @@ TEST(FlatHashSet, EraseIf) { } } -class PoisonInline { +class PoisonSoo { int64_t data_; public: - explicit PoisonInline(int64_t d) : data_(d) { - SanitizerPoisonObject(&data_); - } - PoisonInline(const PoisonInline& that) : PoisonInline(*that) {} - ~PoisonInline() { SanitizerUnpoisonObject(&data_); } + explicit PoisonSoo(int64_t d) : data_(d) { SanitizerPoisonObject(&data_); } + PoisonSoo(const PoisonSoo& that) : PoisonSoo(*that) {} + ~PoisonSoo() { SanitizerUnpoisonObject(&data_); } int64_t operator*() const { SanitizerUnpoisonObject(&data_); @@ -198,45 +196,56 @@ class PoisonInline { return ret; } template - friend H AbslHashValue(H h, const PoisonInline& pi) { + friend H AbslHashValue(H h, const PoisonSoo& pi) { return H::combine(std::move(h), *pi); } - bool operator==(const PoisonInline& rhs) const { return **this == *rhs; } + bool operator==(const PoisonSoo& rhs) const { return **this == *rhs; } }; -// Tests that we don't touch the poison_ member of PoisonInline. -TEST(FlatHashSet, PoisonInline) { - PoisonInline a(0), b(1); - { // basic usage - flat_hash_set set; - set.insert(a); - EXPECT_THAT(set, UnorderedElementsAre(a)); - set.insert(b); - EXPECT_THAT(set, UnorderedElementsAre(a, b)); - set.erase(a); - EXPECT_THAT(set, UnorderedElementsAre(b)); - set.rehash(0); // shrink to inline - EXPECT_THAT(set, UnorderedElementsAre(b)); - } - { // test move constructor from inline to inline - flat_hash_set set; - set.insert(a); - flat_hash_set set2(std::move(set)); - EXPECT_THAT(set2, UnorderedElementsAre(a)); - } - { // test move assignment from inline to inline - flat_hash_set set, set2; - set.insert(a); - set2 = std::move(set); - EXPECT_THAT(set2, UnorderedElementsAre(a)); - } - { // test alloc move constructor from inline to inline - flat_hash_set set; - set.insert(a); - flat_hash_set set2(std::move(set), - std::allocator()); - EXPECT_THAT(set2, UnorderedElementsAre(a)); - } +TEST(FlatHashSet, PoisonSooBasic) { + PoisonSoo a(0), b(1); + flat_hash_set set; + set.insert(a); + EXPECT_THAT(set, UnorderedElementsAre(a)); + set.insert(b); + EXPECT_THAT(set, UnorderedElementsAre(a, b)); + set.erase(a); + EXPECT_THAT(set, UnorderedElementsAre(b)); + set.rehash(0); // Shrink to SOO. + EXPECT_THAT(set, UnorderedElementsAre(b)); +} + +TEST(FlatHashSet, PoisonSooMoveConstructSooToSoo) { + PoisonSoo a(0); + flat_hash_set set; + set.insert(a); + flat_hash_set set2(std::move(set)); + EXPECT_THAT(set2, UnorderedElementsAre(a)); +} + +TEST(FlatHashSet, PoisonSooAllocMoveConstructSooToSoo) { + PoisonSoo a(0); + flat_hash_set set; + set.insert(a); + flat_hash_set set2(std::move(set), std::allocator()); + EXPECT_THAT(set2, UnorderedElementsAre(a)); +} + +TEST(FlatHashSet, PoisonSooMoveAssignFullSooToEmptySoo) { + PoisonSoo a(0); + flat_hash_set set, set2; + set.insert(a); + set2 = std::move(set); + EXPECT_THAT(set2, UnorderedElementsAre(a)); +} + +TEST(FlatHashSet, PoisonSooMoveAssignFullSooToFullSoo) { + PoisonSoo a(0), b(1); + flat_hash_set set, set2; + set.insert(a); + set2.insert(b); + set2 = std::move(set); + EXPECT_THAT(set2, UnorderedElementsAre(a)); } TEST(FlatHashSet, FlatHashSetPolicyDestroyReturnsTrue) { diff --git a/absl/container/internal/hash_policy_traits.h b/absl/container/internal/hash_policy_traits.h index 86ffd1be..ec08794a 100644 --- a/absl/container/internal/hash_policy_traits.h +++ b/absl/container/internal/hash_policy_traits.h @@ -168,6 +168,9 @@ struct hash_policy_traits : common_policy_traits { #endif } + // Whether small object optimization is enabled. False by default. + static constexpr bool soo_enabled() { return soo_enabled_impl(Rank1{}); } + private: template struct HashElement { @@ -183,6 +186,18 @@ struct hash_policy_traits : common_policy_traits { return Policy::apply(HashElement{*static_cast(hash_fn)}, Policy::element(static_cast(slot))); } + + // Use go/ranked-overloads for dispatching. Rank1 is preferred. + struct Rank0 {}; + struct Rank1 : Rank0 {}; + + // Use auto -> decltype as an enabler. + template + static constexpr auto soo_enabled_impl(Rank1) -> decltype(P::soo_enabled()) { + return P::soo_enabled(); + } + + static constexpr bool soo_enabled_impl(Rank0) { return false; } }; } // namespace container_internal diff --git a/absl/container/internal/raw_hash_set.cc b/absl/container/internal/raw_hash_set.cc index efb3d6f3..f0f840d1 100644 --- a/absl/container/internal/raw_hash_set.cc +++ b/absl/container/internal/raw_hash_set.cc @@ -30,12 +30,14 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace container_internal { +// Represents a control byte corresponding to a full slot with arbitrary hash. +constexpr ctrl_t ZeroCtrlT() { return static_cast(0); } + // We have space for `growth_left` before a single block of control bytes. A // single block of empty control bytes for tables without any slots allocated. // This enables removing a branch in the hot path of find(). In order to ensure // that the control bytes are aligned to 16, we have 16 bytes before the control // bytes even though growth_left only needs 8. -constexpr ctrl_t ZeroCtrlT() { return static_cast(0); } alignas(16) ABSL_CONST_INIT ABSL_DLL const ctrl_t kEmptyGroup[32] = { ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), @@ -46,6 +48,18 @@ alignas(16) ABSL_CONST_INIT ABSL_DLL const ctrl_t kEmptyGroup[32] = { ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty}; +// We need one full byte followed by a sentinel byte for iterator::operator++ to +// work. We have a full group after kSentinel to be safe (in case operator++ is +// changed to read a full group). +ABSL_CONST_INIT ABSL_DLL const ctrl_t kSooControl[17] = { + ZeroCtrlT(), ctrl_t::kSentinel, ZeroCtrlT(), ctrl_t::kEmpty, + ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, + ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, + ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, + ctrl_t::kEmpty}; +static_assert(NumControlBytes(SooCapacity()) <= 17, + "kSooControl capacity too small"); + #ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL constexpr size_t Group::kWidth; #endif @@ -111,6 +125,20 @@ bool ShouldInsertBackwardsForDebug(size_t capacity, size_t hash, return !is_small(capacity) && (H1(hash, ctrl) ^ RandomSeed()) % 13 > 6; } +size_t PrepareInsertAfterSoo(size_t hash, size_t slot_size, + CommonFields& common) { + assert(common.capacity() == NextCapacity(SooCapacity())); + // After resize from capacity 1 to 3, we always have exactly the slot with + // index 1 occupied, so we need to insert either at index 0 or index 2. + assert(HashSetResizeHelper::SooSlotIndex() == 1); + PrepareInsertCommon(common); + const size_t offset = H1(hash, common.control()) & 2; + common.set_growth_left(common.growth_left() - 1); + SetCtrlInSingleGroupTable(common, offset, H2(hash), slot_size); + common.infoz().RecordInsert(hash, /*distance_from_desired=*/0); + return offset; +} + void ConvertDeletedToEmptyAndFullToDeleted(ctrl_t* ctrl, size_t capacity) { assert(ctrl[capacity] == ctrl_t::kSentinel); assert(IsValidCapacity(capacity)); @@ -254,9 +282,10 @@ void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size) { } void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy, - bool reuse) { + bool reuse, bool soo_enabled) { c.set_size(0); if (reuse) { + assert(!soo_enabled || c.capacity() > SooCapacity()); ResetCtrl(c, policy.slot_size); ResetGrowthLeft(c); c.infoz().RecordStorageChanged(0, c.capacity()); @@ -264,12 +293,9 @@ void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy, // We need to record infoz before calling dealloc, which will unregister // infoz. c.infoz().RecordClearedReservation(); - c.infoz().RecordStorageChanged(0, 0); + c.infoz().RecordStorageChanged(0, soo_enabled ? SooCapacity() : 0); (*policy.dealloc)(c, policy); - c.set_control(EmptyGroup()); - c.set_generation_ptr(EmptyGeneration()); - c.set_slots(nullptr); - c.set_capacity(0); + c = soo_enabled ? CommonFields{soo_tag_t{}} : CommonFields{}; } } @@ -286,7 +312,7 @@ void HashSetResizeHelper::GrowIntoSingleGroupShuffleControlBytes( // Copy second half of bytes to the beginning. // We potentially copy more bytes in order to have compile time known size. - // Mirrored bytes from the old_ctrl_ will also be copied. + // Mirrored bytes from the old_ctrl() will also be copied. // In case of old_capacity_ == 3, we will copy 1st element twice. // Examples: // old_ctrl = 0S0EEEEEEE... @@ -297,7 +323,7 @@ void HashSetResizeHelper::GrowIntoSingleGroupShuffleControlBytes( // // old_ctrl = 0123456S0123456EE... // new_ctrl = 456S0123?????????... - std::memcpy(new_ctrl, old_ctrl_ + half_old_capacity + 1, kHalfWidth); + std::memcpy(new_ctrl, old_ctrl() + half_old_capacity + 1, kHalfWidth); // Clean up copied kSentinel from old_ctrl. new_ctrl[half_old_capacity] = ctrl_t::kEmpty; @@ -348,34 +374,55 @@ void HashSetResizeHelper::GrowIntoSingleGroupShuffleControlBytes( new_ctrl[new_capacity] = ctrl_t::kSentinel; } +void HashSetResizeHelper::InitControlBytesAfterSoo(ctrl_t* new_ctrl, ctrl_t h2, + size_t new_capacity) { + assert(is_single_group(new_capacity)); + std::memset(new_ctrl, static_cast(ctrl_t::kEmpty), + NumControlBytes(new_capacity)); + assert(HashSetResizeHelper::SooSlotIndex() == 1); + // This allows us to avoid branching on had_soo_slot_. + assert(had_soo_slot_ || h2 == ctrl_t::kEmpty); + new_ctrl[1] = new_ctrl[new_capacity + 2] = h2; + new_ctrl[new_capacity] = ctrl_t::kSentinel; +} + void HashSetResizeHelper::GrowIntoSingleGroupShuffleTransferableSlots( - void* old_slots, void* new_slots, size_t slot_size) const { + void* new_slots, size_t slot_size) const { assert(old_capacity_ > 0); const size_t half_old_capacity = old_capacity_ / 2; - SanitizerUnpoisonMemoryRegion(old_slots, slot_size * old_capacity_); + SanitizerUnpoisonMemoryRegion(old_slots(), slot_size * old_capacity_); std::memcpy(new_slots, - SlotAddress(old_slots, half_old_capacity + 1, slot_size), + SlotAddress(old_slots(), half_old_capacity + 1, slot_size), slot_size * half_old_capacity); std::memcpy(SlotAddress(new_slots, half_old_capacity + 1, slot_size), - old_slots, slot_size * (half_old_capacity + 1)); + old_slots(), slot_size * (half_old_capacity + 1)); } void HashSetResizeHelper::GrowSizeIntoSingleGroupTransferable( - CommonFields& c, void* old_slots, size_t slot_size) { + CommonFields& c, size_t slot_size) { assert(old_capacity_ < Group::kWidth / 2); assert(is_single_group(c.capacity())); assert(IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity())); GrowIntoSingleGroupShuffleControlBytes(c.control(), c.capacity()); - GrowIntoSingleGroupShuffleTransferableSlots(old_slots, c.slot_array(), - slot_size); + GrowIntoSingleGroupShuffleTransferableSlots(c.slot_array(), slot_size); // We poison since GrowIntoSingleGroupShuffleTransferableSlots // may leave empty slots unpoisoned. PoisonSingleGroupEmptySlots(c, slot_size); } +void HashSetResizeHelper::TransferSlotAfterSoo(CommonFields& c, + size_t slot_size) { + assert(was_soo_); + assert(had_soo_slot_); + assert(is_single_group(c.capacity())); + std::memcpy(SlotAddress(c.slot_array(), SooSlotIndex(), slot_size), + old_soo_data(), slot_size); + PoisonSingleGroupEmptySlots(c, slot_size); +} + } // namespace container_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 514f21ae..0d5a5344 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -100,6 +100,13 @@ // Storing control bytes in a separate array also has beneficial cache effects, // since more logical slots will fit into a cache line. // +// # Small Object Optimization (SOO) +// +// When the size/alignment of the value_type and the capacity of the table are +// small, we enable small object optimization and store the values inline in +// the raw_hash_set object. This optimization allows us to avoid +// allocation/deallocation as well as cache/dTLB misses. +// // # Hashing // // We compute two separate hashes, `H1` and `H2`, from the hash of an object. @@ -531,10 +538,24 @@ ABSL_DLL extern const ctrl_t kEmptyGroup[32]; // Returns a pointer to a control byte group that can be used by empty tables. inline ctrl_t* EmptyGroup() { // Const must be cast away here; no uses of this function will actually write - // to it, because it is only used for empty tables. + // to it because it is only used for empty tables. return const_cast(kEmptyGroup + 16); } +// For use in SOO iterators. +// TODO(b/289225379): we could potentially get rid of this by adding an is_soo +// bit in iterators. This would add branches but reduce cache misses. +ABSL_DLL extern const ctrl_t kSooControl[17]; + +// Returns a pointer to a full byte followed by a sentinel byte. +inline ctrl_t* SooControl() { + // Const must be cast away here; no uses of this function will actually write + // to it because it is only used for SOO iterators. + return const_cast(kSooControl); +} +// Whether ctrl is from the SooControl array. +inline bool IsSooControl(const ctrl_t* ctrl) { return ctrl == SooControl(); } + // Returns a pointer to a generation to use for an empty hashtable. GenerationType* EmptyGeneration(); @@ -591,9 +612,7 @@ inline h2_t H2(size_t hash) { return hash & 0x7F; } // Helpers for checking the state of a control byte. inline bool IsEmpty(ctrl_t c) { return c == ctrl_t::kEmpty; } -inline bool IsFull(ctrl_t c) { - return static_cast>(c) >= 0; -} +inline bool IsFull(ctrl_t c) { return c >= static_cast(0); } inline bool IsDeleted(ctrl_t c) { return c == ctrl_t::kDeleted; } inline bool IsEmptyOrDeleted(ctrl_t c) { return c < ctrl_t::kSentinel; } @@ -1047,7 +1066,7 @@ inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; } constexpr size_t NumClonedBytes() { return Group::kWidth - 1; } // Returns the number of control bytes including cloned. -inline size_t NumControlBytes(size_t capacity) { +constexpr size_t NumControlBytes(size_t capacity) { return capacity + 1 + NumClonedBytes(); } @@ -1098,12 +1117,64 @@ class RawHashSetLayout { size_t slot_offset_; }; +// We only allow a maximum of 1 SOO element, which makes the implementation +// much simpler. Complications with multiple SOO elements include: +// - Satisfying the guarantee that erasing one element doesn't invalidate +// iterators to other elements means we would probably need actual SOO +// control bytes. +// - In order to prevent user code from depending on iteration order for small +// tables, we would need to randomize the iteration order somehow. +constexpr size_t SooCapacity() { return 1; } +// Sentinel type to indicate SOO CommonFields construction. +struct soo_tag_t {}; +// Sentinel type to indicate SOO CommonFields construction with full size. +struct full_soo_tag_t {}; + +// This allows us to work around an uninitialized memory warning when +// constructing begin() iterators in empty hashtables. +union MaybeInitializedPtr { + void* p; +}; + +struct HeapPtrs { + HeapPtrs() = default; + explicit HeapPtrs(ctrl_t* c) : control(c) {} + + // The control bytes (and, also, a pointer near to the base of the backing + // array). + // + // This contains `capacity + 1 + NumClonedBytes()` entries, even + // when the table is empty (hence EmptyGroup). + // + // Note that growth_left is stored immediately before this pointer. + // May be uninitialized for SOO tables. + ctrl_t* control; + + // The beginning of the slots, located at `SlotOffset()` bytes after + // `control`. May be uninitialized for empty tables. + // Note: we can't use `slots` because Qt defines "slots" as a macro. + MaybeInitializedPtr slot_array; +}; + +// Manages the backing array pointers or the SOO slot. When raw_hash_set::is_soo +// is true, the SOO slot is stored in `soo_data`. Otherwise, we use `heap`. +union HeapOrSoo { + HeapOrSoo() = default; + explicit HeapOrSoo(ctrl_t* c) : heap(c) {} + + HeapPtrs heap; + unsigned char soo_data[sizeof(HeapPtrs)]; +}; + // CommonFields hold the fields in raw_hash_set that do not depend // on template parameters. This allows us to conveniently pass all // of this state to helper functions as a single argument. class CommonFields : public CommonFieldsGenerationInfo { public: - CommonFields() = default; + CommonFields() : capacity_(0), size_(0), heap_or_soo_(EmptyGroup()) {} + explicit CommonFields(soo_tag_t) : capacity_(SooCapacity()), size_(0) {} + explicit CommonFields(full_soo_tag_t) + : capacity_(SooCapacity()), size_(size_t{1} << HasInfozShift()) {} // Not copyable CommonFields(const CommonFields&) = delete; @@ -1113,8 +1184,20 @@ class CommonFields : public CommonFieldsGenerationInfo { CommonFields(CommonFields&& that) = default; CommonFields& operator=(CommonFields&&) = default; - ctrl_t* control() const { return control_; } - void set_control(ctrl_t* c) { control_ = c; } + template + static CommonFields CreateDefault() { + return kSooEnabled ? CommonFields{soo_tag_t{}} : CommonFields{}; + } + + // The inline data for SOO is written on top of control_/slots_. + const void* soo_data() const { return heap_or_soo_.soo_data; } + void* soo_data() { return heap_or_soo_.soo_data; } + + HeapOrSoo heap_or_soo() const { return heap_or_soo_; } + const HeapOrSoo& heap_or_soo_ref() const { return heap_or_soo_; } + + ctrl_t* control() const { return heap_or_soo_.heap.control; } + void set_control(ctrl_t* c) { heap_or_soo_.heap.control = c; } void* backing_array_start() const { // growth_left (and maybe infoz) is stored before control bytes. assert(reinterpret_cast(control()) % alignof(size_t) == 0); @@ -1122,14 +1205,33 @@ class CommonFields : public CommonFieldsGenerationInfo { } // Note: we can't use slots() because Qt defines "slots" as a macro. - void* slot_array() const { return slots_; } - void set_slots(void* s) { slots_ = s; } + void* slot_array() const { return heap_or_soo_.heap.slot_array.p; } + MaybeInitializedPtr slots_union() const { + // Suppress erroneous uninitialized memory errors on GCC. +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + return heap_or_soo_.heap.slot_array; +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + } + void set_slots(void* s) { heap_or_soo_.heap.slot_array.p = s; } // The number of filled slots. size_t size() const { return size_ >> HasInfozShift(); } void set_size(size_t s) { size_ = (s << HasInfozShift()) | (size_ & HasInfozMask()); } + void set_empty_soo() { + AssertInSooMode(); + size_ = 0; + } + void set_full_soo() { + AssertInSooMode(); + size_ = size_t{1} << HasInfozShift(); + } void increment_size() { assert(size() < capacity()); size_ += size_t{1} << HasInfozShift(); @@ -1148,6 +1250,8 @@ class CommonFields : public CommonFieldsGenerationInfo { // The number of slots we can still fill without needing to rehash. // This is stored in the heap allocation before the control bytes. + // TODO(b/289225379): experiment with moving growth_left back inline to + // increase room for SOO. size_t growth_left() const { const size_t* gl_ptr = reinterpret_cast(control()) - 1; assert(reinterpret_cast(gl_ptr) % alignof(size_t) == 0); @@ -1184,10 +1288,6 @@ class CommonFields : public CommonFieldsGenerationInfo { return CommonFieldsGenerationInfo:: should_rehash_for_bug_detection_on_move(control(), capacity()); } - void maybe_increment_generation_on_move() { - if (capacity() == 0) return; - increment_generation(); - } void reset_reserved_growth(size_t reservation) { CommonFieldsGenerationInfo::reset_reserved_growth(reservation, size()); } @@ -1198,6 +1298,14 @@ class CommonFields : public CommonFieldsGenerationInfo { .alloc_size(slot_size); } + // Move fields other than heap_or_soo_. + void move_non_heap_or_soo_fields(CommonFields& that) { + static_cast(*this) = + std::move(static_cast(that)); + capacity_ = that.capacity_; + size_ = that.size_; + } + // Returns the number of control bytes set to kDeleted. For testing only. size_t TombstonesCount() const { return static_cast( @@ -1211,21 +1319,12 @@ class CommonFields : public CommonFieldsGenerationInfo { return (size_t{1} << HasInfozShift()) - 1; } - // TODO(b/182800944): Investigate removing some of these fields: - // - control/slots can be derived from each other - - // The control bytes (and, also, a pointer near to the base of the backing - // array). - // - // This contains `capacity + 1 + NumClonedBytes()` entries, even - // when the table is empty (hence EmptyGroup). - // - // Note that growth_left is stored immediately before this pointer. - ctrl_t* control_ = EmptyGroup(); - - // The beginning of the slots, located at `SlotOffset()` bytes after - // `control`. May be null for empty tables. - void* slots_ = nullptr; + // We can't assert that SOO is enabled because we don't have SooEnabled(), but + // we assert what we can. + void AssertInSooMode() const { + assert(capacity() == SooCapacity()); + assert(!has_infoz()); + } // The number of slots in the backing array. This is always 2^N-1 for an // integer N. NOTE: we tried experimenting with compressing the capacity and @@ -1233,10 +1332,16 @@ class CommonFields : public CommonFieldsGenerationInfo { // power (N in 2^N-1), and (b) storing 2^N as the most significant bit of // size_ and storing size in the low bits. Both of these experiments were // regressions, presumably because we need capacity to do find operations. - size_t capacity_ = 0; + size_t capacity_; // The size and also has one bit that stores whether we have infoz. - size_t size_ = 0; + // TODO(b/289225379): we could put size_ into HeapOrSoo and make capacity_ + // encode the size in SOO case. We would be making size()/capacity() more + // expensive in order to have more SOO space. + size_t size_; + + // Either the control/slots pointers or the SOO slot. + HeapOrSoo heap_or_soo_; }; template @@ -1399,6 +1504,10 @@ inline bool AreItersFromSameContainer(const ctrl_t* ctrl_a, const void* const& slot_b) { // If either control byte is null, then we can't tell. if (ctrl_a == nullptr || ctrl_b == nullptr) return true; + const bool a_is_soo = IsSooControl(ctrl_a); + if (a_is_soo != IsSooControl(ctrl_b)) return false; + if (a_is_soo) return slot_a == slot_b; + const void* low_slot = slot_a; const void* hi_slot = slot_b; if (ctrl_a > ctrl_b) { @@ -1422,41 +1531,45 @@ inline void AssertSameContainer(const ctrl_t* ctrl_a, const ctrl_t* ctrl_b, // - use `ABSL_PREDICT_FALSE()` to provide a compiler hint for code layout // - use `ABSL_RAW_LOG()` with a format string to reduce code size and improve // the chances that the hot paths will be inlined. + + // fail_if(is_invalid, message) crashes when is_invalid is true and provides + // an error message based on `message`. + const auto fail_if = [](bool is_invalid, const char* message) { + if (ABSL_PREDICT_FALSE(is_invalid)) { + ABSL_RAW_LOG(FATAL, "Invalid iterator comparison. %s", message); + } + }; + const bool a_is_default = ctrl_a == EmptyGroup(); const bool b_is_default = ctrl_b == EmptyGroup(); - if (ABSL_PREDICT_FALSE(a_is_default != b_is_default)) { - ABSL_RAW_LOG( - FATAL, - "Invalid iterator comparison. Comparing default-constructed iterator " - "with non-default-constructed iterator."); - } if (a_is_default && b_is_default) return; + fail_if(a_is_default != b_is_default, + "Comparing default-constructed hashtable iterator with a " + "non-default-constructed hashtable iterator."); if (SwisstableGenerationsEnabled()) { if (ABSL_PREDICT_TRUE(generation_ptr_a == generation_ptr_b)) return; + // Users don't need to know whether the tables are SOO so don't mention SOO + // in the debug message. + const bool a_is_soo = IsSooControl(ctrl_a); + const bool b_is_soo = IsSooControl(ctrl_b); + fail_if(a_is_soo != b_is_soo || (a_is_soo && b_is_soo), + "Comparing iterators from different hashtables."); + const bool a_is_empty = IsEmptyGeneration(generation_ptr_a); const bool b_is_empty = IsEmptyGeneration(generation_ptr_b); - if (a_is_empty != b_is_empty) { - ABSL_RAW_LOG(FATAL, - "Invalid iterator comparison. Comparing iterator from a " - "non-empty hashtable with an iterator from an empty " - "hashtable."); - } - if (a_is_empty && b_is_empty) { - ABSL_RAW_LOG(FATAL, - "Invalid iterator comparison. Comparing iterators from " - "different empty hashtables."); - } + fail_if(a_is_empty != b_is_empty, + "Comparing an iterator from an empty hashtable with an iterator " + "from a non-empty hashtable."); + fail_if(a_is_empty && b_is_empty, + "Comparing iterators from different empty hashtables."); + const bool a_is_end = ctrl_a == nullptr; const bool b_is_end = ctrl_b == nullptr; - if (a_is_end || b_is_end) { - ABSL_RAW_LOG(FATAL, - "Invalid iterator comparison. Comparing iterator with an " - "end() iterator from a different hashtable."); - } - ABSL_RAW_LOG(FATAL, - "Invalid iterator comparison. Comparing non-end() iterators " - "from different hashtables."); + fail_if(a_is_end || b_is_end, + "Comparing iterator with an end() iterator from a different " + "hashtable."); + fail_if(true, "Comparing non-end() iterators from different hashtables."); } else { ABSL_HARDENING_ASSERT( AreItersFromSameContainer(ctrl_a, ctrl_b, slot_a, slot_b) && @@ -1548,31 +1661,49 @@ inline void ResetCtrl(CommonFields& common, size_t slot_size) { SanitizerPoisonMemoryRegion(common.slot_array(), slot_size * capacity); } -// Sets `ctrl[i]` to `h`. -// -// Unlike setting it directly, this function will perform bounds checks and -// mirror the value to the cloned tail if necessary. -inline void SetCtrl(const CommonFields& common, size_t i, ctrl_t h, - size_t slot_size) { - const size_t capacity = common.capacity(); - assert(i < capacity); - - auto* slot_i = static_cast(common.slot_array()) + i * slot_size; +// Sets sanitizer poisoning for slot corresponding to control byte being set. +inline void DoSanitizeOnSetCtrl(const CommonFields& c, size_t i, ctrl_t h, + size_t slot_size) { + assert(i < c.capacity()); + auto* slot_i = static_cast(c.slot_array()) + i * slot_size; if (IsFull(h)) { SanitizerUnpoisonMemoryRegion(slot_i, slot_size); } else { SanitizerPoisonMemoryRegion(slot_i, slot_size); } +} - ctrl_t* ctrl = common.control(); +// Sets `ctrl[i]` to `h`. +// +// Unlike setting it directly, this function will perform bounds checks and +// mirror the value to the cloned tail if necessary. +inline void SetCtrl(const CommonFields& c, size_t i, ctrl_t h, + size_t slot_size) { + DoSanitizeOnSetCtrl(c, i, h, slot_size); + ctrl_t* ctrl = c.control(); ctrl[i] = h; - ctrl[((i - NumClonedBytes()) & capacity) + (NumClonedBytes() & capacity)] = h; + ctrl[((i - NumClonedBytes()) & c.capacity()) + + (NumClonedBytes() & c.capacity())] = h; +} +// Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`. +inline void SetCtrl(const CommonFields& c, size_t i, h2_t h, size_t slot_size) { + SetCtrl(c, i, static_cast(h), slot_size); } +// Like SetCtrl, but in a single group table, we can save some operations when +// setting the cloned control byte. +inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, ctrl_t h, + size_t slot_size) { + assert(is_single_group(c.capacity())); + DoSanitizeOnSetCtrl(c, i, h, slot_size); + ctrl_t* ctrl = c.control(); + ctrl[i] = h; + ctrl[i + c.capacity() + 1] = h; +} // Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`. -inline void SetCtrl(const CommonFields& common, size_t i, h2_t h, - size_t slot_size) { - SetCtrl(common, i, static_cast(h), slot_size); +inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, h2_t h, + size_t slot_size) { + SetCtrlInSingleGroupTable(c, i, static_cast(h), slot_size); } // growth_left (which is a size_t) is stored with the backing array. @@ -1633,10 +1764,11 @@ ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots( // See GrowIntoSingleGroupShuffleControlBytes for details. class HashSetResizeHelper { public: - explicit HashSetResizeHelper(CommonFields& c) - : old_ctrl_(c.control()), - old_capacity_(c.capacity()), - had_infoz_(c.has_infoz()) {} + explicit HashSetResizeHelper(CommonFields& c, bool was_soo, bool had_soo_slot) + : old_capacity_(c.capacity()), + had_infoz_(c.has_infoz()), + was_soo_(was_soo), + had_soo_slot_(had_soo_slot) {} // Optimized for small groups version of `find_first_non_full`. // Beneficial only right after calling `raw_hash_set::resize`. @@ -1667,9 +1799,25 @@ class HashSetResizeHelper { return FindInfo{offset, 0}; } - ctrl_t* old_ctrl() const { return old_ctrl_; } + HeapOrSoo& old_heap_or_soo() { return old_heap_or_soo_; } + void* old_soo_data() { return old_heap_or_soo_.soo_data; } + ctrl_t* old_ctrl() const { + assert(!was_soo_); + return old_heap_or_soo_.heap.control; + } + void* old_slots() const { + assert(!was_soo_); + return old_heap_or_soo_.heap.slot_array.p; + } size_t old_capacity() const { return old_capacity_; } + // Returns the index of the SOO slot when growing from SOO to non-SOO in a + // single group. See also InitControlBytesAfterSoo(). It's important to use + // index 1 so that when resizing from capacity 1 to 3, we can still have + // random iteration order between the first two inserted elements. + // I.e. it allows inserting the second element at either index 0 or 2. + static size_t SooSlotIndex() { return 1; } + // Allocates a backing array for the hashtable. // Reads `capacity` and updates all other fields based on the result of // the allocation. @@ -1705,8 +1853,8 @@ class HashSetResizeHelper { // Returns IsGrowingIntoSingleGroupApplicable result to avoid recomputation. template - ABSL_ATTRIBUTE_NOINLINE bool InitializeSlots(CommonFields& c, void* old_slots, - Alloc alloc) { + ABSL_ATTRIBUTE_NOINLINE bool InitializeSlots(CommonFields& c, Alloc alloc, + ctrl_t soo_slot_h2) { assert(c.capacity()); // Folks with custom allocators often make unwarranted assumptions about the // behavior of their classes vis-a-vis trivial destructability and what @@ -1715,9 +1863,13 @@ class HashSetResizeHelper { // a workaround while we plan the exact guarantee we want to provide. const size_t sample_size = (std::is_same>::value && - c.slot_array() == nullptr) + (was_soo_ || old_capacity_ == 0)) ? SizeOfSlot : 0; + // TODO(b/289225379): when SOO is enabled, we should still sample on first + // insertion and if Sample is non-null, then we should force a heap + // allocation. Note that we'll also have to force capacity of 3 so that + // is_soo() still works. HashtablezInfoHandle infoz = sample_size > 0 ? Sample(sample_size) : c.infoz(); @@ -1735,10 +1887,15 @@ class HashSetResizeHelper { const bool grow_single_group = IsGrowingIntoSingleGroupApplicable(old_capacity_, layout.capacity()); - if (old_capacity_ != 0 && grow_single_group) { + if (was_soo_ && grow_single_group) { + InitControlBytesAfterSoo(c.control(), soo_slot_h2, layout.capacity()); + if (TransferUsesMemcpy && had_soo_slot_) { + TransferSlotAfterSoo(c, SizeOfSlot); + } + } else if (old_capacity_ != 0 && grow_single_group) { if (TransferUsesMemcpy) { - GrowSizeIntoSingleGroupTransferable(c, old_slots, SizeOfSlot); - DeallocateOld(alloc, SizeOfSlot, old_slots); + GrowSizeIntoSingleGroupTransferable(c, SizeOfSlot); + DeallocateOld(alloc, SizeOfSlot); } else { GrowIntoSingleGroupShuffleControlBytes(c.control(), layout.capacity()); } @@ -1749,7 +1906,7 @@ class HashSetResizeHelper { c.set_has_infoz(has_infoz); if (has_infoz) { infoz.RecordStorageChanged(c.size(), layout.capacity()); - if (grow_single_group || old_capacity_ == 0) { + if (was_soo_ || grow_single_group || old_capacity_ == 0) { infoz.RecordRehash(0); } c.set_infoz(infoz); @@ -1763,21 +1920,22 @@ class HashSetResizeHelper { // PRECONDITIONS: // 1. GrowIntoSingleGroupShuffleControlBytes was already called. template - void GrowSizeIntoSingleGroup(CommonFields& c, Alloc& alloc_ref, - typename PolicyTraits::slot_type* old_slots) { + void GrowSizeIntoSingleGroup(CommonFields& c, Alloc& alloc_ref) { assert(old_capacity_ < Group::kWidth / 2); assert(IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity())); using slot_type = typename PolicyTraits::slot_type; assert(is_single_group(c.capacity())); auto* new_slots = reinterpret_cast(c.slot_array()); + auto* old_slots_ptr = reinterpret_cast(old_slots()); size_t shuffle_bit = old_capacity_ / 2 + 1; for (size_t i = 0; i < old_capacity_; ++i) { - if (IsFull(old_ctrl_[i])) { + if (IsFull(old_ctrl()[i])) { size_t new_i = i ^ shuffle_bit; SanitizerUnpoisonMemoryRegion(new_slots + new_i, sizeof(slot_type)); - PolicyTraits::transfer(&alloc_ref, new_slots + new_i, old_slots + i); + PolicyTraits::transfer(&alloc_ref, new_slots + new_i, + old_slots_ptr + i); } } PoisonSingleGroupEmptySlots(c, sizeof(slot_type)); @@ -1785,11 +1943,11 @@ class HashSetResizeHelper { // Deallocates old backing array. template - void DeallocateOld(CharAlloc alloc_ref, size_t slot_size, void* old_slots) { - SanitizerUnpoisonMemoryRegion(old_slots, slot_size * old_capacity_); + void DeallocateOld(CharAlloc alloc_ref, size_t slot_size) { + SanitizerUnpoisonMemoryRegion(old_slots(), slot_size * old_capacity_); auto layout = RawHashSetLayout(old_capacity_, AlignOfSlot, had_infoz_); Deallocate( - &alloc_ref, old_ctrl_ - layout.control_offset(), + &alloc_ref, old_ctrl() - layout.control_offset(), layout.alloc_size(slot_size)); } @@ -1805,8 +1963,12 @@ class HashSetResizeHelper { // Relocates control bytes and slots into new single group for // transferable objects. // Must be called only if IsGrowingIntoSingleGroupApplicable returned true. - void GrowSizeIntoSingleGroupTransferable(CommonFields& c, void* old_slots, - size_t slot_size); + void GrowSizeIntoSingleGroupTransferable(CommonFields& c, size_t slot_size); + + // If there was an SOO slot and slots are transferable, transfers the SOO slot + // into the new heap allocation. Must be called only if + // IsGrowingIntoSingleGroupApplicable returned true. + void TransferSlotAfterSoo(CommonFields& c, size_t slot_size); // Shuffle control bits deterministically to the next capacity. // Returns offset for newly added element with given hash. @@ -1839,6 +2001,13 @@ class HashSetResizeHelper { void GrowIntoSingleGroupShuffleControlBytes(ctrl_t* new_ctrl, size_t new_capacity) const; + // If the table was SOO, initializes new control bytes. `h2` is the control + // byte corresponding to the full slot. Must be called only if + // IsGrowingIntoSingleGroupApplicable returned true. + // Requires: `had_soo_slot_ || h2 == ctrl_t::kEmpty`. + void InitControlBytesAfterSoo(ctrl_t* new_ctrl, ctrl_t h2, + size_t new_capacity); + // Shuffle trivially transferable slots in the way consistent with // GrowIntoSingleGroupShuffleControlBytes. // @@ -1852,8 +2021,7 @@ class HashSetResizeHelper { // 1. new_slots are transferred from old_slots_ consistent with // GrowIntoSingleGroupShuffleControlBytes. // 2. Empty new_slots are *not* poisoned. - void GrowIntoSingleGroupShuffleTransferableSlots(void* old_slots, - void* new_slots, + void GrowIntoSingleGroupShuffleTransferableSlots(void* new_slots, size_t slot_size) const; // Poison empty slots that were transferred using the deterministic algorithm @@ -1873,11 +2041,22 @@ class HashSetResizeHelper { } } - ctrl_t* old_ctrl_; + HeapOrSoo old_heap_or_soo_; size_t old_capacity_; bool had_infoz_; + bool was_soo_; + bool had_soo_slot_; }; +inline void PrepareInsertCommon(CommonFields& common) { + common.increment_size(); + common.maybe_increment_generation_on_insert(); +} + +// Like prepare_insert, but for the case of inserting into a full SOO table. +size_t PrepareInsertAfterSoo(size_t hash, size_t slot_size, + CommonFields& common); + // PolicyFunctions bundles together some information for a particular // raw_hash_set instantiation. This information is passed to // type-erased functions that want to do small amounts of type-specific @@ -1899,7 +2078,7 @@ struct PolicyFunctions { // or creating a new one based on the value of "reuse". // REQUIRES: c.capacity > 0 void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy, - bool reuse); + bool reuse, bool soo_enabled); // Type-erased version of raw_hash_set::erase_meta_only. void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size); @@ -1988,6 +2167,31 @@ class raw_hash_set { using key_arg = typename KeyArgImpl::template type; private: + // TODO(b/289225379): we could add extra SOO space inside raw_hash_set + // after CommonFields to allow inlining larger slot_types (e.g. std::string), + // but it's a bit complicated if we want to support incomplete mapped_type in + // flat_hash_map. We could potentially do this for flat_hash_set and for an + // allowlist of `mapped_type`s of flat_hash_map that includes e.g. arithmetic + // types, strings, cords, and pairs/tuples of allowlisted types. + constexpr static bool SooEnabled() { + return PolicyTraits::soo_enabled() && + sizeof(slot_type) <= sizeof(HeapOrSoo) && + alignof(slot_type) <= alignof(HeapOrSoo); + } + // TODO(b/289225379): this is used for pretty printing in GDB/LLDB, but if we + // use this instead of SooEnabled(), then we get compile errors in some OSS + // compilers due to incomplete mapped_type in flat_hash_map. We need to + // resolve this before launching SOO. + // constexpr static bool kSooEnabled = SooEnabled(); + + // Whether `size` fits in the SOO capacity of this table. + bool fits_in_soo(size_t size) const { + return SooEnabled() && size <= SooCapacity(); + } + // Whether this table is in SOO mode or non-SOO mode. + bool is_soo() const { return fits_in_soo(capacity()); } + bool is_full_soo() const { return is_soo() && !empty(); } + // Give an early error when key_type is not hashable/eq. auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); @@ -2102,6 +2306,18 @@ class raw_hash_set { // not equal to any end iterator. ABSL_ASSUME(ctrl != nullptr); } + // This constructor is used in begin() to avoid an MSan + // use-of-uninitialized-value error. Delegating from this constructor to + // the previous one doesn't avoid the error. + iterator(ctrl_t* ctrl, MaybeInitializedPtr slot, + const GenerationType* generation_ptr) + : HashSetIteratorGenerationInfo(generation_ptr), + ctrl_(ctrl), + slot_(to_slot(slot.p)) { + // This assumption helps the compiler know that any non-end iterator is + // not equal to any end iterator. + ABSL_ASSUME(ctrl != nullptr); + } // For end() iterators. explicit iterator(const GenerationType* generation_ptr) : HashSetIteratorGenerationInfo(generation_ptr), ctrl_(nullptr) {} @@ -2202,8 +2418,9 @@ class raw_hash_set { size_t bucket_count, const hasher& hash = hasher(), const key_equal& eq = key_equal(), const allocator_type& alloc = allocator_type()) - : settings_(CommonFields{}, hash, eq, alloc) { - if (bucket_count) { + : settings_(CommonFields::CreateDefault(), hash, eq, + alloc) { + if (bucket_count > (SooEnabled() ? SooCapacity() : 0)) { resize(NormalizeCapacity(bucket_count)); } } @@ -2310,6 +2527,15 @@ class raw_hash_set { if (size == 0) { return; } + // We don't use `that.is_soo()` here because `that` can have non-SOO + // capacity but have a size that fits into SOO capacity. + if (fits_in_soo(size)) { + assert(size == 1); + common().set_full_soo(); + emplace_at(soo_iterator(), *that.begin()); + return; + } + assert(!that.is_soo()); const size_t cap = capacity(); // Note about single group tables: // 1. It is correct to have any order of elements. @@ -2367,16 +2593,22 @@ class raw_hash_set { // would create a nullptr functor that cannot be called. // TODO(b/296061262): move instead of copying hash/eq/alloc. // Note: we avoid using exchange for better generated code. - settings_(std::move(that.common()), that.hash_ref(), that.eq_ref(), - that.alloc_ref()) { - that.common() = CommonFields{}; + settings_(PolicyTraits::transfer_uses_memcpy() || !that.is_full_soo() + ? std::move(that.common()) + : CommonFields{full_soo_tag_t{}}, + that.hash_ref(), that.eq_ref(), that.alloc_ref()) { + if (!PolicyTraits::transfer_uses_memcpy() && that.is_full_soo()) { + transfer(soo_slot(), that.soo_slot()); + } + that.common() = CommonFields::CreateDefault(); maybe_increment_generation_or_rehash_on_move(); } raw_hash_set(raw_hash_set&& that, const allocator_type& a) - : settings_(CommonFields{}, that.hash_ref(), that.eq_ref(), a) { + : settings_(CommonFields::CreateDefault(), that.hash_ref(), + that.eq_ref(), a) { if (a == that.alloc_ref()) { - std::swap(common(), that.common()); + swap_common(that); maybe_increment_generation_or_rehash_on_move(); } else { move_elements_allocs_unequal(std::move(that)); @@ -2411,9 +2643,10 @@ class raw_hash_set { ~raw_hash_set() { destructor_impl(); } iterator begin() ABSL_ATTRIBUTE_LIFETIME_BOUND { - // TODO(b/324478958): Consider reverting if no impact. if (ABSL_PREDICT_FALSE(empty())) return end(); - auto it = iterator_at(0); + if (is_soo()) return soo_iterator(); + iterator it = {control(), common().slots_union(), + common().generation_ptr()}; it.skip_empty_or_deleted(); assert(IsFull(*it.control())); return it; @@ -2435,7 +2668,15 @@ class raw_hash_set { bool empty() const { return !size(); } size_t size() const { return common().size(); } - size_t capacity() const { return common().capacity(); } + size_t capacity() const { + const size_t cap = common().capacity(); + // Use local variables because compiler complains about using functions in + // assume. + static constexpr bool kSooEnabled = SooEnabled(); + static constexpr size_t kSooCapacity = SooCapacity(); + ABSL_ASSUME(!kSooEnabled || cap >= kSooCapacity); + return cap; + } size_t max_size() const { return (std::numeric_limits::max)(); } ABSL_ATTRIBUTE_REINITIALIZES void clear() { @@ -2449,9 +2690,13 @@ class raw_hash_set { const size_t cap = capacity(); if (cap == 0) { // Already guaranteed to be empty; so nothing to do. + } else if (is_soo()) { + if (!empty()) destroy(soo_slot()); + common().set_empty_soo(); } else { destroy_slots(); - ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/cap < 128); + ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/cap < 128, + SooEnabled()); } common().set_reserved_growth(0); common().set_reservation_size(0); @@ -2582,7 +2827,7 @@ class raw_hash_set { std::pair emplace(Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND { alignas(slot_type) unsigned char raw[sizeof(slot_type)]; - slot_type* slot = reinterpret_cast(&raw); + slot_type* slot = to_slot(&raw); construct(slot, std::forward(args)...); const auto& elem = PolicyTraits::element(slot); @@ -2690,7 +2935,11 @@ class raw_hash_set { void erase(iterator it) { AssertIsFull(it.control(), it.generation(), it.generation_ptr(), "erase()"); destroy(it.slot()); - erase_meta_only(it); + if (is_soo()) { + common().set_empty_soo(); + } else { + erase_meta_only(it); + } } iterator erase(const_iterator first, @@ -2698,12 +2947,19 @@ class raw_hash_set { // We check for empty first because ClearBackingArray requires that // capacity() > 0 as a precondition. if (empty()) return end(); + if (first == last) return last.inner_; + if (is_soo()) { + destroy(soo_slot()); + common().set_empty_soo(); + return end(); + } if (first == begin() && last == end()) { // TODO(ezb): we access control bytes in destroy_slots so it could make // sense to combine destroy_slots and ClearBackingArray to avoid cache // misses when the table is large. Note that we also do this in clear(). destroy_slots(); - ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/true); + ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/true, + SooEnabled()); common().set_reserved_growth(common().reservation_size()); return end(); } @@ -2718,13 +2974,21 @@ class raw_hash_set { template void merge(raw_hash_set& src) { // NOLINT assert(this != &src); + // Returns whether insertion took place. + const auto insert_slot = [this](slot_type* src_slot) { + return PolicyTraits::apply(InsertSlot{*this, std::move(*src_slot)}, + PolicyTraits::element(src_slot)) + .second; + }; + + if (src.is_soo()) { + if (src.empty()) return; + if (insert_slot(src.soo_slot())) src.common().set_empty_soo(); + return; + } for (auto it = src.begin(), e = src.end(); it != e;) { auto next = std::next(it); - if (PolicyTraits::apply(InsertSlot{*this, std::move(*it.slot())}, - PolicyTraits::element(it.slot())) - .second) { - src.erase_meta_only(it); - } + if (insert_slot(it.slot())) src.erase_meta_only(it); it = next; } } @@ -2738,7 +3002,11 @@ class raw_hash_set { AssertIsFull(position.control(), position.inner_.generation(), position.inner_.generation_ptr(), "extract()"); auto node = CommonAccess::Transfer(alloc_ref(), position.slot()); - erase_meta_only(position); + if (is_soo()) { + common().set_empty_soo(); + } else { + erase_meta_only(position); + } return node; } @@ -2755,7 +3023,7 @@ class raw_hash_set { IsNoThrowSwappable( typename AllocTraits::propagate_on_container_swap{})) { using std::swap; - swap(common(), that.common()); + swap_common(that); swap(hash_ref(), that.hash_ref()); swap(eq_ref(), that.eq_ref()); SwapAlloc(alloc_ref(), that.alloc_ref(), @@ -2763,17 +3031,31 @@ class raw_hash_set { } void rehash(size_t n) { - if (n == 0 && capacity() == 0) return; - if (n == 0 && size() == 0) { - ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/false); - return; + const size_t cap = capacity(); + if (n == 0) { + if (cap == 0 || is_soo()) return; + if (empty()) { + ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/false, + SooEnabled()); + return; + } + if (SooEnabled() && size() <= SooCapacity()) { + alignas(slot_type) unsigned char slot_space[sizeof(slot_type)]; + slot_type* tmp_slot = to_slot(slot_space); + transfer(tmp_slot, begin().slot()); + ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/false, + SooEnabled()); + transfer(soo_slot(), tmp_slot); + common().set_full_soo(); + return; + } } // bitor is a faster way of doing `max` here. We will round up to the next // power-of-2-minus-1, so bitor is good enough. auto m = NormalizeCapacity(n | GrowthToLowerboundCapacity(size())); // n == 0 unconditionally rehashes as per the standard. - if (n == 0 || m > capacity()) { + if (n == 0 || m > cap) { resize(m); // This is after resize, to ensure that we have completed the allocation @@ -2783,7 +3065,9 @@ class raw_hash_set { } void reserve(size_t n) { - if (n > size() + growth_left()) { + const size_t max_size_before_growth = + is_soo() ? SooCapacity() : size() + growth_left(); + if (n > max_size_before_growth) { size_t m = GrowthToLowerboundCapacity(n); resize(NormalizeCapacity(m)); @@ -2816,6 +3100,7 @@ class raw_hash_set { // specific benchmarks indicating its importance. template void prefetch(const key_arg& key) const { + if (SooEnabled() ? is_soo() : capacity() == 0) return; (void)key; // Avoid probing if we won't be able to prefetch the addresses received. #ifdef ABSL_HAVE_PREFETCH @@ -2836,26 +3121,14 @@ class raw_hash_set { template iterator find(const key_arg& key, size_t hash) ABSL_ATTRIBUTE_LIFETIME_BOUND { - auto seq = probe(common(), hash); - slot_type* slot_ptr = slot_array(); - const ctrl_t* ctrl = control(); - while (true) { - Group g{ctrl + seq.offset()}; - for (uint32_t i : g.Match(H2(hash))) { - if (ABSL_PREDICT_TRUE(PolicyTraits::apply( - EqualElement{key, eq_ref()}, - PolicyTraits::element(slot_ptr + seq.offset(i))))) - return iterator_at(seq.offset(i)); - } - if (ABSL_PREDICT_TRUE(g.MaskEmpty())) return end(); - seq.next(); - assert(seq.index() <= capacity() && "full table!"); - } + if (is_soo()) return find_soo(key); + return find_non_soo(key, hash); } template iterator find(const key_arg& key) ABSL_ATTRIBUTE_LIFETIME_BOUND { + if (is_soo()) return find_soo(key); prefetch_heap_block(); - return find(key, hash_ref()(key)); + return find_non_soo(key, hash_ref()(key)); } template @@ -3007,7 +3280,39 @@ class raw_hash_set { PolicyTraits::transfer(&alloc_ref(), to, from); } + // TODO(b/289225379): consider having a helper class that has the impls for + // SOO functionality. + template + iterator find_soo(const key_arg& key) { + assert(is_soo()); + return empty() || !PolicyTraits::apply(EqualElement{key, eq_ref()}, + PolicyTraits::element(soo_slot())) + ? end() + : soo_iterator(); + } + + template + iterator find_non_soo(const key_arg& key, size_t hash) { + assert(!is_soo()); + auto seq = probe(common(), hash); + slot_type* slot_ptr = slot_array(); + const ctrl_t* ctrl = control(); + while (true) { + Group g{ctrl + seq.offset()}; + for (uint32_t i : g.Match(H2(hash))) { + if (ABSL_PREDICT_TRUE(PolicyTraits::apply( + EqualElement{key, eq_ref()}, + PolicyTraits::element(slot_ptr + seq.offset(i))))) + return iterator_at(seq.offset(i)); + } + if (ABSL_PREDICT_TRUE(g.MaskEmpty())) return end(); + seq.next(); + assert(seq.index() <= capacity() && "full table!"); + } + } + inline void destroy_slots() { + assert(!is_soo()); if (PolicyTraits::template destroy_is_trivial()) return; IterateOverFullSlots( common(), slot_array(), @@ -3027,6 +3332,10 @@ class raw_hash_set { inline void destructor_impl() { if (capacity() == 0) return; + if (is_soo()) { + if (!empty()) destroy(soo_slot()); + return; + } destroy_slots(); dealloc(); } @@ -3036,10 +3345,16 @@ class raw_hash_set { // This merely updates the pertinent control byte. This can be used in // conjunction with Policy::transfer to move the object to another place. void erase_meta_only(const_iterator it) { + assert(!is_soo()); EraseMetaOnly(common(), static_cast(it.control() - control()), sizeof(slot_type)); } + size_t hash_of(slot_type* slot) const { + return PolicyTraits::apply(HashElement{hash_ref()}, + PolicyTraits::element(slot)); + } + // Resizes table to the new capacity and move all elements to the new // positions accordingly. // @@ -3050,8 +3365,23 @@ class raw_hash_set { // can be called right after `resize`. ABSL_ATTRIBUTE_NOINLINE void resize(size_t new_capacity) { assert(IsValidCapacity(new_capacity)); - HashSetResizeHelper resize_helper(common()); - auto* old_slots = slot_array(); + assert(!fits_in_soo(new_capacity)); + const bool was_soo = is_soo(); + const bool had_soo_slot = was_soo && !empty(); + const ctrl_t soo_slot_h2 = + had_soo_slot ? static_cast(H2(hash_of(soo_slot()))) + : ctrl_t::kEmpty; + HashSetResizeHelper resize_helper(common(), was_soo, had_soo_slot); + // Initialize HashSetResizeHelper::old_heap_or_soo_. We can't do this in + // HashSetResizeHelper constructor because it can't transfer slots when + // transfer_uses_memcpy is false. + // TODO(b/289225379): try to handle more of the SOO cases inside + // InitializeSlots. See comment on cl/555990034 snapshot #63. + if (PolicyTraits::transfer_uses_memcpy() || !had_soo_slot) { + resize_helper.old_heap_or_soo() = common().heap_or_soo(); + } else { + transfer(to_slot(resize_helper.old_soo_data()), soo_slot()); + } common().set_capacity(new_capacity); // Note that `InitializeSlots` does different number initialization steps // depending on the values of `transfer_uses_memcpy` and capacities. @@ -3060,43 +3390,60 @@ class raw_hash_set { resize_helper.InitializeSlots( - common(), const_cast*>(old_slots), - CharAlloc(alloc_ref())); + common(), CharAlloc(alloc_ref()), soo_slot_h2); - if (resize_helper.old_capacity() == 0) { + // In the SooEnabled() case, capacity is never 0 so we don't check. + if (!SooEnabled() && resize_helper.old_capacity() == 0) { // InitializeSlots did all the work including infoz().RecordRehash(). return; } + assert(resize_helper.old_capacity() > 0); + // Nothing more to do in this case. + if (was_soo && !had_soo_slot) return; + slot_type* new_slots = slot_array(); if (grow_single_group) { if (PolicyTraits::transfer_uses_memcpy()) { // InitializeSlots did all the work. return; } - // We want GrowSizeIntoSingleGroup to be called here in order to make - // InitializeSlots not depend on PolicyTraits. - resize_helper.GrowSizeIntoSingleGroup(common(), alloc_ref(), - old_slots); + if (was_soo) { + transfer(new_slots + resize_helper.SooSlotIndex(), + to_slot(resize_helper.old_soo_data())); + return; + } else { + // We want GrowSizeIntoSingleGroup to be called here in order to make + // InitializeSlots not depend on PolicyTraits. + resize_helper.GrowSizeIntoSingleGroup(common(), + alloc_ref()); + } } else { // InitializeSlots prepares control bytes to correspond to empty table. - auto* new_slots = slot_array(); - size_t total_probe_length = 0; - for (size_t i = 0; i != resize_helper.old_capacity(); ++i) { - if (IsFull(resize_helper.old_ctrl()[i])) { - size_t hash = PolicyTraits::apply( - HashElement{hash_ref()}, PolicyTraits::element(old_slots + i)); - auto target = find_first_non_full(common(), hash); - size_t new_i = target.offset; - total_probe_length += target.probe_length; - SetCtrl(common(), new_i, H2(hash), sizeof(slot_type)); - transfer(new_slots + new_i, old_slots + i); + const auto insert_slot = [&](slot_type* slot) { + size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, + PolicyTraits::element(slot)); + auto target = find_first_non_full(common(), hash); + SetCtrl(common(), target.offset, H2(hash), sizeof(slot_type)); + transfer(new_slots + target.offset, slot); + return target.probe_length; + }; + if (was_soo) { + insert_slot(to_slot(resize_helper.old_soo_data())); + return; + } else { + auto* old_slots = + reinterpret_cast(resize_helper.old_slots()); + size_t total_probe_length = 0; + for (size_t i = 0; i != resize_helper.old_capacity(); ++i) { + if (IsFull(resize_helper.old_ctrl()[i])) { + total_probe_length += insert_slot(old_slots + i); + } } + infoz().RecordRehash(total_probe_length); } - infoz().RecordRehash(total_probe_length); } - resize_helper.DeallocateOld( - CharAlloc(alloc_ref()), sizeof(slot_type), - const_cast*>(old_slots)); + resize_helper.DeallocateOld(CharAlloc(alloc_ref()), + sizeof(slot_type)); } // Prunes control bytes to remove as many tombstones as possible. @@ -3166,8 +3513,46 @@ class raw_hash_set { } } + // Casting directly from e.g. char* to slot_type* can cause compilation errors + // on objective-C. This function converts to void* first, avoiding the issue. + static slot_type* to_slot(void* buf) { + return reinterpret_cast(buf); + } + + // Requires that lhs does not have a full SOO slot. + static void move_common(bool that_is_full_soo, allocator_type& rhs_alloc, + CommonFields& lhs, CommonFields&& rhs) { + if (PolicyTraits::transfer_uses_memcpy() || !that_is_full_soo) { + lhs = std::move(rhs); + } else { + lhs.move_non_heap_or_soo_fields(rhs); + // TODO(b/303305702): add reentrancy guard. + PolicyTraits::transfer(&rhs_alloc, to_slot(lhs.soo_data()), + to_slot(rhs.soo_data())); + } + } + + // Swaps common fields making sure to avoid memcpy'ing a full SOO slot if we + // aren't allowed to do so. + void swap_common(raw_hash_set& that) { + using std::swap; + if (PolicyTraits::transfer_uses_memcpy()) { + swap(common(), that.common()); + return; + } + CommonFields tmp = CommonFields::CreateDefault(); + const bool that_is_full_soo = that.is_full_soo(); + move_common(that_is_full_soo, that.alloc_ref(), tmp, + std::move(that.common())); + move_common(is_full_soo(), alloc_ref(), that.common(), std::move(common())); + move_common(that_is_full_soo, that.alloc_ref(), common(), std::move(tmp)); + } + void maybe_increment_generation_or_rehash_on_move() { - common().maybe_increment_generation_on_move(); + if (!SwisstableGenerationsEnabled() || capacity() == 0 || is_soo()) { + return; + } + common().increment_generation(); if (!empty() && common().should_rehash_for_bug_detection_on_move()) { resize(capacity()); } @@ -3178,13 +3563,14 @@ class raw_hash_set { // We don't bother checking for this/that aliasing. We just need to avoid // breaking the invariants in that case. destructor_impl(); - common() = std::move(that.common()); + move_common(that.is_full_soo(), that.alloc_ref(), common(), + std::move(that.common())); // TODO(b/296061262): move instead of copying hash/eq/alloc. hash_ref() = that.hash_ref(); eq_ref() = that.eq_ref(); CopyAlloc(alloc_ref(), that.alloc_ref(), std::integral_constant()); - that.common() = CommonFields{}; + that.common() = CommonFields::CreateDefault(); maybe_increment_generation_or_rehash_on_move(); return *this; } @@ -3197,8 +3583,8 @@ class raw_hash_set { insert(std::move(PolicyTraits::element(it.slot()))); that.destroy(it.slot()); } - that.dealloc(); - that.common() = CommonFields{}; + if (!that.is_soo()) that.dealloc(); + that.common() = CommonFields::CreateDefault(); maybe_increment_generation_or_rehash_on_move(); return *this; } @@ -3230,6 +3616,20 @@ class raw_hash_set { // `key`'s H2. Returns a bool indicating whether an insertion can take place. template std::pair find_or_prepare_insert(const K& key) { + if (is_soo()) { + if (empty()) { + common().set_full_soo(); + return {soo_iterator(), true}; + } + if (PolicyTraits::apply(EqualElement{key, eq_ref()}, + PolicyTraits::element(soo_slot()))) { + return {soo_iterator(), false}; + } + resize(NextCapacity(SooCapacity())); + const size_t index = + PrepareInsertAfterSoo(hash_ref()(key), sizeof(slot_type), common()); + return {iterator_at(index), true}; + } prefetch_heap_block(); auto hash = hash_ref()(key); auto seq = probe(common(), hash); @@ -3254,6 +3654,7 @@ class raw_hash_set { // // REQUIRES: At least one non-full slot available. size_t prepare_insert(size_t hash) ABSL_ATTRIBUTE_NOINLINE { + assert(!is_soo()); const bool rehash_for_bug_detection = common().should_rehash_for_bug_detection_on_insert(); if (rehash_for_bug_detection) { @@ -3277,10 +3678,9 @@ class raw_hash_set { } else { target = find_first_non_full(common(), hash); } - common().increment_size(); + PrepareInsertCommon(common()); set_growth_left(growth_left() - IsEmpty(control()[target.offset])); SetCtrl(common(), target.offset, H2(hash), sizeof(slot_type)); - common().maybe_increment_generation_on_insert(); infoz().RecordInsert(hash, target.probe_length); return target.offset; } @@ -3305,7 +3705,7 @@ class raw_hash_set { return {control() + i, slot_array() + i, common().generation_ptr()}; } const_iterator iterator_at(size_t i) const ABSL_ATTRIBUTE_LIFETIME_BOUND { - return {control() + i, slot_array() + i, common().generation_ptr()}; + return const_cast(this)->iterator_at(i); } reference unchecked_deref(iterator it) { return it.unchecked_deref(); } @@ -3323,13 +3723,20 @@ class raw_hash_set { // side-effect. // // See `CapacityToGrowth()`. - size_t growth_left() const { return common().growth_left(); } - void set_growth_left(size_t gl) { return common().set_growth_left(gl); } + size_t growth_left() const { + assert(!is_soo()); + return common().growth_left(); + } + void set_growth_left(size_t gl) { + assert(!is_soo()); + return common().set_growth_left(gl); + } // Prefetch the heap-allocated memory region to resolve potential TLB and // cache misses. This is intended to overlap with execution of calculating the // hash for a key. void prefetch_heap_block() const { + if (is_soo()) return; #if ABSL_HAVE_BUILTIN(__builtin_prefetch) || defined(__GNUC__) __builtin_prefetch(control(), 0, 1); #endif @@ -3338,11 +3745,43 @@ class raw_hash_set { CommonFields& common() { return settings_.template get<0>(); } const CommonFields& common() const { return settings_.template get<0>(); } - ctrl_t* control() const { return common().control(); } + ctrl_t* control() const { + assert(!is_soo()); + return common().control(); + } slot_type* slot_array() const { + assert(!is_soo()); + // Suppress erroneous uninitialized memory errors on GCC. GCC thinks that + // the call to slot_array() in find_or_prepare_insert() is reading + // uninitialized memory, but slot_array is only called there when the table + // is non-empty and this memory is initialized when the table is non-empty. +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#pragma GCC diagnostic ignored "-Wuninitialized" +#endif return static_cast(common().slot_array()); +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + } + slot_type* soo_slot() { + assert(is_soo()); + return static_cast(common().soo_data()); + } + const slot_type* soo_slot() const { + return reinterpret_cast(this)->soo_slot(); + } + iterator soo_iterator() { + return {SooControl(), soo_slot(), common().generation_ptr()}; + } + const_iterator soo_iterator() const { + return reinterpret_cast(this)->soo_iterator(); + } + HashtablezInfoHandle infoz() { + assert(!is_soo()); + return common().infoz(); } - HashtablezInfoHandle infoz() { return common().infoz(); } hasher& hash_ref() { return settings_.template get<1>(); } const hasher& hash_ref() const { return settings_.template get<1>(); } @@ -3390,7 +3829,8 @@ class raw_hash_set { // fields that occur after CommonFields. absl::container_internal::CompressedTuple - settings_{CommonFields{}, hasher{}, key_equal{}, allocator_type{}}; + settings_{CommonFields::CreateDefault(), hasher{}, + key_equal{}, allocator_type{}}; }; // Erases all elements that satisfy the predicate `pred` from the container `c`. @@ -3416,6 +3856,7 @@ struct HashtableDebugAccess> { static size_t GetNumProbes(const Set& set, const typename Set::key_type& key) { + if (set.is_soo()) return 0; size_t num_probes = 0; size_t hash = set.hash_ref()(key); auto seq = probe(set.common(), hash); @@ -3439,7 +3880,8 @@ struct HashtableDebugAccess> { static size_t AllocatedByteSize(const Set& c) { size_t capacity = c.capacity(); if (capacity == 0) return 0; - size_t m = c.common().alloc_size(sizeof(Slot), alignof(Slot)); + size_t m = + c.is_soo() ? 0 : c.common().alloc_size(sizeof(Slot), alignof(Slot)); size_t per_slot = Traits::space_used(static_cast(nullptr)); if (per_slot != ~size_t{}) { diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 109340c7..9180db59 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -394,7 +394,7 @@ TEST(Group, CountLeadingEmptyOrDeleted) { } } -template +template struct ValuePolicy { using slot_type = T; using key_type = T; @@ -433,6 +433,8 @@ struct ValuePolicy { static constexpr HashSlotFn get_hash_slot_fn() { return nullptr; } + + static constexpr bool soo_enabled() { return kSoo; } }; using IntPolicy = ValuePolicy; @@ -440,6 +442,44 @@ using Uint8Policy = ValuePolicy; using TranferableIntPolicy = ValuePolicy; +// For testing SOO. +template +class SizedValue { + public: + SizedValue(int64_t v) { // NOLINT + vals_[0] = v; + } + SizedValue() : SizedValue(0) {} + SizedValue(const SizedValue&) = default; + SizedValue& operator=(const SizedValue&) = default; + + int64_t operator*() const { + // Suppress erroneous uninitialized memory errors on GCC. +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + return vals_[0]; +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + } + explicit operator int() const { return **this; } + explicit operator int64_t() const { return **this; } + + template + friend H AbslHashValue(H h, SizedValue sv) { + return H::combine(std::move(h), *sv); + } + bool operator==(const SizedValue& rhs) const { return **this == *rhs; } + + private: + int64_t vals_[N / sizeof(int64_t)]; +}; +template +using SizedValuePolicy = + ValuePolicy, /*kTransferable=*/true, kSoo>; + class StringPolicy { template +template struct ValueTable - : raw_hash_set, hash_default_hash, + : raw_hash_set, hash_default_hash, std::equal_to, std::allocator> { using Base = typename ValueTable::raw_hash_set; using Base::Base; @@ -530,6 +570,11 @@ using Uint8Table = ValueTable; using TransferableIntTable = ValueTable; +constexpr size_t kNonSooSize = sizeof(HeapOrSoo) + 8; +static_assert(sizeof(SizedValue) >= kNonSooSize, "too small"); +using NonSooIntTable = ValueTable>; +using SooIntTable = ValueTable; + template struct CustomAlloc : std::allocator { CustomAlloc() = default; @@ -579,6 +624,16 @@ struct FreezableAlloc : std::allocator { bool* frozen; }; +template +struct FreezableSizedValueSooTable + : raw_hash_set, + container_internal::hash_default_hash>, + std::equal_to>, + FreezableAlloc>> { + using Base = typename FreezableSizedValueSooTable::raw_hash_set; + using Base::Base; +}; + struct BadFastHash { template size_t operator()(const T&) const { @@ -649,20 +704,25 @@ TEST(Table, EmptyFunctorOptimization) { std::equal_to, std::allocator>)); } -TEST(Table, Empty) { - IntTable t; +template +class SooTest : public testing::Test {}; + +TYPED_TEST_SUITE_P(SooTest); + +TYPED_TEST_P(SooTest, Empty) { + TypeParam t; EXPECT_EQ(0, t.size()); EXPECT_TRUE(t.empty()); } -TEST(Table, LookupEmpty) { - IntTable t; +TYPED_TEST_P(SooTest, LookupEmpty) { + TypeParam t; auto it = t.find(0); EXPECT_TRUE(it == t.end()); } -TEST(Table, Insert1) { - IntTable t; +TYPED_TEST_P(SooTest, Insert1) { + TypeParam t; EXPECT_TRUE(t.find(0) == t.end()); auto res = t.emplace(0); EXPECT_TRUE(res.second); @@ -671,8 +731,8 @@ TEST(Table, Insert1) { EXPECT_THAT(*t.find(0), 0); } -TEST(Table, Insert2) { - IntTable t; +TYPED_TEST_P(SooTest, Insert2) { + TypeParam t; EXPECT_TRUE(t.find(0) == t.end()); auto res = t.emplace(0); EXPECT_TRUE(res.second); @@ -734,9 +794,9 @@ TEST(Table, InsertCollisionAndFindAfterDelete) { EXPECT_TRUE(t.empty()); } -TEST(Table, EraseInSmallTables) { +TYPED_TEST_P(SooTest, EraseInSmallTables) { for (int64_t size = 0; size < 64; ++size) { - IntTable t; + TypeParam t; for (int64_t i = 0; i < size; ++i) { t.insert(i); } @@ -751,8 +811,8 @@ TEST(Table, EraseInSmallTables) { } } -TEST(Table, InsertWithinCapacity) { - IntTable t; +TYPED_TEST_P(SooTest, InsertWithinCapacity) { + TypeParam t; t.reserve(10); const size_t original_capacity = t.capacity(); const auto addr = [&](int i) { @@ -841,7 +901,8 @@ TYPED_TEST_P(SmallTableResizeTest, ResizeReduceSmallTables) { REGISTER_TYPED_TEST_SUITE_P(SmallTableResizeTest, InsertIntoSmallTable, ResizeGrowSmallTables, ResizeReduceSmallTables); -using SmallTableTypes = ::testing::Types; +using SmallTableTypes = + ::testing::Types; INSTANTIATE_TYPED_TEST_SUITE_P(InstanceSmallTableResizeTest, SmallTableResizeTest, SmallTableTypes); @@ -863,14 +924,14 @@ TEST(Table, LazyEmplace) { EXPECT_THAT(*it, Pair("abc", "ABC")); } -TEST(Table, ContainsEmpty) { - IntTable t; +TYPED_TEST_P(SooTest, ContainsEmpty) { + TypeParam t; EXPECT_FALSE(t.contains(0)); } -TEST(Table, Contains1) { - IntTable t; +TYPED_TEST_P(SooTest, Contains1) { + TypeParam t; EXPECT_TRUE(t.insert(0).second); EXPECT_TRUE(t.contains(0)); @@ -880,8 +941,8 @@ TEST(Table, Contains1) { EXPECT_FALSE(t.contains(0)); } -TEST(Table, Contains2) { - IntTable t; +TYPED_TEST_P(SooTest, Contains2) { + TypeParam t; EXPECT_TRUE(t.insert(0).second); EXPECT_TRUE(t.contains(0)); @@ -1178,8 +1239,8 @@ TEST(Table, RehashWithNoResize) { } } -TEST(Table, InsertEraseStressTest) { - IntTable t; +TYPED_TEST_P(SooTest, InsertEraseStressTest) { + TypeParam t; const size_t kMinElementCount = 250; std::deque keys; size_t i = 0; @@ -1207,32 +1268,33 @@ TEST(Table, InsertOverloads) { Pair("DEF", "!!!"))); } -TEST(Table, LargeTable) { - IntTable t; +TYPED_TEST_P(SooTest, LargeTable) { + TypeParam t; for (int64_t i = 0; i != 100000; ++i) t.emplace(i << 40); - for (int64_t i = 0; i != 100000; ++i) ASSERT_EQ(i << 40, *t.find(i << 40)); + for (int64_t i = 0; i != 100000; ++i) + ASSERT_EQ(i << 40, static_cast(*t.find(i << 40))); } // Timeout if copy is quadratic as it was in Rust. -TEST(Table, EnsureNonQuadraticAsInRust) { +TYPED_TEST_P(SooTest, EnsureNonQuadraticAsInRust) { static const size_t kLargeSize = 1 << 15; - IntTable t; + TypeParam t; for (size_t i = 0; i != kLargeSize; ++i) { t.insert(i); } // If this is quadratic, the test will timeout. - IntTable t2; + TypeParam t2; for (const auto& entry : t) t2.insert(entry); } -TEST(Table, ClearBug) { +TYPED_TEST_P(SooTest, ClearBug) { if (SwisstableGenerationsEnabled()) { GTEST_SKIP() << "Generations being enabled causes extra rehashes."; } - IntTable t; + TypeParam t; constexpr size_t capacity = container_internal::Group::kWidth - 1; constexpr size_t max_size = capacity / 2 + 1; for (size_t i = 0; i < max_size; ++i) { @@ -1251,11 +1313,11 @@ TEST(Table, ClearBug) { // that they are probably still in the same group. This is not strictly // guaranteed. EXPECT_LT(static_cast(std::abs(original - second)), - capacity * sizeof(IntTable::value_type)); + capacity * sizeof(typename TypeParam::value_type)); } -TEST(Table, Erase) { - IntTable t; +TYPED_TEST_P(SooTest, Erase) { + TypeParam t; EXPECT_TRUE(t.find(0) == t.end()); auto res = t.emplace(0); EXPECT_TRUE(res.second); @@ -1265,8 +1327,8 @@ TEST(Table, Erase) { EXPECT_TRUE(t.find(0) == t.end()); } -TEST(Table, EraseMaintainsValidIterator) { - IntTable t; +TYPED_TEST_P(SooTest, EraseMaintainsValidIterator) { + TypeParam t; const int kNumElements = 100; for (int i = 0; i < kNumElements; i++) { EXPECT_TRUE(t.emplace(i).second); @@ -1284,8 +1346,8 @@ TEST(Table, EraseMaintainsValidIterator) { EXPECT_EQ(num_erase_calls, kNumElements); } -TEST(Table, EraseBeginEnd) { - IntTable t; +TYPED_TEST_P(SooTest, EraseBeginEnd) { + TypeParam t; for (int i = 0; i < 10; ++i) t.insert(i); EXPECT_EQ(t.size(), 10); t.erase(t.begin(), t.end()); @@ -1684,8 +1746,8 @@ TEST(Table, EraseInsertProbing) { EXPECT_THAT(t, UnorderedElementsAre(1, 10, 3, 11, 12)); } -TEST(Table, Clear) { - IntTable t; +TYPED_TEST_P(SooTest, Clear) { + TypeParam t; EXPECT_TRUE(t.find(0) == t.end()); t.clear(); EXPECT_TRUE(t.find(0) == t.end()); @@ -1697,13 +1759,13 @@ TEST(Table, Clear) { EXPECT_TRUE(t.find(0) == t.end()); } -TEST(Table, Swap) { - IntTable t; +TYPED_TEST_P(SooTest, Swap) { + TypeParam t; EXPECT_TRUE(t.find(0) == t.end()); auto res = t.emplace(0); EXPECT_TRUE(res.second); EXPECT_EQ(1, t.size()); - IntTable u; + TypeParam u; t.swap(u); EXPECT_EQ(0, t.size()); EXPECT_EQ(1, u.size()); @@ -1711,8 +1773,8 @@ TEST(Table, Swap) { EXPECT_THAT(*u.find(0), 0); } -TEST(Table, Rehash) { - IntTable t; +TYPED_TEST_P(SooTest, Rehash) { + TypeParam t; EXPECT_TRUE(t.find(0) == t.end()); t.emplace(0); t.emplace(1); @@ -1723,8 +1785,8 @@ TEST(Table, Rehash) { EXPECT_THAT(*t.find(1), 1); } -TEST(Table, RehashDoesNotRehashWhenNotNecessary) { - IntTable t; +TYPED_TEST_P(SooTest, RehashDoesNotRehashWhenNotNecessary) { + TypeParam t; t.emplace(0); t.emplace(1); auto* p = &*t.find(0); @@ -1732,14 +1794,15 @@ TEST(Table, RehashDoesNotRehashWhenNotNecessary) { EXPECT_EQ(p, &*t.find(0)); } +// Following two tests use non-SOO table because they test for 0 capacity. TEST(Table, RehashZeroDoesNotAllocateOnEmptyTable) { - IntTable t; + NonSooIntTable t; t.rehash(0); EXPECT_EQ(0, t.bucket_count()); } TEST(Table, RehashZeroDeallocatesEmptyTable) { - IntTable t; + NonSooIntTable t; t.emplace(0); t.clear(); EXPECT_NE(0, t.bucket_count()); @@ -1747,8 +1810,8 @@ TEST(Table, RehashZeroDeallocatesEmptyTable) { EXPECT_EQ(0, t.bucket_count()); } -TEST(Table, RehashZeroForcesRehash) { - IntTable t; +TYPED_TEST_P(SooTest, RehashZeroForcesRehash) { + TypeParam t; t.emplace(0); t.emplace(1); auto* p = &*t.find(0); @@ -1764,33 +1827,33 @@ TEST(Table, ConstructFromInitList) { StringTable t = {P(), Q(), {}, {{}, {}}}; } -TEST(Table, CopyConstruct) { - IntTable t; +TYPED_TEST_P(SooTest, CopyConstruct) { + TypeParam t; t.emplace(0); EXPECT_EQ(1, t.size()); { - IntTable u(t); + TypeParam u(t); EXPECT_EQ(1, u.size()); EXPECT_THAT(*u.find(0), 0); } { - IntTable u{t}; + TypeParam u{t}; EXPECT_EQ(1, u.size()); EXPECT_THAT(*u.find(0), 0); } { - IntTable u = t; + TypeParam u = t; EXPECT_EQ(1, u.size()); EXPECT_THAT(*u.find(0), 0); } } -TEST(Table, CopyDifferentSizes) { - IntTable t; +TYPED_TEST_P(SooTest, CopyDifferentSizes) { + TypeParam t; for (int i = 0; i < 100; ++i) { t.emplace(i); - IntTable c = t; + TypeParam c = t; for (int j = 0; j <= i; ++j) { ASSERT_TRUE(c.find(j) != c.end()) << "i=" << i << " j=" << j; } @@ -1799,16 +1862,16 @@ TEST(Table, CopyDifferentSizes) { } } -TEST(Table, CopyDifferentCapacities) { +TYPED_TEST_P(SooTest, CopyDifferentCapacities) { for (int cap = 1; cap < 100; cap = cap * 2 + 1) { - IntTable t; + TypeParam t; t.reserve(static_cast(cap)); for (int i = 0; i <= cap; ++i) { t.emplace(i); if (i != cap && i % 5 != 0) { continue; } - IntTable c = t; + TypeParam c = t; for (int j = 0; j <= i; ++j) { ASSERT_TRUE(c.find(j) != c.end()) << "cap=" << cap << " i=" << i << " j=" << j; @@ -1948,8 +2011,8 @@ TEST(Table, Equality3) { EXPECT_NE(u, t); } -TEST(Table, NumDeletedRegression) { - IntTable t; +TYPED_TEST_P(SooTest, NumDeletedRegression) { + TypeParam t; t.emplace(0); t.erase(t.find(0)); // construct over a deleted slot. @@ -1957,8 +2020,8 @@ TEST(Table, NumDeletedRegression) { t.clear(); } -TEST(Table, FindFullDeletedRegression) { - IntTable t; +TYPED_TEST_P(SooTest, FindFullDeletedRegression) { + TypeParam t; for (int i = 0; i < 1000; ++i) { t.emplace(i); t.erase(t.find(i)); @@ -1966,17 +2029,17 @@ TEST(Table, FindFullDeletedRegression) { EXPECT_EQ(0, t.size()); } -TEST(Table, ReplacingDeletedSlotDoesNotRehash) { +TYPED_TEST_P(SooTest, ReplacingDeletedSlotDoesNotRehash) { size_t n; { // Compute n such that n is the maximum number of elements before rehash. - IntTable t; + TypeParam t; t.emplace(0); size_t c = t.bucket_count(); for (n = 1; c == t.bucket_count(); ++n) t.emplace(n); --n; } - IntTable t; + TypeParam t; t.rehash(n); const size_t c = t.bucket_count(); for (size_t i = 0; i != n; ++i) t.emplace(i); @@ -2227,8 +2290,8 @@ TEST(Nodes, ExtractInsert) { EXPECT_FALSE(node); // NOLINT(bugprone-use-after-move) } -TEST(Nodes, HintInsert) { - IntTable t = {1, 2, 3}; +TYPED_TEST_P(SooTest, HintInsert) { + TypeParam t = {1, 2, 3}; auto node = t.extract(1); EXPECT_THAT(t, UnorderedElementsAre(2, 3)); auto it = t.insert(t.begin(), std::move(node)); @@ -2247,14 +2310,18 @@ TEST(Nodes, HintInsert) { EXPECT_TRUE(node); // NOLINT(bugprone-use-after-move) } -IntTable MakeSimpleTable(size_t size) { - IntTable t; +template +T MakeSimpleTable(size_t size) { + T t; while (t.size() < size) t.insert(t.size()); return t; } -std::vector OrderOfIteration(const IntTable& t) { - return {t.begin(), t.end()}; +template +std::vector OrderOfIteration(const T& t) { + std::vector res; + for (auto i : t) res.push_back(static_cast(i)); + return res; } // These IterationOrderChanges tests depend on non-deterministic behavior. @@ -2263,15 +2330,15 @@ std::vector OrderOfIteration(const IntTable& t) { // we are touching different memory pages to cause the ordering to change. // We also need to keep the old tables around to avoid getting the same memory // blocks over and over. -TEST(Table, IterationOrderChangesByInstance) { +TYPED_TEST_P(SooTest, IterationOrderChangesByInstance) { for (size_t size : {2, 6, 12, 20}) { - const auto reference_table = MakeSimpleTable(size); + const auto reference_table = MakeSimpleTable(size); const auto reference = OrderOfIteration(reference_table); - std::vector tables; + std::vector tables; bool found_difference = false; for (int i = 0; !found_difference && i < 5000; ++i) { - tables.push_back(MakeSimpleTable(size)); + tables.push_back(MakeSimpleTable(size)); found_difference = OrderOfIteration(tables.back()) != reference; } if (!found_difference) { @@ -2282,7 +2349,7 @@ TEST(Table, IterationOrderChangesByInstance) { } } -TEST(Table, IterationOrderChangesOnRehash) { +TYPED_TEST_P(SooTest, IterationOrderChangesOnRehash) { // We test different sizes with many small numbers, because small table // resize has a different codepath. // Note: iteration order for size() <= 1 is always the same. @@ -2291,10 +2358,10 @@ TEST(Table, IterationOrderChangesOnRehash) { size_t{0}, // Force rehash is guaranteed. size * 10 // Rehash to the larger capacity is guaranteed. }) { - std::vector garbage; + std::vector garbage; bool ok = false; for (int i = 0; i < 5000; ++i) { - auto t = MakeSimpleTable(size); + auto t = MakeSimpleTable(size); const auto reference = OrderOfIteration(t); // Force rehash. t.rehash(rehash_size); @@ -2315,8 +2382,8 @@ TEST(Table, IterationOrderChangesOnRehash) { // Verify that pointers are invalidated as soon as a second element is inserted. // This prevents dependency on pointer stability on small tables. -TEST(Table, UnstablePointers) { - IntTable table; +TYPED_TEST_P(SooTest, UnstablePointers) { + TypeParam table; const auto addr = [&](int i) { return reinterpret_cast(&*table.find(i)); @@ -2335,11 +2402,11 @@ TEST(TableDeathTest, InvalidIteratorAsserts) { if (!IsAssertEnabled() && !SwisstableGenerationsEnabled()) GTEST_SKIP() << "Assertions not enabled."; - IntTable t; + NonSooIntTable t; // Extra simple "regexp" as regexp support is highly varied across platforms. EXPECT_DEATH_IF_SUPPORTED(t.erase(t.end()), "erase.* called on end.. iterator."); - typename IntTable::iterator iter; + typename NonSooIntTable::iterator iter; EXPECT_DEATH_IF_SUPPORTED( ++iter, "operator.* called on default-constructed iterator."); t.insert(0); @@ -2353,6 +2420,22 @@ TEST(TableDeathTest, InvalidIteratorAsserts) { EXPECT_DEATH_IF_SUPPORTED(++iter, kErasedDeathMessage); } +TEST(TableDeathTest, InvalidIteratorAssertsSoo) { + if (!IsAssertEnabled() && !SwisstableGenerationsEnabled()) + GTEST_SKIP() << "Assertions not enabled."; + + SooIntTable t; + // Extra simple "regexp" as regexp support is highly varied across platforms. + EXPECT_DEATH_IF_SUPPORTED(t.erase(t.end()), + "erase.* called on end.. iterator."); + typename SooIntTable::iterator iter; + EXPECT_DEATH_IF_SUPPORTED( + ++iter, "operator.* called on default-constructed iterator."); + + // We can't detect the erased iterator case as invalid in SOO mode because + // the control is static constant. +} + // Invalid iterator use can trigger use-after-free in asan/hwasan, // use-of-uninitialized-value in msan, or invalidated iterator assertions. constexpr const char* kInvalidIteratorDeathMessage = @@ -2366,11 +2449,11 @@ constexpr bool kMsvc = true; constexpr bool kMsvc = false; #endif -TEST(TableDeathTest, IteratorInvalidAssertsEqualityOperator) { +TYPED_TEST_P(SooTest, IteratorInvalidAssertsEqualityOperator) { if (!IsAssertEnabled() && !SwisstableGenerationsEnabled()) GTEST_SKIP() << "Assertions not enabled."; - IntTable t; + TypeParam t; t.insert(1); t.insert(2); t.insert(3); @@ -2389,35 +2472,50 @@ TEST(TableDeathTest, IteratorInvalidAssertsEqualityOperator) { t.erase(iter2); EXPECT_DEATH_IF_SUPPORTED(void(iter1 == iter2), kErasedDeathMessage); - IntTable t1, t2; + TypeParam t1, t2; t1.insert(0); t2.insert(0); iter1 = t1.begin(); iter2 = t2.begin(); const char* const kContainerDiffDeathMessage = SwisstableGenerationsEnabled() - ? "Invalid iterator comparison.*iterators from different hashtables" + ? "Invalid iterator comparison.*iterators from different.* hashtables" : "Invalid iterator comparison.*may be from different " ".*containers.*config=asan"; EXPECT_DEATH_IF_SUPPORTED(void(iter1 == iter2), kContainerDiffDeathMessage); EXPECT_DEATH_IF_SUPPORTED(void(iter2 == iter1), kContainerDiffDeathMessage); +} + +TYPED_TEST_P(SooTest, IteratorInvalidAssertsEqualityOperatorRehash) { + if (!IsAssertEnabled() && !SwisstableGenerationsEnabled()) + GTEST_SKIP() << "Assertions not enabled."; + if (kMsvc) GTEST_SKIP() << "MSVC doesn't support | in regex."; +#ifdef ABSL_HAVE_THREAD_SANITIZER + GTEST_SKIP() << "ThreadSanitizer test runs fail on use-after-free even in " + "EXPECT_DEATH."; +#endif - for (int i = 0; i < 10; ++i) t1.insert(i); - // There should have been a rehash in t1. - if (kMsvc) return; // MSVC doesn't support | in regex. + TypeParam t; + t.insert(0); + auto iter = t.begin(); + + // Trigger a rehash in t. + for (int i = 0; i < 10; ++i) t.insert(i); - // NOTE(b/293887834): After rehashing, iterators will contain pointers to - // freed memory, which may be detected by ThreadSanitizer. const char* const kRehashedDeathMessage = SwisstableGenerationsEnabled() ? kInvalidIteratorDeathMessage - : "Invalid iterator comparison.*might have rehashed.*config=asan" - "|ThreadSanitizer: heap-use-after-free"; - EXPECT_DEATH_IF_SUPPORTED(void(iter1 == t1.begin()), kRehashedDeathMessage); + : "Invalid iterator comparison.*might have rehashed.*config=asan"; + EXPECT_DEATH_IF_SUPPORTED(void(iter == t.begin()), kRehashedDeathMessage); } #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) -TEST(RawHashSamplerTest, Sample) { +template +class RawHashSamplerTest : public testing::Test {}; +TYPED_TEST_SUITE_P(RawHashSamplerTest); + +TYPED_TEST_P(RawHashSamplerTest, Sample) { + constexpr bool soo_enabled = std::is_same::value; // Enable the feature even if the prod default is off. SetHashtablezEnabled(true); SetHashtablezSampleParameter(100); @@ -2430,7 +2528,7 @@ TEST(RawHashSamplerTest, Sample) { ++start_size; }); - std::vector tables; + std::vector tables; for (int i = 0; i < 1000000; ++i) { tables.emplace_back(); @@ -2459,15 +2557,20 @@ TEST(RawHashSamplerTest, Sample) { std::memory_order_relaxed)]++; reservations[info.max_reserve.load(std::memory_order_relaxed)]++; } - EXPECT_EQ(info.inline_element_size, sizeof(int64_t)); + EXPECT_EQ(info.inline_element_size, sizeof(typename TypeParam::value_type)); ++end_size; }); EXPECT_NEAR((end_size - start_size) / static_cast(tables.size()), 0.01, 0.005); - EXPECT_EQ(observed_checksums.size(), 5); - for (const auto& [_, count] : observed_checksums) { - EXPECT_NEAR((100 * count) / static_cast(tables.size()), 0.2, 0.05); + if (soo_enabled) { + EXPECT_EQ(observed_checksums.size(), 9); + } else { + EXPECT_EQ(observed_checksums.size(), 5); + for (const auto& [_, count] : observed_checksums) { + EXPECT_NEAR((100 * count) / static_cast(tables.size()), 0.2, + 0.05); + } } EXPECT_EQ(reservations.size(), 10); @@ -2479,6 +2582,10 @@ TEST(RawHashSamplerTest, Sample) { << reservation; } } + +REGISTER_TYPED_TEST_SUITE_P(RawHashSamplerTest, Sample); +using RawHashSamplerTestTypes = ::testing::Types; +INSTANTIATE_TYPED_TEST_SUITE_P(My, RawHashSamplerTest, RawHashSamplerTestTypes); #endif // ABSL_INTERNAL_HASHTABLEZ_SAMPLE TEST(RawHashSamplerTest, DoNotSampleCustomAllocators) { @@ -2531,9 +2638,10 @@ using SanitizerTableTypes = ::testing::Types; INSTANTIATE_TYPED_TEST_SUITE_P(InstanceSanitizerTest, SanitizerTest, SanitizerTableTypes); +// TODO(b/289225379): poison inline space when empty SOO. TEST(Sanitizer, PoisoningOnErase) { - IntTable t; - int64_t& v = *t.insert(0).first; + NonSooIntTable t; + auto& v = *t.insert(0).first; EXPECT_FALSE(__asan_address_is_poisoned(&v)); t.erase(0); @@ -2581,7 +2689,7 @@ TEST(Iterator, InvalidUseCrashesWithSanitizers) { if (!SwisstableGenerationsEnabled()) GTEST_SKIP() << "Generations disabled."; if (kMsvc) GTEST_SKIP() << "MSVC doesn't support | in regexp."; - IntTable t; + NonSooIntTable t; // Start with 1 element so that `it` is never an end iterator. t.insert(-1); for (int i = 0; i < 10; ++i) { @@ -2628,11 +2736,11 @@ TEST(Iterator, InvalidUseWithMoveCrashesWithSanitizers) { if (!SwisstableGenerationsEnabled()) GTEST_SKIP() << "Generations disabled."; if (kMsvc) GTEST_SKIP() << "MSVC doesn't support | in regexp."; - IntTable t1, t2; + NonSooIntTable t1, t2; t1.insert(1); auto it = t1.begin(); // ptr will become invalidated on rehash. - const int64_t* ptr = &*it; + const auto* ptr = &*it; (void)ptr; t2 = std::move(t1); @@ -2640,12 +2748,12 @@ TEST(Iterator, InvalidUseWithMoveCrashesWithSanitizers) { EXPECT_DEATH_IF_SUPPORTED(void(it == t2.begin()), kInvalidIteratorDeathMessage); #ifdef ABSL_HAVE_ADDRESS_SANITIZER - EXPECT_DEATH_IF_SUPPORTED(std::cout << *ptr, "heap-use-after-free"); + EXPECT_DEATH_IF_SUPPORTED(std::cout << **ptr, "heap-use-after-free"); #endif } -TEST(Table, ReservedGrowthUpdatesWhenTableDoesntGrow) { - IntTable t; +TYPED_TEST_P(SooTest, ReservedGrowthUpdatesWhenTableDoesntGrow) { + TypeParam t; for (int i = 0; i < 8; ++i) t.insert(i); // Want to insert twice without invalidating iterators so reserve. const size_t cap = t.capacity(); @@ -2687,7 +2795,7 @@ TEST(Table, GenerationInfoResetsOnClear) { if (!SwisstableGenerationsEnabled()) GTEST_SKIP() << "Generations disabled."; if (kMsvc) GTEST_SKIP() << "MSVC doesn't support | in regexp."; - IntTable t; + NonSooIntTable t; for (int i = 0; i < 1000; ++i) t.insert(i); t.reserve(t.size() + 100); @@ -2705,24 +2813,24 @@ TEST(Table, InvalidReferenceUseCrashesWithSanitizers) { GTEST_SKIP() << "MSan fails to detect some of these rehashes."; #endif - IntTable t; + NonSooIntTable t; t.insert(0); // Rehashing is guaranteed on every insertion while capacity is less than // RehashProbabilityConstant(). - int64_t i = 0; + int i = 0; while (t.capacity() <= RehashProbabilityConstant()) { // ptr will become invalidated on rehash. - const int64_t* ptr = &*t.begin(); + const auto* ptr = &*t.begin(); t.insert(++i); - EXPECT_DEATH_IF_SUPPORTED(std::cout << *ptr, "use-after-free") << i; + EXPECT_DEATH_IF_SUPPORTED(std::cout << **ptr, "use-after-free") << i; } } TEST(Iterator, InvalidComparisonDifferentTables) { if (!SwisstableGenerationsEnabled()) GTEST_SKIP() << "Generations disabled."; - IntTable t1, t2; - IntTable::iterator default_constructed_iter; + NonSooIntTable t1, t2; + NonSooIntTable::iterator default_constructed_iter; // We randomly use one of N empty generations for generations from empty // hashtables. In general, we won't always detect when iterators from // different empty hashtables are compared, but in this test case, we @@ -2813,10 +2921,11 @@ TEST(Table, CountedHash) { } } +// IterateOverFullSlots doesn't support SOO. TEST(Table, IterateOverFullSlotsEmpty) { - IntTable t; - auto fail_if_any = [](const ctrl_t*, int64_t* i) { - FAIL() << "expected no slots " << i; + NonSooIntTable t; + auto fail_if_any = [](const ctrl_t*, auto* i) { + FAIL() << "expected no slots " << **i; }; container_internal::IterateOverFullSlots( RawHashSetTestOnlyAccess::GetCommon(t), @@ -2830,7 +2939,7 @@ TEST(Table, IterateOverFullSlotsEmpty) { } TEST(Table, IterateOverFullSlotsFull) { - IntTable t; + NonSooIntTable t; std::vector expected_slots; for (int64_t i = 0; i < 128; ++i) { @@ -2841,17 +2950,106 @@ TEST(Table, IterateOverFullSlotsFull) { container_internal::IterateOverFullSlots( RawHashSetTestOnlyAccess::GetCommon(t), RawHashSetTestOnlyAccess::GetSlots(t), - [&t, &slots](const ctrl_t* ctrl, int64_t* i) { + [&t, &slots](const ctrl_t* ctrl, auto* i) { ptrdiff_t ctrl_offset = ctrl - RawHashSetTestOnlyAccess::GetCommon(t).control(); ptrdiff_t slot_offset = i - RawHashSetTestOnlyAccess::GetSlots(t); ASSERT_EQ(ctrl_offset, slot_offset); - slots.push_back(*i); + slots.push_back(**i); }); EXPECT_THAT(slots, testing::UnorderedElementsAreArray(expected_slots)); } } +template +class SooTable : public testing::Test {}; +TYPED_TEST_SUITE_P(SooTable); + +TYPED_TEST_P(SooTable, Basic) { + bool frozen = true; + TypeParam t{FreezableAlloc(&frozen)}; + if (t.capacity() != SooCapacity()) { + EXPECT_LT(sizeof(void*), 8); + GTEST_SKIP() << "not SOO on this platform"; + } + + t.insert(0); + EXPECT_EQ(t.capacity(), 1); + auto it = t.find(0); + EXPECT_EQ(it, t.begin()); + ASSERT_NE(it, t.end()); + EXPECT_EQ(*it, 0); + EXPECT_EQ(++it, t.end()); + EXPECT_EQ(t.find(1), t.end()); + EXPECT_EQ(t.size(), 1); + + t.erase(0); + EXPECT_EQ(t.size(), 0); + t.insert(1); + it = t.find(1); + EXPECT_EQ(it, t.begin()); + ASSERT_NE(it, t.end()); + EXPECT_EQ(*it, 1); + + t.clear(); + EXPECT_EQ(t.size(), 0); +} + +REGISTER_TYPED_TEST_SUITE_P(SooTable, Basic); +using FreezableSooTableTypes = + ::testing::Types, + FreezableSizedValueSooTable<16>>; +INSTANTIATE_TYPED_TEST_SUITE_P(My, SooTable, FreezableSooTableTypes); + +TEST(Table, RehashToSoo) { + SooIntTable t; + if (t.capacity() != SooCapacity()) { + EXPECT_LT(sizeof(void*), 8); + GTEST_SKIP() << "not SOO on this platform"; + } + + t.reserve(10); + t.insert(0); + EXPECT_EQ(*t.begin(), 0); + + t.rehash(0); // Rehash back down to SOO table. + + EXPECT_EQ(t.capacity(), SooCapacity()); + EXPECT_EQ(t.size(), 1); + EXPECT_EQ(*t.begin(), 0); + EXPECT_EQ(t.find(0), t.begin()); + EXPECT_EQ(t.find(1), t.end()); +} + +TEST(Table, ResizeToNonSoo) { + for (int reserve_capacity : {8, 100000}) { + SooIntTable t; + t.insert(0); + + t.reserve(reserve_capacity); + + EXPECT_EQ(t.find(0), t.begin()); + EXPECT_EQ(t.size(), 1); + EXPECT_EQ(*t.begin(), 0); + EXPECT_EQ(t.find(1), t.end()); + } +} + +REGISTER_TYPED_TEST_SUITE_P( + SooTest, Empty, Clear, ClearBug, Contains1, Contains2, ContainsEmpty, + CopyConstruct, CopyDifferentCapacities, CopyDifferentSizes, + EnsureNonQuadraticAsInRust, Erase, EraseBeginEnd, EraseInSmallTables, + EraseMaintainsValidIterator, FindFullDeletedRegression, HintInsert, Insert1, + Insert2, InsertEraseStressTest, InsertWithinCapacity, + IterationOrderChangesByInstance, IterationOrderChangesOnRehash, + IteratorInvalidAssertsEqualityOperator, + IteratorInvalidAssertsEqualityOperatorRehash, LargeTable, LookupEmpty, + NumDeletedRegression, Rehash, RehashDoesNotRehashWhenNotNecessary, + RehashZeroForcesRehash, ReplacingDeletedSlotDoesNotRehash, + ReservedGrowthUpdatesWhenTableDoesntGrow, Swap, UnstablePointers); +using SooTableTypes = ::testing::Types; +INSTANTIATE_TYPED_TEST_SUITE_P(My, SooTest, SooTableTypes); + } // namespace } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/sample_element_size_test.cc b/absl/container/sample_element_size_test.cc index b23626b4..22470b49 100644 --- a/absl/container/sample_element_size_test.cc +++ b/absl/container/sample_element_size_test.cc @@ -12,6 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include +#include +#include + #include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/container/flat_hash_map.h" @@ -38,15 +43,16 @@ void TestInlineElementSize( // set cannot be flat_hash_set, however, since that would introduce a mutex // deadlock. std::unordered_set& preexisting_info, // NOLINT - std::vector& tables, const typename Table::value_type& elt, + std::vector
& tables, + const std::vector& values, size_t expected_element_size) { for (int i = 0; i < 10; ++i) { // We create a new table and must store it somewhere so that when we store // a pointer to the resulting `HashtablezInfo` into `preexisting_info` // that we aren't storing a dangling pointer. tables.emplace_back(); - // We must insert an element to get a hashtablez to instantiate. - tables.back().insert(elt); + // We must insert elements to get a hashtablez to instantiate. + tables.back().insert(values.begin(), values.end()); } size_t new_count = 0; sampler.Iterate([&](const HashtablezInfo& info) { @@ -82,6 +88,9 @@ TEST(FlatHashMap, SampleElementSize) { std::vector> flat_set_tables; std::vector> node_map_tables; std::vector> node_set_tables; + std::vector set_values = {bigstruct{{0}}, bigstruct{{1}}}; + std::vector> map_values = {{0, bigstruct{}}, + {1, bigstruct{}}}; // It takes thousands of new tables after changing the sampling parameters // before you actually get some instrumentation. And if you must actually @@ -97,14 +106,14 @@ TEST(FlatHashMap, SampleElementSize) { std::unordered_set preexisting_info; // NOLINT sampler.Iterate( [&](const HashtablezInfo& info) { preexisting_info.insert(&info); }); - TestInlineElementSize(sampler, preexisting_info, flat_map_tables, - {0, bigstruct{}}, sizeof(int) + sizeof(bigstruct)); - TestInlineElementSize(sampler, preexisting_info, node_map_tables, - {0, bigstruct{}}, sizeof(void*)); - TestInlineElementSize(sampler, preexisting_info, flat_set_tables, // - bigstruct{}, sizeof(bigstruct)); - TestInlineElementSize(sampler, preexisting_info, node_set_tables, // - bigstruct{}, sizeof(void*)); + TestInlineElementSize(sampler, preexisting_info, flat_map_tables, map_values, + sizeof(int) + sizeof(bigstruct)); + TestInlineElementSize(sampler, preexisting_info, node_map_tables, map_values, + sizeof(void*)); + TestInlineElementSize(sampler, preexisting_info, flat_set_tables, set_values, + sizeof(bigstruct)); + TestInlineElementSize(sampler, preexisting_info, node_set_tables, set_values, + sizeof(void*)); #endif } -- cgit v1.2.3 From 6f0bb2747d0a910de4a958eeeab2b9d615156382 Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Wed, 6 Mar 2024 12:32:04 -0800 Subject: Add ABSL_ATTRIBUTE_UNUSED to variables used in an ABSL_ASSUME. PiperOrigin-RevId: 613305668 Change-Id: Ifc247f48ea476745eaaf0dd41dbdab8404a6cafb --- absl/container/internal/raw_hash_set.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 0d5a5344..d4418339 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -2672,8 +2672,8 @@ class raw_hash_set { const size_t cap = common().capacity(); // Use local variables because compiler complains about using functions in // assume. - static constexpr bool kSooEnabled = SooEnabled(); - static constexpr size_t kSooCapacity = SooCapacity(); + ABSL_ATTRIBUTE_UNUSED static constexpr bool kSooEnabled = SooEnabled(); + ABSL_ATTRIBUTE_UNUSED static constexpr size_t kSooCapacity = SooCapacity(); ABSL_ASSUME(!kSooEnabled || cap >= kSooCapacity); return cap; } -- cgit v1.2.3 From d03f54ef130a3070965618eae4e0e8f97cdd4ca6 Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Thu, 7 Mar 2024 08:18:47 -0800 Subject: Avoid MSan: use-of-uninitialized-value error in find_non_soo. PiperOrigin-RevId: 613590317 Change-Id: I69f095681102e5492916085ada0eed085a75765b --- absl/container/internal/raw_hash_set.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index d4418339..81f99366 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -3295,14 +3295,13 @@ class raw_hash_set { iterator find_non_soo(const key_arg& key, size_t hash) { assert(!is_soo()); auto seq = probe(common(), hash); - slot_type* slot_ptr = slot_array(); const ctrl_t* ctrl = control(); while (true) { Group g{ctrl + seq.offset()}; for (uint32_t i : g.Match(H2(hash))) { if (ABSL_PREDICT_TRUE(PolicyTraits::apply( EqualElement{key, eq_ref()}, - PolicyTraits::element(slot_ptr + seq.offset(i))))) + PolicyTraits::element(slot_array() + seq.offset(i))))) return iterator_at(seq.offset(i)); } if (ABSL_PREDICT_TRUE(g.MaskEmpty())) return end(); -- cgit v1.2.3 From d802708117c6ef6b9783efe499b2a2d0d0536c77 Mon Sep 17 00:00:00 2001 From: Derek Mauro Date: Mon, 11 Mar 2024 09:14:45 -0700 Subject: Replace usages of absl::move, absl::forward, and absl::exchange with their std:: equivalents PiperOrigin-RevId: 614687225 Change-Id: I07421db08ee9c221e561f42e3bf8345fb5321401 --- absl/cleanup/cleanup_test.cc | 2 +- absl/container/internal/btree.h | 4 +- absl/container/internal/compressed_tuple.h | 19 ++-- absl/container/internal/compressed_tuple_test.cc | 5 +- absl/container/internal/layout.h | 2 +- absl/functional/bind_front.h | 5 +- absl/functional/internal/front_binder.h | 18 ++-- absl/types/internal/optional.h | 4 +- absl/types/internal/variant.h | 109 ++++++++++++----------- absl/types/optional.h | 33 ++++--- absl/types/variant.h | 43 ++++----- absl/types/variant_test.cc | 76 ++++++++-------- 12 files changed, 157 insertions(+), 163 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/cleanup/cleanup_test.cc b/absl/cleanup/cleanup_test.cc index 46b88589..72d7ff2a 100644 --- a/absl/cleanup/cleanup_test.cc +++ b/absl/cleanup/cleanup_test.cc @@ -48,7 +48,7 @@ class FunctorClass { explicit FunctorClass(Callback callback) : callback_(std::move(callback)) {} FunctorClass(FunctorClass&& other) - : callback_(absl::exchange(other.callback_, Callback())) {} + : callback_(std::exchange(other.callback_, Callback())) {} FunctorClass(const FunctorClass&) = delete; diff --git a/absl/container/internal/btree.h b/absl/container/internal/btree.h index 91df57a3..fd7860da 100644 --- a/absl/container/internal/btree.h +++ b/absl/container/internal/btree.h @@ -1407,9 +1407,9 @@ class btree { copy_or_move_values_in_order(other); } btree(btree &&other) noexcept - : root_(absl::exchange(other.root_, EmptyNode())), + : root_(std::exchange(other.root_, EmptyNode())), rightmost_(std::move(other.rightmost_)), - size_(absl::exchange(other.size_, 0u)) { + size_(std::exchange(other.size_, 0u)) { other.mutable_rightmost() = EmptyNode(); } btree(btree &&other, const allocator_type &alloc) diff --git a/absl/container/internal/compressed_tuple.h b/absl/container/internal/compressed_tuple.h index 59e70eb2..f05a1fdc 100644 --- a/absl/container/internal/compressed_tuple.h +++ b/absl/container/internal/compressed_tuple.h @@ -87,10 +87,10 @@ struct Storage { constexpr Storage() = default; template explicit constexpr Storage(absl::in_place_t, V&& v) - : value(absl::forward(v)) {} + : value(std::forward(v)) {} constexpr const T& get() const& { return value; } T& get() & { return value; } - constexpr const T&& get() const&& { return absl::move(*this).value; } + constexpr const T&& get() const&& { return std::move(*this).value; } T&& get() && { return std::move(*this).value; } }; @@ -99,12 +99,11 @@ struct ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC Storage : T { constexpr Storage() = default; template - explicit constexpr Storage(absl::in_place_t, V&& v) - : T(absl::forward(v)) {} + explicit constexpr Storage(absl::in_place_t, V&& v) : T(std::forward(v)) {} constexpr const T& get() const& { return *this; } T& get() & { return *this; } - constexpr const T&& get() const&& { return absl::move(*this); } + constexpr const T&& get() const&& { return std::move(*this); } T&& get() && { return std::move(*this); } }; @@ -123,7 +122,7 @@ struct ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTupleImpl< constexpr CompressedTupleImpl() = default; template explicit constexpr CompressedTupleImpl(absl::in_place_t, Vs&&... args) - : Storage(absl::in_place, absl::forward(args))... {} + : Storage(absl::in_place, std::forward(args))... {} friend CompressedTuple; }; @@ -135,7 +134,7 @@ struct ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTupleImpl< constexpr CompressedTupleImpl() = default; template explicit constexpr CompressedTupleImpl(absl::in_place_t, Vs&&... args) - : Storage(absl::in_place, absl::forward(args))... {} + : Storage(absl::in_place, std::forward(args))... {} friend CompressedTuple; }; @@ -234,8 +233,8 @@ class ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple bool> = true> explicit constexpr CompressedTuple(First&& first, Vs&&... base) : CompressedTuple::CompressedTupleImpl(absl::in_place, - absl::forward(first), - absl::forward(base)...) {} + std::forward(first), + std::forward(base)...) {} template ElemT& get() & { @@ -254,7 +253,7 @@ class ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple template constexpr const ElemT&& get() const&& { - return absl::move(*this).StorageT::get(); + return std::move(*this).StorageT::get(); } }; diff --git a/absl/container/internal/compressed_tuple_test.cc b/absl/container/internal/compressed_tuple_test.cc index da07baab..49818fb8 100644 --- a/absl/container/internal/compressed_tuple_test.cc +++ b/absl/container/internal/compressed_tuple_test.cc @@ -16,6 +16,7 @@ #include #include +#include #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -384,8 +385,8 @@ TEST(CompressedTupleTest, Constexpr) { #if defined(__clang__) // An apparent bug in earlier versions of gcc claims these are ambiguous. - constexpr int x2m = absl::move(x.get<2>()).get<0>(); - constexpr CallType x3m = absl::move(x).get<3>().value(); + constexpr int x2m = std::move(x.get<2>()).get<0>(); + constexpr CallType x3m = std::move(x).get<3>().value(); EXPECT_EQ(x2m, 5); EXPECT_EQ(x3m, CallType::kConstMove); #endif diff --git a/absl/container/internal/layout.h b/absl/container/internal/layout.h index a4ba6101..1bf739cc 100644 --- a/absl/container/internal/layout.h +++ b/absl/container/internal/layout.h @@ -706,7 +706,7 @@ class Layout : public internal_layout::LayoutType { template static constexpr PartialType Partial(Sizes&&... sizes) { static_assert(sizeof...(Sizes) <= sizeof...(Ts), ""); - return PartialType(absl::forward(sizes)...); + return PartialType(std::forward(sizes)...); } // Creates a layout with the sizes of all arrays specified. If you know diff --git a/absl/functional/bind_front.h b/absl/functional/bind_front.h index a956eb02..885f24b8 100644 --- a/absl/functional/bind_front.h +++ b/absl/functional/bind_front.h @@ -34,6 +34,8 @@ #include // For std::bind_front. #endif // defined(__cpp_lib_bind_front) && __cpp_lib_bind_front >= 201907L +#include + #include "absl/functional/internal/front_binder.h" #include "absl/utility/utility.h" @@ -182,8 +184,7 @@ template constexpr functional_internal::bind_front_t bind_front( F&& func, BoundArgs&&... args) { return functional_internal::bind_front_t( - absl::in_place, absl::forward(func), - absl::forward(args)...); + absl::in_place, std::forward(func), std::forward(args)...); } #endif // defined(__cpp_lib_bind_front) && __cpp_lib_bind_front >= 201907L diff --git a/absl/functional/internal/front_binder.h b/absl/functional/internal/front_binder.h index 45f52de7..44a54928 100644 --- a/absl/functional/internal/front_binder.h +++ b/absl/functional/internal/front_binder.h @@ -34,8 +34,8 @@ namespace functional_internal { template R Apply(Tuple&& bound, absl::index_sequence, Args&&... free) { return base_internal::invoke( - absl::forward(bound).template get()..., - absl::forward(free)...); + std::forward(bound).template get()..., + std::forward(free)...); } template @@ -48,13 +48,13 @@ class FrontBinder { public: template constexpr explicit FrontBinder(absl::in_place_t, Ts&&... ts) - : bound_args_(absl::forward(ts)...) {} + : bound_args_(std::forward(ts)...) {} template > R operator()(FreeArgs&&... free_args) & { return functional_internal::Apply(bound_args_, Idx(), - absl::forward(free_args)...); + std::forward(free_args)...); } template > R operator()(FreeArgs&&... free_args) const& { return functional_internal::Apply(bound_args_, Idx(), - absl::forward(free_args)...); + std::forward(free_args)...); } template (absl::move(bound_args_), Idx(), - absl::forward(free_args)...); + return functional_internal::Apply(std::move(bound_args_), Idx(), + std::forward(free_args)...); } template (absl::move(bound_args_), Idx(), - absl::forward(free_args)...); + return functional_internal::Apply(std::move(bound_args_), Idx(), + std::forward(free_args)...); } }; diff --git a/absl/types/internal/optional.h b/absl/types/internal/optional.h index a96d260a..5731a5bc 100644 --- a/absl/types/internal/optional.h +++ b/absl/types/internal/optional.h @@ -81,7 +81,7 @@ class optional_data_dtor_base { template constexpr explicit optional_data_dtor_base(in_place_t, Args&&... args) - : engaged_(true), data_(absl::forward(args)...) {} + : engaged_(true), data_(std::forward(args)...) {} ~optional_data_dtor_base() { destruct(); } }; @@ -110,7 +110,7 @@ class optional_data_dtor_base { template constexpr explicit optional_data_dtor_base(in_place_t, Args&&... args) - : engaged_(true), data_(absl::forward(args)...) {} + : engaged_(true), data_(std::forward(args)...) {} }; template diff --git a/absl/types/internal/variant.h b/absl/types/internal/variant.h index 263d7b09..40f57c40 100644 --- a/absl/types/internal/variant.h +++ b/absl/types/internal/variant.h @@ -26,6 +26,7 @@ #include #include #include +#include #include "absl/base/config.h" #include "absl/base/internal/identity.h" @@ -214,7 +215,7 @@ constexpr ReturnType call_with_indices(FunctionObject&& function) { std::is_same()( SizeT()...))>::value, "Not all visitation overloads have the same return type."); - return absl::forward(function)(SizeT()...); + return std::forward(function)(SizeT()...); } template @@ -284,7 +285,7 @@ struct UnreachableSwitchCase { assert(false); // NOLINT // Hack to silence potential no return warning -- cause an infinite loop. - return Run(absl::forward(op)); + return Run(std::forward(op)); #endif // Checks for __builtin_unreachable } }; @@ -292,7 +293,7 @@ struct UnreachableSwitchCase { template struct ReachableSwitchCase { static VisitIndicesResultT Run(Op&& op) { - return absl::base_internal::invoke(absl::forward(op), SizeT()); + return absl::base_internal::invoke(std::forward(op), SizeT()); } }; @@ -357,74 +358,74 @@ struct VisitIndicesSwitch { static VisitIndicesResultT Run(Op&& op, std::size_t i) { switch (i) { case 0: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 1: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 2: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 3: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 4: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 5: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 6: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 7: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 8: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 9: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 10: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 11: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 12: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 13: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 14: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 15: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 16: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 17: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 18: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 19: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 20: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 21: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 22: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 23: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 24: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 25: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 26: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 27: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 28: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 29: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 30: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 31: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); case 32: - return PickCase::Run(absl::forward(op)); + return PickCase::Run(std::forward(op)); default: ABSL_ASSERT(i == variant_npos); - return absl::base_internal::invoke(absl::forward(op), NPos()); + return absl::base_internal::invoke(std::forward(op), NPos()); } } }; @@ -437,7 +438,7 @@ struct VisitIndicesFallback { MakeVisitationMatrix, Op, index_sequence<(EndIndices + 1)...>, index_sequence<>>::Run(), - (indices + 1)...)(absl::forward(op)); + (indices + 1)...)(std::forward(op)); } }; @@ -489,7 +490,7 @@ struct VisitIndicesVariadicImpl, EndIndices...> { VisitIndicesResultT operator()( SizeT /*index*/) && { return base_internal::invoke( - absl::forward(op), + std::forward(op), SizeT::value - std::size_t{1}>()...); } @@ -501,7 +502,7 @@ struct VisitIndicesVariadicImpl, EndIndices...> { static VisitIndicesResultT Run(Op&& op, SizeType... i) { return VisitIndicesSwitch::value>::Run( - FlattenedOp{absl::forward(op)}, + FlattenedOp{std::forward(op)}, FlattenIndices<(EndIndices + std::size_t{1})...>::Run( (i + std::size_t{1})...)); } @@ -612,7 +613,7 @@ struct VariantCoreAccess { TypedThrowBadVariantAccess>(); } - return Access(absl::forward(self)); + return Access(std::forward(self)); } // The implementation of the move-assignment operation for a variant. @@ -684,7 +685,7 @@ struct VariantCoreAccess { void operator()(SizeT /*old_i*/ ) const { - Access(*left) = absl::forward(other); + Access(*left) = std::forward(other); } template @@ -695,13 +696,13 @@ struct VariantCoreAccess { if (std::is_nothrow_constructible::value || !std::is_nothrow_move_constructible::value) { left->template emplace( - absl::forward(other)); + std::forward(other)); } else { // the standard says "equivalent to // operator=(variant(std::forward(t)))", but we use `emplace` here // because the variant's move assignment operator could be deleted. left->template emplace( - New(absl::forward(other))); + New(std::forward(other))); } } @@ -712,7 +713,7 @@ struct VariantCoreAccess { template static ConversionAssignVisitor MakeConversionAssignVisitor(Left* left, QualifiedNew&& qual) { - return {left, absl::forward(qual)}; + return {left, std::forward(qual)}; } // Backend for operations for `emplace()` which destructs `*self` then @@ -723,7 +724,7 @@ struct VariantCoreAccess { Destroy(*self); using New = typename absl::variant_alternative::type; New* const result = ::new (static_cast(&self->state_)) - New(absl::forward(args)...); + New(std::forward(args)...); self->index_ = NewIndex; return *result; } @@ -919,9 +920,9 @@ struct PerformVisitation { Is, QualifiedVariants>...)>>::value, "All visitation overloads must have the same return type."); return absl::base_internal::invoke( - absl::forward(op), + std::forward(op), VariantCoreAccess::Access( - absl::forward(std::get(variant_tup)))...); + std::forward(std::get(variant_tup)))...); } template @@ -969,11 +970,11 @@ union Union { template explicit constexpr Union(EmplaceTag<0>, P&&... args) - : head(absl::forward

(args)...) {} + : head(std::forward

(args)...) {} template explicit constexpr Union(EmplaceTag, P&&... args) - : tail(EmplaceTag{}, absl::forward

(args)...) {} + : tail(EmplaceTag{}, std::forward

(args)...) {} Head head; TailUnion tail; @@ -1001,11 +1002,11 @@ union DestructibleUnionImpl { template explicit constexpr DestructibleUnionImpl(EmplaceTag<0>, P&&... args) - : head(absl::forward

(args)...) {} + : head(std::forward

(args)...) {} template explicit constexpr DestructibleUnionImpl(EmplaceTag, P&&... args) - : tail(EmplaceTag{}, absl::forward

(args)...) {} + : tail(EmplaceTag{}, std::forward

(args)...) {} ~DestructibleUnionImpl() {} @@ -1036,7 +1037,7 @@ class VariantStateBase { template explicit constexpr VariantStateBase(EmplaceTag tag, P&&... args) - : state_(tag, absl::forward

(args)...), index_(I) {} + : state_(tag, std::forward

(args)...), index_(I) {} explicit constexpr VariantStateBase(NoopConstructorTag) : state_(NoopConstructorTag()), index_(variant_npos) {} @@ -1321,7 +1322,7 @@ class VariantMoveBaseNontrivial : protected VariantStateBaseDestructor { using Alternative = typename absl::variant_alternative>::type; ::new (static_cast(&self->state_)) Alternative( - variant_internal::AccessUnion(absl::move(other->state_), i)); + variant_internal::AccessUnion(std::move(other->state_), i)); } void operator()(SizeT /*i*/) const {} diff --git a/absl/types/optional.h b/absl/types/optional.h index 395fe62f..cf7249cb 100644 --- a/absl/types/optional.h +++ b/absl/types/optional.h @@ -151,7 +151,7 @@ class optional : private optional_internal::optional_data, std::is_same, std::is_constructible >::value>* = nullptr> constexpr explicit optional(InPlaceT, Args&&... args) - : data_base(in_place_t(), absl::forward(args)...) {} + : data_base(in_place_t(), std::forward(args)...) {} // Constructs a non-empty `optional` direct-initialized value of type `T` from // the arguments of an initializer_list and `std::forward(args)...`. @@ -162,8 +162,7 @@ class optional : private optional_internal::optional_data, T, std::initializer_list&, Args&&...>::value>::type> constexpr explicit optional(in_place_t, std::initializer_list il, Args&&... args) - : data_base(in_place_t(), il, absl::forward(args)...) { - } + : data_base(in_place_t(), il, std::forward(args)...) {} // Value constructor (implicit) template < @@ -176,21 +175,21 @@ class optional : private optional_internal::optional_data, std::is_convertible, std::is_constructible >::value, bool>::type = false> - constexpr optional(U&& v) : data_base(in_place_t(), absl::forward(v)) {} + constexpr optional(U&& v) : data_base(in_place_t(), std::forward(v)) {} // Value constructor (explicit) template < typename U = T, typename std::enable_if< absl::conjunction::type>>, + in_place_t, typename std::decay::type> >, absl::negation, typename std::decay::type>>, - absl::negation>, - std::is_constructible>::value, + optional, typename std::decay::type> >, + absl::negation >, + std::is_constructible >::value, bool>::type = false> explicit constexpr optional(U&& v) - : data_base(in_place_t(), absl::forward(v)) {} + : data_base(in_place_t(), std::forward(v)) {} // Converting copy constructor (implicit) template , return reference(); } constexpr const T&& operator*() const&& ABSL_ATTRIBUTE_LIFETIME_BOUND { - return ABSL_HARDENING_ASSERT(this->engaged_), absl::move(reference()); + return ABSL_HARDENING_ASSERT(this->engaged_), std::move(reference()); } T&& operator*() && ABSL_ATTRIBUTE_LIFETIME_BOUND { ABSL_HARDENING_ASSERT(this->engaged_); @@ -492,7 +491,7 @@ class optional : private optional_internal::optional_data, } constexpr const T&& value() const&& ABSL_ATTRIBUTE_LIFETIME_BOUND { // NOLINT(build/c++11) - return absl::move( + return std::move( static_cast(*this) ? reference() : (optional_internal::throw_bad_optional_access(), reference())); @@ -511,9 +510,8 @@ class optional : private optional_internal::optional_data, "optional::value_or: T must be copy constructible"); static_assert(std::is_convertible::value, "optional::value_or: U must be convertible to T"); - return static_cast(*this) - ? **this - : static_cast(absl::forward(v)); + return static_cast(*this) ? **this + : static_cast(std::forward(v)); } template T value_or(U&& v) && { // NOLINT(build/c++11) @@ -573,19 +571,18 @@ void swap(optional& a, optional& b) noexcept(noexcept(a.swap(b))) { // static_assert(opt.value() == 1, ""); template constexpr optional::type> make_optional(T&& v) { - return optional::type>(absl::forward(v)); + return optional::type>(std::forward(v)); } template constexpr optional make_optional(Args&&... args) { - return optional(in_place_t(), absl::forward(args)...); + return optional(in_place_t(), std::forward(args)...); } template constexpr optional make_optional(std::initializer_list il, Args&&... args) { - return optional(in_place_t(), il, - absl::forward(args)...); + return optional(in_place_t(), il, std::forward(args)...); } // Relational operators [optional.relops] diff --git a/absl/types/variant.h b/absl/types/variant.h index ac93464b..56a7e05e 100644 --- a/absl/types/variant.h +++ b/absl/types/variant.h @@ -303,11 +303,10 @@ constexpr T& get(variant& v) { // NOLINT } // Overload for getting a variant's rvalue by type. -// Note: `absl::move()` is required to allow use of constexpr in C++11. template constexpr T&& get(variant&& v) { return variant_internal::VariantCoreAccess::CheckedAccess< - variant_internal::IndexOf::value>(absl::move(v)); + variant_internal::IndexOf::value>(std::move(v)); } // Overload for getting a variant's const lvalue by type. @@ -318,11 +317,10 @@ constexpr const T& get(const variant& v) { } // Overload for getting a variant's const rvalue by type. -// Note: `absl::move()` is required to allow use of constexpr in C++11. template constexpr const T&& get(const variant&& v) { return variant_internal::VariantCoreAccess::CheckedAccess< - variant_internal::IndexOf::value>(absl::move(v)); + variant_internal::IndexOf::value>(std::move(v)); } // Overload for getting a variant's lvalue by index. @@ -333,11 +331,10 @@ constexpr variant_alternative_t>& get( } // Overload for getting a variant's rvalue by index. -// Note: `absl::move()` is required to allow use of constexpr in C++11. template constexpr variant_alternative_t>&& get( variant&& v) { - return variant_internal::VariantCoreAccess::CheckedAccess(absl::move(v)); + return variant_internal::VariantCoreAccess::CheckedAccess(std::move(v)); } // Overload for getting a variant's const lvalue by index. @@ -348,11 +345,10 @@ constexpr const variant_alternative_t>& get( } // Overload for getting a variant's const rvalue by index. -// Note: `absl::move()` is required to allow use of constexpr in C++11. template constexpr const variant_alternative_t>&& get( const variant&& v) { - return variant_internal::VariantCoreAccess::CheckedAccess(absl::move(v)); + return variant_internal::VariantCoreAccess::CheckedAccess(std::move(v)); } // get_if() @@ -432,8 +428,8 @@ variant_internal::VisitResult visit(Visitor&& vis, return variant_internal:: VisitIndices >::value...>::Run( variant_internal::PerformVisitation{ - std::forward_as_tuple(absl::forward(vars)...), - absl::forward(vis)}, + std::forward_as_tuple(std::forward(vars)...), + std::forward(vis)}, vars.index()...); } @@ -504,13 +500,12 @@ class variant : private variant_internal::VariantBase { class T, std::size_t I = std::enable_if< variant_internal::IsNeitherSelfNorInPlace>::value, - variant_internal::IndexOfConstructedType>::type::value, + absl::decay_t >::value, + variant_internal::IndexOfConstructedType >::type::value, class Tj = absl::variant_alternative_t, - absl::enable_if_t::value>* = - nullptr> + absl::enable_if_t::value>* = nullptr> constexpr variant(T&& t) noexcept(std::is_nothrow_constructible::value) - : Base(variant_internal::EmplaceTag(), absl::forward(t)) {} + : Base(variant_internal::EmplaceTag(), std::forward(t)) {} // Constructs a variant of an alternative type from the arguments through // direct-initialization. @@ -524,7 +519,7 @@ class variant : private variant_internal::VariantBase { constexpr explicit variant(in_place_type_t, Args&&... args) : Base(variant_internal::EmplaceTag< variant_internal::UnambiguousIndexOf::value>(), - absl::forward(args)...) {} + std::forward(args)...) {} // Constructs a variant of an alternative type from an initializer list // and other arguments through direct-initialization. @@ -539,7 +534,7 @@ class variant : private variant_internal::VariantBase { Args&&... args) : Base(variant_internal::EmplaceTag< variant_internal::UnambiguousIndexOf::value>(), - il, absl::forward(args)...) {} + il, std::forward(args)...) {} // Constructs a variant of an alternative type from a provided index, // through value-initialization using the provided forwarded arguments. @@ -548,7 +543,7 @@ class variant : private variant_internal::VariantBase { variant_internal::VariantAlternativeSfinaeT, Args...>::value>::type* = nullptr> constexpr explicit variant(in_place_index_t, Args&&... args) - : Base(variant_internal::EmplaceTag(), absl::forward(args)...) {} + : Base(variant_internal::EmplaceTag(), std::forward(args)...) {} // Constructs a variant of an alternative type from a provided index, // through value-initialization of an initializer list and the provided @@ -560,7 +555,7 @@ class variant : private variant_internal::VariantBase { constexpr explicit variant(in_place_index_t, std::initializer_list il, Args&&... args) : Base(variant_internal::EmplaceTag(), il, - absl::forward(args)...) {} + std::forward(args)...) {} // Destructors @@ -595,7 +590,7 @@ class variant : private variant_internal::VariantBase { std::is_nothrow_constructible::value) { variant_internal::VisitIndices::Run( variant_internal::VariantCoreAccess::MakeConversionAssignVisitor( - this, absl::forward(t)), + this, std::forward(t)), index()); return *this; @@ -623,7 +618,7 @@ class variant : private variant_internal::VariantBase { T& emplace(Args&&... args) { return variant_internal::VariantCoreAccess::Replace< variant_internal::UnambiguousIndexOf::value>( - this, absl::forward(args)...); + this, std::forward(args)...); } // Constructs a value of the given alternative type T within the variant using @@ -644,7 +639,7 @@ class variant : private variant_internal::VariantBase { T& emplace(std::initializer_list il, Args&&... args) { return variant_internal::VariantCoreAccess::Replace< variant_internal::UnambiguousIndexOf::value>( - this, il, absl::forward(args)...); + this, il, std::forward(args)...); } // Destroys the current value of the variant (provided that @@ -663,7 +658,7 @@ class variant : private variant_internal::VariantBase { Args...>::value>::type* = nullptr> absl::variant_alternative_t& emplace(Args&&... args) { return variant_internal::VariantCoreAccess::Replace( - this, absl::forward(args)...); + this, std::forward(args)...); } // Destroys the current value of the variant (provided that @@ -681,7 +676,7 @@ class variant : private variant_internal::VariantBase { absl::variant_alternative_t& emplace(std::initializer_list il, Args&&... args) { return variant_internal::VariantCoreAccess::Replace( - this, il, absl::forward(args)...); + this, il, std::forward(args)...); } // variant::valueless_by_exception() diff --git a/absl/types/variant_test.cc b/absl/types/variant_test.cc index 4cd5b7a3..91b142ca 100644 --- a/absl/types/variant_test.cc +++ b/absl/types/variant_test.cc @@ -389,7 +389,7 @@ TEST(VariantTest, TestMoveConstruct) { using V = variant, MoveOnly, MoveOnly>; V v(in_place_index<1>, 10); - V v2 = absl::move(v); + V v2 = std::move(v); EXPECT_EQ(10, absl::get<1>(v2).value); } @@ -1209,60 +1209,60 @@ TEST(VariantTest, GetIndex) { Var v(absl::in_place_index<0>, 0); using LValueGetType = decltype(absl::get<0>(v)); - using RValueGetType = decltype(absl::get<0>(absl::move(v))); + using RValueGetType = decltype(absl::get<0>(std::move(v))); EXPECT_TRUE((std::is_same::value)); EXPECT_TRUE((std::is_same::value)); EXPECT_EQ(absl::get<0>(v), 0); - EXPECT_EQ(absl::get<0>(absl::move(v)), 0); + EXPECT_EQ(absl::get<0>(std::move(v)), 0); const Var& const_v = v; using ConstLValueGetType = decltype(absl::get<0>(const_v)); - using ConstRValueGetType = decltype(absl::get<0>(absl::move(const_v))); + using ConstRValueGetType = decltype(absl::get<0>(std::move(const_v))); EXPECT_TRUE((std::is_same::value)); EXPECT_TRUE((std::is_same::value)); EXPECT_EQ(absl::get<0>(const_v), 0); - EXPECT_EQ(absl::get<0>(absl::move(const_v)), 0); + EXPECT_EQ(absl::get<0>(std::move(const_v)), 0); } { Var v = std::string("Hello"); using LValueGetType = decltype(absl::get<1>(v)); - using RValueGetType = decltype(absl::get<1>(absl::move(v))); + using RValueGetType = decltype(absl::get<1>(std::move(v))); EXPECT_TRUE((std::is_same::value)); EXPECT_TRUE((std::is_same::value)); EXPECT_EQ(absl::get<1>(v), "Hello"); - EXPECT_EQ(absl::get<1>(absl::move(v)), "Hello"); + EXPECT_EQ(absl::get<1>(std::move(v)), "Hello"); const Var& const_v = v; using ConstLValueGetType = decltype(absl::get<1>(const_v)); - using ConstRValueGetType = decltype(absl::get<1>(absl::move(const_v))); + using ConstRValueGetType = decltype(absl::get<1>(std::move(const_v))); EXPECT_TRUE((std::is_same::value)); EXPECT_TRUE((std::is_same::value)); EXPECT_EQ(absl::get<1>(const_v), "Hello"); - EXPECT_EQ(absl::get<1>(absl::move(const_v)), "Hello"); + EXPECT_EQ(absl::get<1>(std::move(const_v)), "Hello"); } { Var v = 2.0; using LValueGetType = decltype(absl::get<2>(v)); - using RValueGetType = decltype(absl::get<2>(absl::move(v))); + using RValueGetType = decltype(absl::get<2>(std::move(v))); EXPECT_TRUE((std::is_same::value)); EXPECT_TRUE((std::is_same::value)); EXPECT_EQ(absl::get<2>(v), 2.); - EXPECT_EQ(absl::get<2>(absl::move(v)), 2.); + EXPECT_EQ(absl::get<2>(std::move(v)), 2.); const Var& const_v = v; using ConstLValueGetType = decltype(absl::get<2>(const_v)); - using ConstRValueGetType = decltype(absl::get<2>(absl::move(const_v))); + using ConstRValueGetType = decltype(absl::get<2>(std::move(const_v))); EXPECT_TRUE((std::is_same::value)); EXPECT_TRUE((std::is_same::value)); EXPECT_EQ(absl::get<2>(const_v), 2.); - EXPECT_EQ(absl::get<2>(absl::move(const_v)), 2.); + EXPECT_EQ(absl::get<2>(std::move(const_v)), 2.); } { @@ -1270,20 +1270,20 @@ TEST(VariantTest, GetIndex) { v.emplace<3>(1); using LValueGetType = decltype(absl::get<3>(v)); - using RValueGetType = decltype(absl::get<3>(absl::move(v))); + using RValueGetType = decltype(absl::get<3>(std::move(v))); EXPECT_TRUE((std::is_same::value)); EXPECT_TRUE((std::is_same::value)); EXPECT_EQ(absl::get<3>(v), 1); - EXPECT_EQ(absl::get<3>(absl::move(v)), 1); + EXPECT_EQ(absl::get<3>(std::move(v)), 1); const Var& const_v = v; using ConstLValueGetType = decltype(absl::get<3>(const_v)); - using ConstRValueGetType = decltype(absl::get<3>(absl::move(const_v))); + using ConstRValueGetType = decltype(absl::get<3>(std::move(const_v))); EXPECT_TRUE((std::is_same::value)); EXPECT_TRUE((std::is_same::value)); EXPECT_EQ(absl::get<3>(const_v), 1); - EXPECT_EQ(absl::get<3>(absl::move(const_v)), 1); // NOLINT + EXPECT_EQ(absl::get<3>(std::move(const_v)), 1); // NOLINT } } @@ -1322,60 +1322,60 @@ TEST(VariantTest, GetType) { Var v = 1; using LValueGetType = decltype(absl::get(v)); - using RValueGetType = decltype(absl::get(absl::move(v))); + using RValueGetType = decltype(absl::get(std::move(v))); EXPECT_TRUE((std::is_same::value)); EXPECT_TRUE((std::is_same::value)); EXPECT_EQ(absl::get(v), 1); - EXPECT_EQ(absl::get(absl::move(v)), 1); + EXPECT_EQ(absl::get(std::move(v)), 1); const Var& const_v = v; using ConstLValueGetType = decltype(absl::get(const_v)); - using ConstRValueGetType = decltype(absl::get(absl::move(const_v))); + using ConstRValueGetType = decltype(absl::get(std::move(const_v))); EXPECT_TRUE((std::is_same::value)); EXPECT_TRUE((std::is_same::value)); EXPECT_EQ(absl::get(const_v), 1); - EXPECT_EQ(absl::get(absl::move(const_v)), 1); + EXPECT_EQ(absl::get(std::move(const_v)), 1); } { Var v = std::string("Hello"); using LValueGetType = decltype(absl::get<1>(v)); - using RValueGetType = decltype(absl::get<1>(absl::move(v))); + using RValueGetType = decltype(absl::get<1>(std::move(v))); EXPECT_TRUE((std::is_same::value)); EXPECT_TRUE((std::is_same::value)); EXPECT_EQ(absl::get(v), "Hello"); - EXPECT_EQ(absl::get(absl::move(v)), "Hello"); + EXPECT_EQ(absl::get(std::move(v)), "Hello"); const Var& const_v = v; using ConstLValueGetType = decltype(absl::get<1>(const_v)); - using ConstRValueGetType = decltype(absl::get<1>(absl::move(const_v))); + using ConstRValueGetType = decltype(absl::get<1>(std::move(const_v))); EXPECT_TRUE((std::is_same::value)); EXPECT_TRUE((std::is_same::value)); EXPECT_EQ(absl::get(const_v), "Hello"); - EXPECT_EQ(absl::get(absl::move(const_v)), "Hello"); + EXPECT_EQ(absl::get(std::move(const_v)), "Hello"); } { Var v = 2.0; using LValueGetType = decltype(absl::get<2>(v)); - using RValueGetType = decltype(absl::get<2>(absl::move(v))); + using RValueGetType = decltype(absl::get<2>(std::move(v))); EXPECT_TRUE((std::is_same::value)); EXPECT_TRUE((std::is_same::value)); EXPECT_EQ(absl::get(v), 2.); - EXPECT_EQ(absl::get(absl::move(v)), 2.); + EXPECT_EQ(absl::get(std::move(v)), 2.); const Var& const_v = v; using ConstLValueGetType = decltype(absl::get<2>(const_v)); - using ConstRValueGetType = decltype(absl::get<2>(absl::move(const_v))); + using ConstRValueGetType = decltype(absl::get<2>(std::move(const_v))); EXPECT_TRUE((std::is_same::value)); EXPECT_TRUE((std::is_same::value)); EXPECT_EQ(absl::get(const_v), 2.); - EXPECT_EQ(absl::get(absl::move(const_v)), 2.); + EXPECT_EQ(absl::get(std::move(const_v)), 2.); } } @@ -1825,13 +1825,13 @@ TEST(VariantTest, VisitRValue) { int operator()(std::string&&, std::string&&) const { return 3; } // NOLINT }; EXPECT_FALSE(absl::visit(Visitor{}, v)); - EXPECT_TRUE(absl::visit(Visitor{}, absl::move(v))); + EXPECT_TRUE(absl::visit(Visitor{}, std::move(v))); // Also test the variadic overload. EXPECT_EQ(0, absl::visit(Visitor{}, v, v)); - EXPECT_EQ(1, absl::visit(Visitor{}, v, absl::move(v))); - EXPECT_EQ(2, absl::visit(Visitor{}, absl::move(v), v)); - EXPECT_EQ(3, absl::visit(Visitor{}, absl::move(v), absl::move(v))); + EXPECT_EQ(1, absl::visit(Visitor{}, v, std::move(v))); + EXPECT_EQ(2, absl::visit(Visitor{}, std::move(v), v)); + EXPECT_EQ(3, absl::visit(Visitor{}, std::move(v), std::move(v))); } TEST(VariantTest, VisitRValueVisitor) { @@ -1862,12 +1862,12 @@ TEST(VariantTest, VisitResultTypeDifferent) { (std::is_same::value)); EXPECT_TRUE( (std::is_same::value)); + decltype(absl::visit(visitor, std::move(v)))>::value)); EXPECT_TRUE(( std::is_same::value)); EXPECT_TRUE( (std::is_same::value)); + decltype(absl::visit(Visitor{}, std::move(v)))>::value)); } TEST(VariantTest, VisitVariadic) { @@ -2225,7 +2225,7 @@ TEST(VariantTest, TestMoveSemantics) { EXPECT_TRUE(absl::holds_alternative>(v)); // Construct a variant by moving from another variant. - Variant v2(absl::move(v)); + Variant v2(std::move(v)); ASSERT_TRUE(absl::holds_alternative>(v2)); ASSERT_NE(nullptr, absl::get>(v2)); EXPECT_EQ(10, *absl::get>(v2)); @@ -2242,7 +2242,7 @@ TEST(VariantTest, TestMoveSemantics) { EXPECT_EQ("foo", *absl::get>(v)); // Move-assign a variant. - v2 = absl::move(v); + v2 = std::move(v); ASSERT_TRUE(absl::holds_alternative>(v2)); EXPECT_EQ("foo", *absl::get>(v2)); EXPECT_TRUE(absl::holds_alternative>(v)); @@ -2568,7 +2568,7 @@ TEST(VariantTest, TestVectorOfMoveonlyVariant) { vec.push_back(absl::make_unique(42)); vec.emplace_back("Hello"); vec.reserve(3); - auto another_vec = absl::move(vec); + auto another_vec = std::move(vec); // As a sanity check, verify vector contents. ASSERT_EQ(2u, another_vec.size()); EXPECT_EQ(42, *absl::get>(another_vec[0])); -- cgit v1.2.3 From b9690836ac18bb85b9ae44db3ca328131281a848 Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Mon, 11 Mar 2024 09:59:17 -0700 Subject: Move GCC uninitialized memory warning suppression into MaybeInitializedPtr. PiperOrigin-RevId: 614701769 Change-Id: I7c2143dd467e376eb4936ef894f3413bba681419 --- absl/container/internal/raw_hash_set.h | 37 +++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 16 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 81f99366..750f51ef 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -1133,6 +1133,23 @@ struct full_soo_tag_t {}; // This allows us to work around an uninitialized memory warning when // constructing begin() iterators in empty hashtables. union MaybeInitializedPtr { + void* get() const { + // Suppress erroneous uninitialized memory errors on GCC. GCC thinks that + // the call to slot_array() in find_or_prepare_insert() is reading + // uninitialized memory, but slot_array is only called there when the table + // is non-empty and this memory is initialized when the table is non-empty. +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#pragma GCC diagnostic ignored "-Wuninitialized" +#endif + return p; +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + } + void set(void* ptr) { p = ptr; } + void* p; }; @@ -1205,7 +1222,7 @@ class CommonFields : public CommonFieldsGenerationInfo { } // Note: we can't use slots() because Qt defines "slots" as a macro. - void* slot_array() const { return heap_or_soo_.heap.slot_array.p; } + void* slot_array() const { return heap_or_soo_.heap.slot_array.get(); } MaybeInitializedPtr slots_union() const { // Suppress erroneous uninitialized memory errors on GCC. #if !defined(__clang__) && defined(__GNUC__) @@ -1217,7 +1234,7 @@ class CommonFields : public CommonFieldsGenerationInfo { #pragma GCC diagnostic pop #endif } - void set_slots(void* s) { heap_or_soo_.heap.slot_array.p = s; } + void set_slots(void* s) { heap_or_soo_.heap.slot_array.set(s); } // The number of filled slots. size_t size() const { return size_ >> HasInfozShift(); } @@ -1807,7 +1824,7 @@ class HashSetResizeHelper { } void* old_slots() const { assert(!was_soo_); - return old_heap_or_soo_.heap.slot_array.p; + return old_heap_or_soo_.heap.slot_array.get(); } size_t old_capacity() const { return old_capacity_; } @@ -2313,7 +2330,7 @@ class raw_hash_set { const GenerationType* generation_ptr) : HashSetIteratorGenerationInfo(generation_ptr), ctrl_(ctrl), - slot_(to_slot(slot.p)) { + slot_(to_slot(slot.get())) { // This assumption helps the compiler know that any non-end iterator is // not equal to any end iterator. ABSL_ASSUME(ctrl != nullptr); @@ -3750,19 +3767,7 @@ class raw_hash_set { } slot_type* slot_array() const { assert(!is_soo()); - // Suppress erroneous uninitialized memory errors on GCC. GCC thinks that - // the call to slot_array() in find_or_prepare_insert() is reading - // uninitialized memory, but slot_array is only called there when the table - // is non-empty and this memory is initialized when the table is non-empty. -#if !defined(__clang__) && defined(__GNUC__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#pragma GCC diagnostic ignored "-Wuninitialized" -#endif return static_cast(common().slot_array()); -#if !defined(__clang__) && defined(__GNUC__) -#pragma GCC diagnostic pop -#endif } slot_type* soo_slot() { assert(is_soo()); -- cgit v1.2.3 From 686aae12d4766631cb2e0f9da0895f4df6db8c30 Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Tue, 12 Mar 2024 12:20:46 -0700 Subject: Make swisstable SOO support GDB pretty printing and still compile in OSS. PiperOrigin-RevId: 615131303 Change-Id: I68fcbdd943594983c67f8e07810b05d5fa9a6f2e --- absl/container/internal/raw_hash_set.h | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 750f51ef..258458b0 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -2195,11 +2195,6 @@ class raw_hash_set { sizeof(slot_type) <= sizeof(HeapOrSoo) && alignof(slot_type) <= alignof(HeapOrSoo); } - // TODO(b/289225379): this is used for pretty printing in GDB/LLDB, but if we - // use this instead of SooEnabled(), then we get compile errors in some OSS - // compilers due to incomplete mapped_type in flat_hash_map. We need to - // resolve this before launching SOO. - // constexpr static bool kSooEnabled = SooEnabled(); // Whether `size` fits in the SOO capacity of this table. bool fits_in_soo(size_t size) const { @@ -2689,9 +2684,9 @@ class raw_hash_set { const size_t cap = common().capacity(); // Use local variables because compiler complains about using functions in // assume. - ABSL_ATTRIBUTE_UNUSED static constexpr bool kSooEnabled = SooEnabled(); - ABSL_ATTRIBUTE_UNUSED static constexpr size_t kSooCapacity = SooCapacity(); - ABSL_ASSUME(!kSooEnabled || cap >= kSooCapacity); + ABSL_ATTRIBUTE_UNUSED static constexpr bool kEnabled = SooEnabled(); + ABSL_ATTRIBUTE_UNUSED static constexpr size_t kCapacity = SooCapacity(); + ABSL_ASSUME(!kEnabled || cap >= kCapacity); return cap; } size_t max_size() const { return (std::numeric_limits::max)(); } -- cgit v1.2.3 From 038561296676d1cae4a3cee30f8c924befbb6083 Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Tue, 12 Mar 2024 14:29:39 -0700 Subject: Add extern templates for common swisstable types. Motivation: mitigate linker input size increase from swisstable optimizations. Note: the changes in raw_hash_set.h are fixing build errors that happened when adding the explicit instantiations. The change in unchecked_deref is because set iterators have const reference access whereas map iterators have mutable reference access and the function is never actually called for sets (it's used in raw_hash_map) so it wasn't needed before. I'm not sure why the soo_slot/soo_iterator problems didn't cause compile errors earlier. PiperOrigin-RevId: 615174043 Change-Id: Iac5eb2332a76e9b70021156fbb2b8def47a5391d --- CMake/AbseilDll.cmake | 4 ++++ absl/container/BUILD.bazel | 10 +++++++++ absl/container/CMakeLists.txt | 14 ++++++++++++ absl/container/flat_hash_map.cc | 40 +++++++++++++++++++++++++++++++++ absl/container/flat_hash_map.h | 41 ++++++++++++++++++++++++++++++++++ absl/container/flat_hash_set.cc | 36 +++++++++++++++++++++++++++++ absl/container/flat_hash_set.h | 29 ++++++++++++++++++++++++ absl/container/internal/raw_hash_set.h | 32 ++++++++++++++++++++------ absl/container/node_hash_map.cc | 40 +++++++++++++++++++++++++++++++++ absl/container/node_hash_map.h | 40 +++++++++++++++++++++++++++++++++ absl/container/node_hash_set.cc | 36 +++++++++++++++++++++++++++++ absl/container/node_hash_set.h | 30 +++++++++++++++++++++++++ 12 files changed, 345 insertions(+), 7 deletions(-) create mode 100644 absl/container/flat_hash_map.cc create mode 100644 absl/container/flat_hash_set.cc create mode 100644 absl/container/node_hash_map.cc create mode 100644 absl/container/node_hash_set.cc (limited to 'absl/container/internal') diff --git a/CMake/AbseilDll.cmake b/CMake/AbseilDll.cmake index 47f3beeb..4beafd7a 100644 --- a/CMake/AbseilDll.cmake +++ b/CMake/AbseilDll.cmake @@ -68,7 +68,9 @@ set(ABSL_INTERNAL_DLL_FILES "container/btree_set.h" "container/fixed_array.h" "container/flat_hash_map.h" + "container/flat_hash_map.cc" "container/flat_hash_set.h" + "container/flat_hash_set.cc" "container/inlined_vector.h" "container/internal/btree.h" "container/internal/btree_container.h" @@ -91,7 +93,9 @@ set(ABSL_INTERNAL_DLL_FILES "container/internal/raw_hash_set.h" "container/internal/tracked.h" "container/node_hash_map.h" + "container/node_hash_map.cc" "container/node_hash_set.h" + "container/node_hash_set.cc" "crc/crc32c.cc" "crc/crc32c.h" "crc/internal/cpu_detect.cc" diff --git a/absl/container/BUILD.bazel b/absl/container/BUILD.bazel index 0de45263..366bf3cd 100644 --- a/absl/container/BUILD.bazel +++ b/absl/container/BUILD.bazel @@ -242,6 +242,7 @@ NOTEST_TAGS_MOBILE = [ cc_library( name = "flat_hash_map", + srcs = ["flat_hash_map.cc"], hdrs = ["flat_hash_map.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -249,7 +250,9 @@ cc_library( ":container_memory", ":hash_function_defaults", ":raw_hash_map", + ":raw_hash_set", "//absl/algorithm:container", + "//absl/base:config", "//absl/base:core_headers", "//absl/memory", ], @@ -279,6 +282,7 @@ cc_test( cc_library( name = "flat_hash_set", + srcs = ["flat_hash_set.cc"], hdrs = ["flat_hash_set.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -287,6 +291,7 @@ cc_library( ":hash_function_defaults", ":raw_hash_set", "//absl/algorithm:container", + "//absl/base:config", "//absl/base:core_headers", "//absl/memory", ], @@ -318,6 +323,7 @@ cc_test( cc_library( name = "node_hash_map", + srcs = ["node_hash_map.cc"], hdrs = ["node_hash_map.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -326,7 +332,9 @@ cc_library( ":hash_function_defaults", ":node_slot_policy", ":raw_hash_map", + ":raw_hash_set", "//absl/algorithm:container", + "//absl/base:config", "//absl/base:core_headers", "//absl/memory", ], @@ -353,6 +361,7 @@ cc_test( cc_library( name = "node_hash_set", + srcs = ["node_hash_set.cc"], hdrs = ["node_hash_set.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -362,6 +371,7 @@ cc_library( ":node_slot_policy", ":raw_hash_set", "//absl/algorithm:container", + "//absl/base:config", "//absl/base:core_headers", "//absl/memory", ], diff --git a/absl/container/CMakeLists.txt b/absl/container/CMakeLists.txt index 4b08e6a3..8e64adb1 100644 --- a/absl/container/CMakeLists.txt +++ b/absl/container/CMakeLists.txt @@ -283,13 +283,17 @@ absl_cc_library( flat_hash_map HDRS "flat_hash_map.h" + SRCS + "flat_hash_map.cc" COPTS ${ABSL_DEFAULT_COPTS} DEPS + absl::config absl::container_memory absl::core_headers absl::hash_function_defaults absl::raw_hash_map + absl::raw_hash_set absl::algorithm_container absl::memory PUBLIC @@ -321,9 +325,12 @@ absl_cc_library( flat_hash_set HDRS "flat_hash_set.h" + SRCS + "flat_hash_set.cc" COPTS ${ABSL_DEFAULT_COPTS} DEPS + absl::config absl::container_memory absl::hash_function_defaults absl::raw_hash_set @@ -362,14 +369,18 @@ absl_cc_library( node_hash_map HDRS "node_hash_map.h" + SRCS + "node_hash_map.cc" COPTS ${ABSL_DEFAULT_COPTS} DEPS + absl::config absl::container_memory absl::core_headers absl::hash_function_defaults absl::node_slot_policy absl::raw_hash_map + absl::raw_hash_set absl::algorithm_container absl::memory PUBLIC @@ -398,9 +409,12 @@ absl_cc_library( node_hash_set HDRS "node_hash_set.h" + SRCS + "node_hash_set.cc" COPTS ${ABSL_DEFAULT_COPTS} DEPS + absl::config absl::container_memory absl::core_headers absl::hash_function_defaults diff --git a/absl/container/flat_hash_map.cc b/absl/container/flat_hash_map.cc new file mode 100644 index 00000000..d5a25317 --- /dev/null +++ b/absl/container/flat_hash_map.cc @@ -0,0 +1,40 @@ +// Copyright 2024 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/flat_hash_map.h" + +#include +#include + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, int32_t, int32_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, std::string, int32_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, int32_t, std::string); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, int64_t, int64_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, std::string, int64_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, int64_t, std::string); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, uint32_t, uint32_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, std::string, uint32_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, uint32_t, std::string); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, uint64_t, uint64_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, std::string, uint64_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, uint64_t, std::string); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, std::string, std::string); + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/container/flat_hash_map.h b/absl/container/flat_hash_map.h index a33c794f..2f4457d6 100644 --- a/absl/container/flat_hash_map.h +++ b/absl/container/flat_hash_map.h @@ -31,15 +31,20 @@ #define ABSL_CONTAINER_FLAT_HASH_MAP_H_ #include +#include +#include #include +#include #include #include #include "absl/algorithm/container.h" +#include "absl/base/config.h" #include "absl/base/macros.h" #include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export #include "absl/container/internal/raw_hash_map.h" // IWYU pragma: export +#include "absl/container/internal/raw_hash_set.h" // IWYU pragma: export #include "absl/memory/memory.h" namespace absl { @@ -632,6 +637,42 @@ struct IsUnorderedContainer< } // namespace container_algorithm_internal +// Explicit template instantiations for common map types in order to decrease +// linker input size. Note that explicitly instantiating flat_hash_map itself +// doesn't help because it has no non-alias members. If we need to decrease +// linker input size more, we could potentially (a) add more key/value types, +// e.g. string_view/Cord, (b) instantiate some template member functions, e.g. +// operator[]/find. The EXTERN argument is `extern` for the declaration and +// empty for the definition. +#define ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(TEMPLATE, KEY, VALUE) \ + TEMPLATE class absl::container_internal::raw_hash_map< \ + absl::container_internal::FlatHashMapPolicy, \ + absl::container_internal::hash_default_hash, \ + absl::container_internal::hash_default_eq, \ + std::allocator>>; \ + TEMPLATE class absl::container_internal::raw_hash_set< \ + absl::container_internal::FlatHashMapPolicy, \ + absl::container_internal::hash_default_hash, \ + absl::container_internal::hash_default_eq, \ + std::allocator>>; + +// We use exact-width integer types rather than `int`/`long`/`long long` because +// these are the types recommended in the Google C++ style guide and which are +// commonly used in Google code. +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, int32_t, int32_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, std::string, int32_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, int32_t, std::string); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, int64_t, int64_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, std::string, int64_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, int64_t, std::string); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, uint32_t, uint32_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, std::string, uint32_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, uint32_t, std::string); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, uint64_t, uint64_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, std::string, uint64_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, uint64_t, std::string); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, std::string, std::string); + ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/container/flat_hash_set.cc b/absl/container/flat_hash_set.cc new file mode 100644 index 00000000..d07081b8 --- /dev/null +++ b/absl/container/flat_hash_set.cc @@ -0,0 +1,36 @@ +// Copyright 2024 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/flat_hash_set.h" + +#include +#include + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(template, int8_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(template, int16_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(template, int32_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(template, int64_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(template, uint8_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(template, uint16_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(template, uint32_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(template, uint64_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(template, std::string); + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/container/flat_hash_set.h b/absl/container/flat_hash_set.h index 5f72f954..88c7ab21 100644 --- a/absl/container/flat_hash_set.h +++ b/absl/container/flat_hash_set.h @@ -30,11 +30,14 @@ #define ABSL_CONTAINER_FLAT_HASH_SET_H_ #include +#include #include +#include #include #include #include "absl/algorithm/container.h" +#include "absl/base/config.h" #include "absl/base/macros.h" #include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export @@ -523,6 +526,32 @@ struct IsUnorderedContainer> } // namespace container_algorithm_internal +// Explicit template instantiations for common set types in order to decrease +// linker input size. Note that explicitly instantiating flat_hash_set itself +// doesn't help because it has no non-alias members. If we need to decrease +// linker input size more, we could potentially (a) add more key types, e.g. +// string_view/Cord, (b) instantiate some template member functions, e.g. +// find/insert/emplace. The EXTERN argument is `extern` for the declaration and +// empty for the definition. +#define ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(TEMPLATE, KEY) \ + TEMPLATE class absl::container_internal::raw_hash_set< \ + absl::container_internal::FlatHashSetPolicy, \ + absl::container_internal::hash_default_hash, \ + absl::container_internal::hash_default_eq, std::allocator>; + +// We use exact-width integer types rather than `int`/`long`/`long long` because +// these are the types recommended in the Google C++ style guide and which are +// commonly used in Google code. +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(extern template, int8_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(extern template, int16_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(extern template, int32_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(extern template, int64_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(extern template, uint8_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(extern template, uint16_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(extern template, uint32_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(extern template, uint64_t); +ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(extern template, std::string); + ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 258458b0..575491c7 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -1839,7 +1839,7 @@ class HashSetResizeHelper { // Reads `capacity` and updates all other fields based on the result of // the allocation. // - // It also may do the folowing actions: + // It also may do the following actions: // 1. initialize control bytes // 2. initialize slots // 3. deallocate old slots. @@ -2204,9 +2204,14 @@ class raw_hash_set { bool is_soo() const { return fits_in_soo(capacity()); } bool is_full_soo() const { return is_soo() && !empty(); } - // Give an early error when key_type is not hashable/eq. - auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); - auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); + // Give an early error when key_type is not hashable/eq. Definitions are + // provided because otherwise explicit template instantiation fails on MSVC. + auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)) { + ABSL_UNREACHABLE(); + } + auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)) { + ABSL_UNREACHABLE(); + } using AllocTraits = absl::allocator_traits; using SlotAlloc = typename absl::allocator_traits< @@ -2395,7 +2400,18 @@ class raw_hash_set { const_iterator operator++(int) { return inner_++; } friend bool operator==(const const_iterator& a, const const_iterator& b) { + // Suppress erroneous uninitialized memory errors on GCC. This seems to be + // because the slot pointer in the inner_ iterator is uninitialized, even + // though that pointer is not used when uninitialized. + // Similar bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=112637. +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif return a.inner_ == b.inner_; +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic pop +#endif } friend bool operator!=(const const_iterator& a, const const_iterator& b) { return !(a == b); @@ -3719,7 +3735,9 @@ class raw_hash_set { return const_cast(this)->iterator_at(i); } - reference unchecked_deref(iterator it) { return it.unchecked_deref(); } + reference unchecked_deref(iterator it) { + return const_cast(it.unchecked_deref()); + } private: friend struct RawHashSetTestOnlyAccess; @@ -3769,13 +3787,13 @@ class raw_hash_set { return static_cast(common().soo_data()); } const slot_type* soo_slot() const { - return reinterpret_cast(this)->soo_slot(); + return const_cast(this)->soo_slot(); } iterator soo_iterator() { return {SooControl(), soo_slot(), common().generation_ptr()}; } const_iterator soo_iterator() const { - return reinterpret_cast(this)->soo_iterator(); + return const_cast(this)->soo_iterator(); } HashtablezInfoHandle infoz() { assert(!is_soo()); diff --git a/absl/container/node_hash_map.cc b/absl/container/node_hash_map.cc new file mode 100644 index 00000000..ccf8c599 --- /dev/null +++ b/absl/container/node_hash_map.cc @@ -0,0 +1,40 @@ +// Copyright 2024 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/node_hash_map.h" + +#include +#include + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, int32_t, int32_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, std::string, int32_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, int32_t, std::string); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, int64_t, int64_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, std::string, int64_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, int64_t, std::string); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, uint32_t, uint32_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, std::string, uint32_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, uint32_t, std::string); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, uint64_t, uint64_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, std::string, uint64_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, uint64_t, std::string); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, std::string, std::string); + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/container/node_hash_map.h b/absl/container/node_hash_map.h index cb41543c..acc21542 100644 --- a/absl/container/node_hash_map.h +++ b/absl/container/node_hash_map.h @@ -37,11 +37,15 @@ #define ABSL_CONTAINER_NODE_HASH_MAP_H_ #include +#include +#include +#include #include #include #include #include "absl/algorithm/container.h" +#include "absl/base/config.h" #include "absl/base/macros.h" #include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export @@ -623,6 +627,42 @@ struct IsUnorderedContainer< } // namespace container_algorithm_internal +// Explicit template instantiations for common map types in order to decrease +// linker input size. Note that explicitly instantiating node_hash_map itself +// doesn't help because it has no non-alias members. If we need to decrease +// linker input size more, we could potentially (a) add more key/value types, +// e.g. string_view/Cord, (b) instantiate some template member functions, e.g. +// operator[]/find. The EXTERN argument is `extern` for the declaration and +// empty for the definition. +#define ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(TEMPLATE, KEY, VALUE) \ + TEMPLATE class absl::container_internal::raw_hash_map< \ + absl::container_internal::NodeHashMapPolicy, \ + absl::container_internal::hash_default_hash, \ + absl::container_internal::hash_default_eq, \ + std::allocator>>; \ + TEMPLATE class absl::container_internal::raw_hash_set< \ + absl::container_internal::NodeHashMapPolicy, \ + absl::container_internal::hash_default_hash, \ + absl::container_internal::hash_default_eq, \ + std::allocator>>; + +// We use exact-width integer types rather than `int`/`long`/`long long` because +// these are the types recommended in the Google C++ style guide and which are +// commonly used in Google code. +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, int32_t, int32_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, std::string, int32_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, int32_t, std::string); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, int64_t, int64_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, std::string, int64_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, int64_t, std::string); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, uint32_t, uint32_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, std::string, uint32_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, uint32_t, std::string); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, uint64_t, uint64_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, std::string, uint64_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, uint64_t, std::string); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, std::string, std::string); + ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/container/node_hash_set.cc b/absl/container/node_hash_set.cc new file mode 100644 index 00000000..39226c61 --- /dev/null +++ b/absl/container/node_hash_set.cc @@ -0,0 +1,36 @@ +// Copyright 2024 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/node_hash_set.h" + +#include +#include + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(template, int8_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(template, int16_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(template, int32_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(template, int64_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(template, uint8_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(template, uint16_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(template, uint32_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(template, uint64_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(template, std::string); + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/container/node_hash_set.h b/absl/container/node_hash_set.h index 8cc4b624..77b49f9b 100644 --- a/absl/container/node_hash_set.h +++ b/absl/container/node_hash_set.h @@ -36,9 +36,13 @@ #define ABSL_CONTAINER_NODE_HASH_SET_H_ #include +#include +#include +#include #include #include "absl/algorithm/container.h" +#include "absl/base/config.h" #include "absl/base/macros.h" #include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export @@ -518,6 +522,32 @@ struct IsUnorderedContainer> : std::true_type {}; } // namespace container_algorithm_internal + +// Explicit template instantiations for common set types in order to decrease +// linker input size. Note that explicitly instantiating node_hash_set itself +// doesn't help because it has no non-alias members. If we need to decrease +// linker input size more, we could potentially (a) add more key types, e.g. +// string_view/Cord, (b) instantiate some template member functions, e.g. +// find/insert/emplace. +#define ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(TEMPLATE, KEY) \ + TEMPLATE class absl::container_internal::raw_hash_set< \ + absl::container_internal::NodeHashSetPolicy, \ + absl::container_internal::hash_default_hash, \ + absl::container_internal::hash_default_eq, std::allocator>; + +// We use exact-width integer types rather than `int`/`long`/`long long` because +// these are the types recommended in the Google C++ style guide and which are +// commonly used in Google code. +ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(extern template, int8_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(extern template, int16_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(extern template, int32_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(extern template, int64_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(extern template, uint8_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(extern template, uint16_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(extern template, uint32_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(extern template, uint64_t); +ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(extern template, std::string); + ABSL_NAMESPACE_END } // namespace absl -- cgit v1.2.3 From 321addf0511571e9910dc13a25221473656419d4 Mon Sep 17 00:00:00 2001 From: Vitaly Goldshteyn Date: Wed, 13 Mar 2024 05:34:23 -0700 Subject: Test that rehash(0) reduces capacity to minimum. PiperOrigin-RevId: 615380243 Change-Id: I5400b40a6bc5ac52ece5d4fa6da7df9e4ff50855 --- absl/container/internal/raw_hash_set_test.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 9180db59..66baeb64 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -885,12 +885,17 @@ TYPED_TEST_P(SmallTableResizeTest, ResizeReduceSmallTables) { for (size_t source_size = 0; source_size < 32; ++source_size) { for (size_t target_size = 0; target_size <= source_size; ++target_size) { TypeParam t; - t.reserve(source_size); size_t inserted_count = std::min(source_size, 5); for (size_t i = 0; i < inserted_count; ++i) { t.insert(static_cast(i)); } + const size_t minimum_capacity = t.capacity(); + t.reserve(source_size); t.rehash(target_size); + if (target_size == 0) { + EXPECT_EQ(t.capacity(), minimum_capacity) + << "rehash(0) must resize to the minimum capacity"; + } for (size_t i = 0; i < inserted_count; ++i) { EXPECT_TRUE(t.find(static_cast(i)) != t.end()); EXPECT_EQ(*t.find(static_cast(i)), static_cast(i)); -- cgit v1.2.3 From c6ed744ea531edf745b55c4e5679d7e435406c0d Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Wed, 13 Mar 2024 11:59:30 -0700 Subject: Roll back extern template instatiations in swisstable due to binary size increases in shared libraries. PiperOrigin-RevId: 615497725 Change-Id: Ic29db8923ea4ea7cd0b01b396896fa9fff8c74b0 --- CMake/AbseilDll.cmake | 4 ---- absl/container/BUILD.bazel | 10 --------- absl/container/CMakeLists.txt | 14 ------------ absl/container/flat_hash_map.cc | 40 --------------------------------- absl/container/flat_hash_map.h | 41 ---------------------------------- absl/container/flat_hash_set.cc | 36 ----------------------------- absl/container/flat_hash_set.h | 29 ------------------------ absl/container/internal/raw_hash_set.h | 32 ++++++-------------------- absl/container/node_hash_map.cc | 40 --------------------------------- absl/container/node_hash_map.h | 40 --------------------------------- absl/container/node_hash_set.cc | 36 ----------------------------- absl/container/node_hash_set.h | 30 ------------------------- 12 files changed, 7 insertions(+), 345 deletions(-) delete mode 100644 absl/container/flat_hash_map.cc delete mode 100644 absl/container/flat_hash_set.cc delete mode 100644 absl/container/node_hash_map.cc delete mode 100644 absl/container/node_hash_set.cc (limited to 'absl/container/internal') diff --git a/CMake/AbseilDll.cmake b/CMake/AbseilDll.cmake index 4beafd7a..47f3beeb 100644 --- a/CMake/AbseilDll.cmake +++ b/CMake/AbseilDll.cmake @@ -68,9 +68,7 @@ set(ABSL_INTERNAL_DLL_FILES "container/btree_set.h" "container/fixed_array.h" "container/flat_hash_map.h" - "container/flat_hash_map.cc" "container/flat_hash_set.h" - "container/flat_hash_set.cc" "container/inlined_vector.h" "container/internal/btree.h" "container/internal/btree_container.h" @@ -93,9 +91,7 @@ set(ABSL_INTERNAL_DLL_FILES "container/internal/raw_hash_set.h" "container/internal/tracked.h" "container/node_hash_map.h" - "container/node_hash_map.cc" "container/node_hash_set.h" - "container/node_hash_set.cc" "crc/crc32c.cc" "crc/crc32c.h" "crc/internal/cpu_detect.cc" diff --git a/absl/container/BUILD.bazel b/absl/container/BUILD.bazel index 366bf3cd..0de45263 100644 --- a/absl/container/BUILD.bazel +++ b/absl/container/BUILD.bazel @@ -242,7 +242,6 @@ NOTEST_TAGS_MOBILE = [ cc_library( name = "flat_hash_map", - srcs = ["flat_hash_map.cc"], hdrs = ["flat_hash_map.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -250,9 +249,7 @@ cc_library( ":container_memory", ":hash_function_defaults", ":raw_hash_map", - ":raw_hash_set", "//absl/algorithm:container", - "//absl/base:config", "//absl/base:core_headers", "//absl/memory", ], @@ -282,7 +279,6 @@ cc_test( cc_library( name = "flat_hash_set", - srcs = ["flat_hash_set.cc"], hdrs = ["flat_hash_set.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -291,7 +287,6 @@ cc_library( ":hash_function_defaults", ":raw_hash_set", "//absl/algorithm:container", - "//absl/base:config", "//absl/base:core_headers", "//absl/memory", ], @@ -323,7 +318,6 @@ cc_test( cc_library( name = "node_hash_map", - srcs = ["node_hash_map.cc"], hdrs = ["node_hash_map.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -332,9 +326,7 @@ cc_library( ":hash_function_defaults", ":node_slot_policy", ":raw_hash_map", - ":raw_hash_set", "//absl/algorithm:container", - "//absl/base:config", "//absl/base:core_headers", "//absl/memory", ], @@ -361,7 +353,6 @@ cc_test( cc_library( name = "node_hash_set", - srcs = ["node_hash_set.cc"], hdrs = ["node_hash_set.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, @@ -371,7 +362,6 @@ cc_library( ":node_slot_policy", ":raw_hash_set", "//absl/algorithm:container", - "//absl/base:config", "//absl/base:core_headers", "//absl/memory", ], diff --git a/absl/container/CMakeLists.txt b/absl/container/CMakeLists.txt index 8e64adb1..4b08e6a3 100644 --- a/absl/container/CMakeLists.txt +++ b/absl/container/CMakeLists.txt @@ -283,17 +283,13 @@ absl_cc_library( flat_hash_map HDRS "flat_hash_map.h" - SRCS - "flat_hash_map.cc" COPTS ${ABSL_DEFAULT_COPTS} DEPS - absl::config absl::container_memory absl::core_headers absl::hash_function_defaults absl::raw_hash_map - absl::raw_hash_set absl::algorithm_container absl::memory PUBLIC @@ -325,12 +321,9 @@ absl_cc_library( flat_hash_set HDRS "flat_hash_set.h" - SRCS - "flat_hash_set.cc" COPTS ${ABSL_DEFAULT_COPTS} DEPS - absl::config absl::container_memory absl::hash_function_defaults absl::raw_hash_set @@ -369,18 +362,14 @@ absl_cc_library( node_hash_map HDRS "node_hash_map.h" - SRCS - "node_hash_map.cc" COPTS ${ABSL_DEFAULT_COPTS} DEPS - absl::config absl::container_memory absl::core_headers absl::hash_function_defaults absl::node_slot_policy absl::raw_hash_map - absl::raw_hash_set absl::algorithm_container absl::memory PUBLIC @@ -409,12 +398,9 @@ absl_cc_library( node_hash_set HDRS "node_hash_set.h" - SRCS - "node_hash_set.cc" COPTS ${ABSL_DEFAULT_COPTS} DEPS - absl::config absl::container_memory absl::core_headers absl::hash_function_defaults diff --git a/absl/container/flat_hash_map.cc b/absl/container/flat_hash_map.cc deleted file mode 100644 index d5a25317..00000000 --- a/absl/container/flat_hash_map.cc +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2024 The Abseil Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/container/flat_hash_map.h" - -#include -#include - -#include "absl/base/config.h" - -namespace absl { -ABSL_NAMESPACE_BEGIN - -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, int32_t, int32_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, std::string, int32_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, int32_t, std::string); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, int64_t, int64_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, std::string, int64_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, int64_t, std::string); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, uint32_t, uint32_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, std::string, uint32_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, uint32_t, std::string); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, uint64_t, uint64_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, std::string, uint64_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, uint64_t, std::string); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(template, std::string, std::string); - -ABSL_NAMESPACE_END -} // namespace absl diff --git a/absl/container/flat_hash_map.h b/absl/container/flat_hash_map.h index 2f4457d6..a33c794f 100644 --- a/absl/container/flat_hash_map.h +++ b/absl/container/flat_hash_map.h @@ -31,20 +31,15 @@ #define ABSL_CONTAINER_FLAT_HASH_MAP_H_ #include -#include -#include #include -#include #include #include #include "absl/algorithm/container.h" -#include "absl/base/config.h" #include "absl/base/macros.h" #include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export #include "absl/container/internal/raw_hash_map.h" // IWYU pragma: export -#include "absl/container/internal/raw_hash_set.h" // IWYU pragma: export #include "absl/memory/memory.h" namespace absl { @@ -637,42 +632,6 @@ struct IsUnorderedContainer< } // namespace container_algorithm_internal -// Explicit template instantiations for common map types in order to decrease -// linker input size. Note that explicitly instantiating flat_hash_map itself -// doesn't help because it has no non-alias members. If we need to decrease -// linker input size more, we could potentially (a) add more key/value types, -// e.g. string_view/Cord, (b) instantiate some template member functions, e.g. -// operator[]/find. The EXTERN argument is `extern` for the declaration and -// empty for the definition. -#define ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(TEMPLATE, KEY, VALUE) \ - TEMPLATE class absl::container_internal::raw_hash_map< \ - absl::container_internal::FlatHashMapPolicy, \ - absl::container_internal::hash_default_hash, \ - absl::container_internal::hash_default_eq, \ - std::allocator>>; \ - TEMPLATE class absl::container_internal::raw_hash_set< \ - absl::container_internal::FlatHashMapPolicy, \ - absl::container_internal::hash_default_hash, \ - absl::container_internal::hash_default_eq, \ - std::allocator>>; - -// We use exact-width integer types rather than `int`/`long`/`long long` because -// these are the types recommended in the Google C++ style guide and which are -// commonly used in Google code. -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, int32_t, int32_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, std::string, int32_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, int32_t, std::string); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, int64_t, int64_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, std::string, int64_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, int64_t, std::string); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, uint32_t, uint32_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, std::string, uint32_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, uint32_t, std::string); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, uint64_t, uint64_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, std::string, uint64_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, uint64_t, std::string); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_MAP(extern template, std::string, std::string); - ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/container/flat_hash_set.cc b/absl/container/flat_hash_set.cc deleted file mode 100644 index d07081b8..00000000 --- a/absl/container/flat_hash_set.cc +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2024 The Abseil Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/container/flat_hash_set.h" - -#include -#include - -#include "absl/base/config.h" - -namespace absl { -ABSL_NAMESPACE_BEGIN - -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(template, int8_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(template, int16_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(template, int32_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(template, int64_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(template, uint8_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(template, uint16_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(template, uint32_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(template, uint64_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(template, std::string); - -ABSL_NAMESPACE_END -} // namespace absl diff --git a/absl/container/flat_hash_set.h b/absl/container/flat_hash_set.h index 88c7ab21..5f72f954 100644 --- a/absl/container/flat_hash_set.h +++ b/absl/container/flat_hash_set.h @@ -30,14 +30,11 @@ #define ABSL_CONTAINER_FLAT_HASH_SET_H_ #include -#include #include -#include #include #include #include "absl/algorithm/container.h" -#include "absl/base/config.h" #include "absl/base/macros.h" #include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export @@ -526,32 +523,6 @@ struct IsUnorderedContainer> } // namespace container_algorithm_internal -// Explicit template instantiations for common set types in order to decrease -// linker input size. Note that explicitly instantiating flat_hash_set itself -// doesn't help because it has no non-alias members. If we need to decrease -// linker input size more, we could potentially (a) add more key types, e.g. -// string_view/Cord, (b) instantiate some template member functions, e.g. -// find/insert/emplace. The EXTERN argument is `extern` for the declaration and -// empty for the definition. -#define ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(TEMPLATE, KEY) \ - TEMPLATE class absl::container_internal::raw_hash_set< \ - absl::container_internal::FlatHashSetPolicy, \ - absl::container_internal::hash_default_hash, \ - absl::container_internal::hash_default_eq, std::allocator>; - -// We use exact-width integer types rather than `int`/`long`/`long long` because -// these are the types recommended in the Google C++ style guide and which are -// commonly used in Google code. -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(extern template, int8_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(extern template, int16_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(extern template, int32_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(extern template, int64_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(extern template, uint8_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(extern template, uint16_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(extern template, uint32_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(extern template, uint64_t); -ABSL_INTERNAL_TEMPLATE_FLAT_HASH_SET(extern template, std::string); - ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 575491c7..258458b0 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -1839,7 +1839,7 @@ class HashSetResizeHelper { // Reads `capacity` and updates all other fields based on the result of // the allocation. // - // It also may do the following actions: + // It also may do the folowing actions: // 1. initialize control bytes // 2. initialize slots // 3. deallocate old slots. @@ -2204,14 +2204,9 @@ class raw_hash_set { bool is_soo() const { return fits_in_soo(capacity()); } bool is_full_soo() const { return is_soo() && !empty(); } - // Give an early error when key_type is not hashable/eq. Definitions are - // provided because otherwise explicit template instantiation fails on MSVC. - auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)) { - ABSL_UNREACHABLE(); - } - auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)) { - ABSL_UNREACHABLE(); - } + // Give an early error when key_type is not hashable/eq. + auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); + auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); using AllocTraits = absl::allocator_traits; using SlotAlloc = typename absl::allocator_traits< @@ -2400,18 +2395,7 @@ class raw_hash_set { const_iterator operator++(int) { return inner_++; } friend bool operator==(const const_iterator& a, const const_iterator& b) { - // Suppress erroneous uninitialized memory errors on GCC. This seems to be - // because the slot pointer in the inner_ iterator is uninitialized, even - // though that pointer is not used when uninitialized. - // Similar bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=112637. -#if !defined(__clang__) && defined(__GNUC__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif return a.inner_ == b.inner_; -#if !defined(__clang__) && defined(__GNUC__) -#pragma GCC diagnostic pop -#endif } friend bool operator!=(const const_iterator& a, const const_iterator& b) { return !(a == b); @@ -3735,9 +3719,7 @@ class raw_hash_set { return const_cast(this)->iterator_at(i); } - reference unchecked_deref(iterator it) { - return const_cast(it.unchecked_deref()); - } + reference unchecked_deref(iterator it) { return it.unchecked_deref(); } private: friend struct RawHashSetTestOnlyAccess; @@ -3787,13 +3769,13 @@ class raw_hash_set { return static_cast(common().soo_data()); } const slot_type* soo_slot() const { - return const_cast(this)->soo_slot(); + return reinterpret_cast(this)->soo_slot(); } iterator soo_iterator() { return {SooControl(), soo_slot(), common().generation_ptr()}; } const_iterator soo_iterator() const { - return const_cast(this)->soo_iterator(); + return reinterpret_cast(this)->soo_iterator(); } HashtablezInfoHandle infoz() { assert(!is_soo()); diff --git a/absl/container/node_hash_map.cc b/absl/container/node_hash_map.cc deleted file mode 100644 index ccf8c599..00000000 --- a/absl/container/node_hash_map.cc +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2024 The Abseil Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/container/node_hash_map.h" - -#include -#include - -#include "absl/base/config.h" - -namespace absl { -ABSL_NAMESPACE_BEGIN - -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, int32_t, int32_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, std::string, int32_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, int32_t, std::string); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, int64_t, int64_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, std::string, int64_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, int64_t, std::string); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, uint32_t, uint32_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, std::string, uint32_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, uint32_t, std::string); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, uint64_t, uint64_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, std::string, uint64_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, uint64_t, std::string); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(template, std::string, std::string); - -ABSL_NAMESPACE_END -} // namespace absl diff --git a/absl/container/node_hash_map.h b/absl/container/node_hash_map.h index acc21542..cb41543c 100644 --- a/absl/container/node_hash_map.h +++ b/absl/container/node_hash_map.h @@ -37,15 +37,11 @@ #define ABSL_CONTAINER_NODE_HASH_MAP_H_ #include -#include -#include -#include #include #include #include #include "absl/algorithm/container.h" -#include "absl/base/config.h" #include "absl/base/macros.h" #include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export @@ -627,42 +623,6 @@ struct IsUnorderedContainer< } // namespace container_algorithm_internal -// Explicit template instantiations for common map types in order to decrease -// linker input size. Note that explicitly instantiating node_hash_map itself -// doesn't help because it has no non-alias members. If we need to decrease -// linker input size more, we could potentially (a) add more key/value types, -// e.g. string_view/Cord, (b) instantiate some template member functions, e.g. -// operator[]/find. The EXTERN argument is `extern` for the declaration and -// empty for the definition. -#define ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(TEMPLATE, KEY, VALUE) \ - TEMPLATE class absl::container_internal::raw_hash_map< \ - absl::container_internal::NodeHashMapPolicy, \ - absl::container_internal::hash_default_hash, \ - absl::container_internal::hash_default_eq, \ - std::allocator>>; \ - TEMPLATE class absl::container_internal::raw_hash_set< \ - absl::container_internal::NodeHashMapPolicy, \ - absl::container_internal::hash_default_hash, \ - absl::container_internal::hash_default_eq, \ - std::allocator>>; - -// We use exact-width integer types rather than `int`/`long`/`long long` because -// these are the types recommended in the Google C++ style guide and which are -// commonly used in Google code. -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, int32_t, int32_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, std::string, int32_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, int32_t, std::string); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, int64_t, int64_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, std::string, int64_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, int64_t, std::string); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, uint32_t, uint32_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, std::string, uint32_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, uint32_t, std::string); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, uint64_t, uint64_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, std::string, uint64_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, uint64_t, std::string); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_MAP(extern template, std::string, std::string); - ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/container/node_hash_set.cc b/absl/container/node_hash_set.cc deleted file mode 100644 index 39226c61..00000000 --- a/absl/container/node_hash_set.cc +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2024 The Abseil Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/container/node_hash_set.h" - -#include -#include - -#include "absl/base/config.h" - -namespace absl { -ABSL_NAMESPACE_BEGIN - -ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(template, int8_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(template, int16_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(template, int32_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(template, int64_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(template, uint8_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(template, uint16_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(template, uint32_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(template, uint64_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(template, std::string); - -ABSL_NAMESPACE_END -} // namespace absl diff --git a/absl/container/node_hash_set.h b/absl/container/node_hash_set.h index 77b49f9b..8cc4b624 100644 --- a/absl/container/node_hash_set.h +++ b/absl/container/node_hash_set.h @@ -36,13 +36,9 @@ #define ABSL_CONTAINER_NODE_HASH_SET_H_ #include -#include -#include -#include #include #include "absl/algorithm/container.h" -#include "absl/base/config.h" #include "absl/base/macros.h" #include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export @@ -522,32 +518,6 @@ struct IsUnorderedContainer> : std::true_type {}; } // namespace container_algorithm_internal - -// Explicit template instantiations for common set types in order to decrease -// linker input size. Note that explicitly instantiating node_hash_set itself -// doesn't help because it has no non-alias members. If we need to decrease -// linker input size more, we could potentially (a) add more key types, e.g. -// string_view/Cord, (b) instantiate some template member functions, e.g. -// find/insert/emplace. -#define ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(TEMPLATE, KEY) \ - TEMPLATE class absl::container_internal::raw_hash_set< \ - absl::container_internal::NodeHashSetPolicy, \ - absl::container_internal::hash_default_hash, \ - absl::container_internal::hash_default_eq, std::allocator>; - -// We use exact-width integer types rather than `int`/`long`/`long long` because -// these are the types recommended in the Google C++ style guide and which are -// commonly used in Google code. -ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(extern template, int8_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(extern template, int16_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(extern template, int32_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(extern template, int64_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(extern template, uint8_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(extern template, uint16_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(extern template, uint32_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(extern template, uint64_t); -ABSL_INTERNAL_TEMPLATE_NODE_HASH_SET(extern template, std::string); - ABSL_NAMESPACE_END } // namespace absl -- cgit v1.2.3 From 153186b6931d0755448da8d200e1b8e43acede48 Mon Sep 17 00:00:00 2001 From: Dino Radakovic Date: Fri, 15 Mar 2024 12:55:53 -0700 Subject: `layout`: Use auto return type for functions that explicitly instantiate std::tuple in return statements This improves readability by avoiding spelling the same type twice, the first time with even more boilerplate (e.g. `typename`). Return type deduction is a C++14 feature, and Abseil [currently supports](https://github.com/google/oss-policies-info/blob/9a9bfe8a4a12be20757497074fc2f0ecb77438ad/foundational-cxx-support-matrix.md) C++ >= 14. PiperOrigin-RevId: 616218396 Change-Id: I82aeec878dd69001d2cf822db6512f5a62baec02 --- absl/container/internal/layout.h | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/layout.h b/absl/container/internal/layout.h index 1bf739cc..aa42a8ec 100644 --- a/absl/container/internal/layout.h +++ b/absl/container/internal/layout.h @@ -504,9 +504,7 @@ class LayoutImpl, absl::index_sequence, // Note: We're not using ElementType alias here because it does not compile // under MSVC. template - std::tuple::type>*...> - Pointers(Char* p) const { + auto Pointers(Char* p) const { return std::tuple>*...>( Pointer(p)...); } @@ -562,9 +560,7 @@ class LayoutImpl, absl::index_sequence, // Note: We're not using ElementType alias here because it does not compile // under MSVC. template - std::tuple::type>>...> - Slices(Char* p) const { + auto Slices(Char* p) const { // Workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63875 (fixed // in 6.1). (void)p; -- cgit v1.2.3 From 56d3f227154af7a646302ae4a0891f14a2d3616b Mon Sep 17 00:00:00 2001 From: Dino Radakovic Date: Fri, 15 Mar 2024 14:36:31 -0700 Subject: `layout`: Mark parameter of Slices with ABSL_ATTRIBUTE_UNUSED, remove old workaround The workaround was added for a bug in GCC < 6.1., which had since been fixed. Abseil only [supports](https://github.com/google/oss-policies-info/blob/9a9bfe8a4a12be20757497074fc2f0ecb77438ad/foundational-cxx-support-matrix.md) GCC > 7.3.1. However, removing the workaround still trips even more recent GCC, such as 13.1. When `SizeSeq` is empty, `p` is not actually used. Marking it as `ABSL_ATTRIBUTE_UNUSED` silences the GCC warning for that case too. We could disable `Slices` altogether when `SizeSeq` is empty, but that would be a breaking change (even though this API is internal), and possibly hurt generic programming use (they'd have to check if their parameter packs are empty). PiperOrigin-RevId: 616245873 Change-Id: I77f7b0b921dfd63fb01c5223851ad1d8a7da233b --- absl/container/internal/layout.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/layout.h b/absl/container/internal/layout.h index aa42a8ec..341c8262 100644 --- a/absl/container/internal/layout.h +++ b/absl/container/internal/layout.h @@ -171,6 +171,7 @@ #include #include +#include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/debugging/internal/demangle.h" #include "absl/meta/type_traits.h" @@ -559,11 +560,11 @@ class LayoutImpl, absl::index_sequence, // // Note: We're not using ElementType alias here because it does not compile // under MSVC. + // + // Note: We mark the parameter as unused because GCC detects it is not used + // when `SizeSeq` is empty [-Werror=unused-but-set-parameter]. template - auto Slices(Char* p) const { - // Workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63875 (fixed - // in 6.1). - (void)p; + auto Slices(ABSL_ATTRIBUTE_UNUSED Char* p) const { return std::tuple>>...>( Slice(p)...); } -- cgit v1.2.3 From 5839a14828f31f1c2876003677e0ce8e28544b1f Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Mon, 18 Mar 2024 07:11:24 -0700 Subject: Add a feature to container_internal::Layout that lets you specify some array sizes at compile-time as template parameters. This can make offset and size calculations faster. In particular, it seems to always improve the performance of AllocSize(), and it sometimes improves the performance of other functions, e.g. when the Layout object outlives the function that created it. PiperOrigin-RevId: 616817169 Change-Id: Id1d318d7d2af68783f9f59090d89c642be6ae558 --- absl/container/internal/layout.h | 245 ++++++++++++----- absl/container/internal/layout_benchmark.cc | 178 ++++++++++++- absl/container/internal/layout_test.cc | 396 +++++++++++++++++++++++++++- 3 files changed, 753 insertions(+), 66 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/layout.h b/absl/container/internal/layout.h index 341c8262..7f2b83a2 100644 --- a/absl/container/internal/layout.h +++ b/absl/container/internal/layout.h @@ -81,9 +81,30 @@ // } // // The layout we used above combines fixed-size with dynamically-sized fields. -// This is quite common. Layout is optimized for this use case and generates -// optimal code. All computations that can be performed at compile time are -// indeed performed at compile time. +// This is quite common. Layout is optimized for this use case and attempts to +// generate optimal code. To help the compiler do that in more cases, you can +// specify the fixed sizes using `WithStaticSizes`. This ensures that all +// computations that can be performed at compile time are indeed performed at +// compile time. E.g.: +// +// using SL = L::WithStaticSizes<1, 1>; +// +// void Use(unsigned char* p) { +// // First, extract N and M. +// // Using `prefix` we can access the first three arrays but not more. +// // +// // More details: The first element always has offset 0. `SL` +// // has offsets for the second and third array based on sizes of +// // the first and second array, specified via `WithStaticSizes`. +// constexpr auto prefix = SL::Partial(); +// size_t n = *prefix.Pointer<0>(p); +// size_t m = *prefix.Pointer<1>(p); +// +// // Now we can get a pointer to the final payload. +// const SL layout(n, m); +// double* a = layout.Pointer(p); +// int* b = layout.Pointer(p); +// } // // Efficiency tip: The order of fields matters. In `Layout` try to // ensure that `alignof(T1) >= ... >= alignof(TN)`. This way you'll have no @@ -107,7 +128,7 @@ // CompactString(const char* s = "") { // const size_t size = strlen(s); // // size_t[1] followed by char[size + 1]. -// const L layout(1, size + 1); +// const L layout(size + 1); // p_.reset(new unsigned char[layout.AllocSize()]); // // If running under ASAN, mark the padding bytes, if any, to catch // // memory errors. @@ -125,14 +146,13 @@ // // const char* c_str() const { // // Equivalent to reinterpret_cast(p.get() + sizeof(size_t)). -// // The argument in Partial(1) specifies that we have size_t[1] in front -// // of the characters. -// return L::Partial(1).Pointer(p_.get()); +// return L::Partial().Pointer(p_.get()); // } // // private: -// // Our heap allocation contains a size_t followed by an array of chars. -// using L = Layout; +// // Our heap allocation contains a single size_t followed by an array of +// // chars. +// using L = Layout::WithStaticSizes<1>; // std::unique_ptr p_; // }; // @@ -146,11 +166,12 @@ // // The interface exported by this file consists of: // - class `Layout<>` and its public members. -// - The public members of class `internal_layout::LayoutImpl<>`. That class -// isn't intended to be used directly, and its name and template parameter -// list are internal implementation details, but the class itself provides -// most of the functionality in this file. See comments on its members for -// detailed documentation. +// - The public members of classes `internal_layout::LayoutWithStaticSizes<>` +// and `internal_layout::LayoutImpl<>`. Those classes aren't intended to be +// used directly, and their name and template parameter list are internal +// implementation details, but the classes themselves provide most of the +// functionality in this file. See comments on their members for detailed +// documentation. // // `Layout::Partial(count1,..., countm)` (where `m` <= `n`) returns a // `LayoutImpl<>` object. `Layout layout(count1,..., countn)` @@ -164,7 +185,7 @@ #include #include -#include +#include #include #include #include @@ -210,9 +231,6 @@ struct NotAligned> { template using IntToSize = size_t; -template -using TypeToSize = size_t; - template struct Type : NotAligned { using type = T; @@ -309,7 +327,8 @@ using IsLegalElementType = std::integral_constant< !std::is_volatile::type>::value && adl_barrier::IsPow2(AlignOf::value)>; -template +template class LayoutImpl; // Public base class of `Layout` and the result type of `Layout::Partial()`. @@ -317,31 +336,49 @@ class LayoutImpl; // `Elements...` contains all template arguments of `Layout` that created this // instance. // -// `SizeSeq...` is `[0, NumSizes)` where `NumSizes` is the number of arguments -// passed to `Layout::Partial()` or `Layout::Layout()`. +// `StaticSizeSeq...` is an index_sequence containing the sizes specified at +// compile-time. +// +// `RuntimeSizeSeq...` is `[0, NumRuntimeSizes)`, where `NumRuntimeSizes` is the +// number of arguments passed to `Layout::Partial()` or `Layout::Layout()`. +// +// `SizeSeq...` is `[0, NumSizes)` where `NumSizes` is `NumRuntimeSizes` plus +// the number of sizes in `StaticSizeSeq`. // // `OffsetSeq...` is `[0, NumOffsets)` where `NumOffsets` is // `Min(sizeof...(Elements), NumSizes + 1)` (the number of arrays for which we // can compute offsets). -template -class LayoutImpl, absl::index_sequence, - absl::index_sequence> { +template +class LayoutImpl< + std::tuple, absl::index_sequence, + absl::index_sequence, absl::index_sequence, + absl::index_sequence> { private: static_assert(sizeof...(Elements) > 0, "At least one field is required"); static_assert(absl::conjunction...>::value, "Invalid element type (see IsLegalElementType)"); + static_assert(sizeof...(StaticSizeSeq) <= sizeof...(Elements), + "Too many static sizes specified"); enum { NumTypes = sizeof...(Elements), + NumStaticSizes = sizeof...(StaticSizeSeq), + NumRuntimeSizes = sizeof...(RuntimeSizeSeq), NumSizes = sizeof...(SizeSeq), NumOffsets = sizeof...(OffsetSeq), }; // These are guaranteed by `Layout`. + static_assert(NumStaticSizes + NumRuntimeSizes == NumSizes, "Internal error"); + static_assert(NumSizes <= NumTypes, "Internal error"); static_assert(NumOffsets == adl_barrier::Min(NumTypes, NumSizes + 1), "Internal error"); static_assert(NumTypes > 0, "Internal error"); + static constexpr std::array kStaticSizes = { + StaticSizeSeq...}; + // Returns the index of `T` in `Elements...`. Results in a compilation error // if `Elements...` doesn't contain exactly one instance of `T`. template @@ -364,7 +401,7 @@ class LayoutImpl, absl::index_sequence, template using ElementType = typename std::tuple_element::type; - constexpr explicit LayoutImpl(IntToSize... sizes) + constexpr explicit LayoutImpl(IntToSize... sizes) : size_{sizes...} {} // Alignment of the layout, equal to the strictest alignment of all elements. @@ -390,7 +427,7 @@ class LayoutImpl, absl::index_sequence, constexpr size_t Offset() const { static_assert(N < NumOffsets, "Index out of bounds"); return adl_barrier::Align( - Offset() + SizeOf>::value * size_[N - 1], + Offset() + SizeOf>::value * Size(), ElementAlignment::value); } @@ -412,8 +449,7 @@ class LayoutImpl, absl::index_sequence, return {{Offset()...}}; } - // The number of elements in the Nth array. This is the Nth argument of - // `Layout::Partial()` or `Layout::Layout()` (zero-based). + // The number of elements in the Nth array (zero-based). // // // int[3], 4 bytes of padding, double[4]. // Layout x(3, 4); @@ -421,10 +457,15 @@ class LayoutImpl, absl::index_sequence, // assert(x.Size<1>() == 4); // // Requires: `N < NumSizes`. - template + template = 0> + constexpr size_t Size() const { + return kStaticSizes[N]; + } + + template = NumStaticSizes)> = 0> constexpr size_t Size() const { static_assert(N < NumSizes, "Index out of bounds"); - return size_[N]; + return size_[N - NumStaticSizes]; } // The number of elements in the array with the specified element type. @@ -579,7 +620,7 @@ class LayoutImpl, absl::index_sequence, constexpr size_t AllocSize() const { static_assert(NumTypes == NumSizes, "You must specify sizes of all fields"); return Offset() + - SizeOf>::value * size_[NumTypes - 1]; + SizeOf>::value * Size(); } // If built with --config=asan, poisons padding bytes (if any) in the @@ -603,7 +644,7 @@ class LayoutImpl, absl::index_sequence, // The `if` is an optimization. It doesn't affect the observable behaviour. if (ElementAlignment::value % ElementAlignment::value) { size_t start = - Offset() + SizeOf>::value * size_[N - 1]; + Offset() + SizeOf>::value * Size(); ASAN_POISON_MEMORY_REGION(p + start, Offset() - start); } #endif @@ -632,47 +673,66 @@ class LayoutImpl, absl::index_sequence, adl_barrier::TypeName>()...}; std::string res = absl::StrCat("@0", types[0], "(", sizes[0], ")"); for (size_t i = 0; i != NumOffsets - 1; ++i) { - absl::StrAppend(&res, "[", size_[i], "]; @", offsets[i + 1], types[i + 1], - "(", sizes[i + 1], ")"); + absl::StrAppend(&res, "[", DebugSize(i), "]; @", offsets[i + 1], + types[i + 1], "(", sizes[i + 1], ")"); } // NumSizes is a constant that may be zero. Some compilers cannot see that // inside the if statement "size_[NumSizes - 1]" must be valid. int last = static_cast(NumSizes) - 1; if (NumTypes == NumSizes && last >= 0) { - absl::StrAppend(&res, "[", size_[last], "]"); + absl::StrAppend(&res, "[", DebugSize(static_cast(last)), "]"); } return res; } private: + size_t DebugSize(size_t n) const { + if (n < NumStaticSizes) { + return kStaticSizes[n]; + } else { + return size_[n - NumStaticSizes]; + } + } + // Arguments of `Layout::Partial()` or `Layout::Layout()`. - size_t size_[NumSizes > 0 ? NumSizes : 1]; + size_t size_[NumRuntimeSizes > 0 ? NumRuntimeSizes : 1]; }; -template -using LayoutType = LayoutImpl< - std::tuple, absl::make_index_sequence, - absl::make_index_sequence>; +// Defining a constexpr static class member variable is redundant and deprecated +// in C++17, but required in C++14. +template +constexpr std::array LayoutImpl< + std::tuple, absl::index_sequence, + absl::index_sequence, absl::index_sequence, + absl::index_sequence>::kStaticSizes; -} // namespace internal_layout +template +using LayoutType = LayoutImpl< + std::tuple, StaticSizeSeq, + absl::make_index_sequence, + absl::make_index_sequence, + absl::make_index_sequence>; + +template +class LayoutWithStaticSizes + : public LayoutType { + private: + using Super = + LayoutType; -// Descriptor of arrays of various types and sizes laid out in memory one after -// another. See the top of the file for documentation. -// -// Check out the public API of internal_layout::LayoutImpl above. The type is -// internal to the library but its methods are public, and they are inherited -// by `Layout`. -template -class Layout : public internal_layout::LayoutType { public: - static_assert(sizeof...(Ts) > 0, "At least one field is required"); - static_assert( - absl::conjunction...>::value, - "Invalid element type (see IsLegalElementType)"); - // The result type of `Partial()` with `NumSizes` arguments. template - using PartialType = internal_layout::LayoutType; + using PartialType = + internal_layout::LayoutType; // `Layout` knows the element types of the arrays we want to lay out in // memory but not the number of elements in each array. @@ -698,14 +758,18 @@ class Layout : public internal_layout::LayoutType { // Note: The sizes of the arrays must be specified in number of elements, // not in bytes. // - // Requires: `sizeof...(Sizes) <= sizeof...(Ts)`. + // Requires: `sizeof...(Sizes) + NumStaticSizes <= sizeof...(Ts)`. // Requires: all arguments are convertible to `size_t`. template static constexpr PartialType Partial(Sizes&&... sizes) { - static_assert(sizeof...(Sizes) <= sizeof...(Ts), ""); - return PartialType(std::forward(sizes)...); + static_assert(sizeof...(Sizes) + StaticSizeSeq::size() <= sizeof...(Ts), + ""); + return PartialType( + static_cast(std::forward(sizes))...); } + // Inherit LayoutType's constructor. + // // Creates a layout with the sizes of all arrays specified. If you know // only the sizes of the first N arrays (where N can be zero), you can use // `Partial()` defined above. The constructor is essentially equivalent to @@ -714,8 +778,69 @@ class Layout : public internal_layout::LayoutType { // // Note: The sizes of the arrays must be specified in number of elements, // not in bytes. - constexpr explicit Layout(internal_layout::TypeToSize... sizes) - : internal_layout::LayoutType(sizes...) {} + // + // Implementation note: we do this via a `using` declaration instead of + // defining our own explicit constructor because the signature of LayoutType's + // constructor depends on RuntimeSizeSeq, which we don't have access to here. + // If we defined our own constructor here, it would have to use a parameter + // pack and then cast the arguments to size_t when calling the superclass + // constructor, similar to what Partial() does. But that would suffer from the + // same problem that Partial() has, which is that the parameter types are + // inferred from the arguments, which may be signed types, which must then be + // cast to size_t. This can lead to negative values being silently (i.e. with + // no compiler warnings) cast to an unsigned type. Having a constructor with + // size_t parameters helps the compiler generate better warnings about + // potential bad casts, while avoiding false warnings when positive literal + // arguments are used. If an argument is a positive literal integer (e.g. + // `1`), the compiler will understand that it can be safely converted to + // size_t, and hence not generate a warning. But if a negative literal (e.g. + // `-1`) or a variable with signed type is used, then it can generate a + // warning about a potentially unsafe implicit cast. It would be great if we + // could do this for Partial() too, but unfortunately as of C++23 there seems + // to be no way to define a function with a variable number of paramters of a + // certain type, a.k.a. homogenous function parameter packs. So we're forced + // to choose between explicitly casting the arguments to size_t, which + // suppresses all warnings, even potentially valid ones, or implicitly casting + // them to size_t, which generates bogus warnings whenever literal arguments + // are used, even if they're positive. + using Super::Super; +}; + +} // namespace internal_layout + +// Descriptor of arrays of various types and sizes laid out in memory one after +// another. See the top of the file for documentation. +// +// Check out the public API of internal_layout::LayoutWithStaticSizes and +// internal_layout::LayoutImpl above. Those types are internal to the library +// but their methods are public, and they are inherited by `Layout`. +template +class Layout : public internal_layout::LayoutWithStaticSizes< + absl::make_index_sequence<0>, Ts...> { + private: + using Super = + internal_layout::LayoutWithStaticSizes, + Ts...>; + + public: + // If you know the sizes of some or all of the arrays at compile time, you can + // use `WithStaticSizes` or `WithStaticSizeSequence` to create a `Layout` type + // with those sizes baked in. This can help the compiler generate optimal code + // for calculating array offsets and AllocSize(). + // + // Like `Partial()`, the N sizes you specify are for the first N arrays, and + // they specify the number of elements in each array, not the number of bytes. + template + using WithStaticSizeSequence = + internal_layout::LayoutWithStaticSizes; + + template + using WithStaticSizes = + WithStaticSizeSequence>; + + // Inherit LayoutWithStaticSizes's constructor, which requires you to specify + // all the array sizes. + using Super::Super; }; } // namespace container_internal diff --git a/absl/container/internal/layout_benchmark.cc b/absl/container/internal/layout_benchmark.cc index 3af35e33..a8fbfa7b 100644 --- a/absl/container/internal/layout_benchmark.cc +++ b/absl/container/internal/layout_benchmark.cc @@ -15,6 +15,9 @@ // Every benchmark should have the same performance as the corresponding // headroom benchmark. +#include +#include + #include "absl/base/internal/raw_logging.h" #include "absl/container/internal/layout.h" #include "benchmark/benchmark.h" @@ -28,6 +31,8 @@ using ::benchmark::DoNotOptimize; using Int128 = int64_t[2]; +constexpr size_t MyAlign(size_t n, size_t m) { return (n + m - 1) & ~(m - 1); } + // This benchmark provides the upper bound on performance for BM_OffsetConstant. template void BM_OffsetConstantHeadroom(benchmark::State& state) { @@ -36,6 +41,15 @@ void BM_OffsetConstantHeadroom(benchmark::State& state) { } } +template +void BM_OffsetConstantStatic(benchmark::State& state) { + using L = typename Layout::template WithStaticSizes<3, 5, 7>; + ABSL_RAW_CHECK(L::Partial().template Offset<3>() == Offset, "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(L::Partial().template Offset<3>()); + } +} + template void BM_OffsetConstant(benchmark::State& state) { using L = Layout; @@ -46,14 +60,75 @@ void BM_OffsetConstant(benchmark::State& state) { } } +template +void BM_OffsetConstantIndirect(benchmark::State& state) { + using L = Layout; + auto p = L::Partial(3, 5, 7); + ABSL_RAW_CHECK(p.template Offset<3>() == Offset, "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(p); + DoNotOptimize(p.template Offset<3>()); + } +} + +template +size_t PartialOffset(size_t k); + +template <> +size_t PartialOffset(size_t k) { + constexpr size_t o = MyAlign(MyAlign(3 * 1, 2) + 5 * 2, 4); + // return Align(o + k * 4, 8); + return (o + k * 4 + 7) & ~7U; +} + +template <> +size_t PartialOffset(size_t k) { + // No alignment is necessary. + return 3 * 16 + 5 * 4 + k * 2; +} + +// This benchmark provides the upper bound on performance for BM_OffsetVariable. +template +void BM_OffsetPartialHeadroom(benchmark::State& state) { + size_t k = 7; + ABSL_RAW_CHECK(PartialOffset(k) == Offset, "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(k); + DoNotOptimize(PartialOffset(k)); + } +} + +template +void BM_OffsetPartialStatic(benchmark::State& state) { + using L = typename Layout::template WithStaticSizes<3, 5>; + size_t k = 7; + ABSL_RAW_CHECK(L::Partial(k).template Offset<3>() == Offset, + "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(k); + DoNotOptimize(L::Partial(k).template Offset<3>()); + } +} + +template +void BM_OffsetPartial(benchmark::State& state) { + using L = Layout; + size_t k = 7; + ABSL_RAW_CHECK(L::Partial(3, 5, k).template Offset<3>() == Offset, + "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(k); + DoNotOptimize(L::Partial(3, 5, k).template Offset<3>()); + } +} + template size_t VariableOffset(size_t n, size_t m, size_t k); template <> size_t VariableOffset(size_t n, size_t m, size_t k) { - auto Align = [](size_t n, size_t m) { return (n + m - 1) & ~(m - 1); }; - return Align(Align(Align(n * 1, 2) + m * 2, 4) + k * 4, 8); + return MyAlign(MyAlign(MyAlign(n * 1, 2) + m * 2, 4) + k * 4, 8); } template <> @@ -94,6 +169,75 @@ void BM_OffsetVariable(benchmark::State& state) { } } +template +size_t AllocSize(size_t x); + +template <> +size_t AllocSize(size_t x) { + constexpr size_t o = + Layout::Partial(3, 5, 7) + .template Offset(); + return o + sizeof(Int128) * x; +} + +template <> +size_t AllocSize(size_t x) { + constexpr size_t o = + Layout::Partial(3, 5, 7) + .template Offset(); + return o + sizeof(int8_t) * x; +} + +// This benchmark provides the upper bound on performance for BM_AllocSize +template +void BM_AllocSizeHeadroom(benchmark::State& state) { + size_t x = 9; + ABSL_RAW_CHECK(AllocSize(x) == Size, "Invalid size"); + for (auto _ : state) { + DoNotOptimize(x); + DoNotOptimize(AllocSize(x)); + } +} + +template +void BM_AllocSizeStatic(benchmark::State& state) { + using L = typename Layout::template WithStaticSizes<3, 5, 7>; + size_t x = 9; + ABSL_RAW_CHECK(L(x).AllocSize() == Size, "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(x); + DoNotOptimize(L(x).AllocSize()); + } +} + +template +void BM_AllocSize(benchmark::State& state) { + using L = Layout; + size_t n = 3; + size_t m = 5; + size_t k = 7; + size_t x = 9; + ABSL_RAW_CHECK(L(n, m, k, x).AllocSize() == Size, "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(n); + DoNotOptimize(m); + DoNotOptimize(k); + DoNotOptimize(x); + DoNotOptimize(L(n, m, k, x).AllocSize()); + } +} + +template +void BM_AllocSizeIndirect(benchmark::State& state) { + using L = Layout; + auto l = L(3, 5, 7, 9); + ABSL_RAW_CHECK(l.AllocSize() == Size, "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(l); + DoNotOptimize(l.AllocSize()); + } +} + // Run all benchmarks in two modes: // // Layout with padding: int8_t[3], int16_t[5], int32_t[7], Int128[?]. @@ -106,16 +250,46 @@ void BM_OffsetVariable(benchmark::State& state) { OFFSET_BENCHMARK(BM_OffsetConstantHeadroom, 48, int8_t, int16_t, int32_t, Int128); +OFFSET_BENCHMARK(BM_OffsetConstantStatic, 48, int8_t, int16_t, int32_t, Int128); OFFSET_BENCHMARK(BM_OffsetConstant, 48, int8_t, int16_t, int32_t, Int128); +OFFSET_BENCHMARK(BM_OffsetConstantIndirect, 48, int8_t, int16_t, int32_t, + Int128); + OFFSET_BENCHMARK(BM_OffsetConstantHeadroom, 82, Int128, int32_t, int16_t, int8_t); +OFFSET_BENCHMARK(BM_OffsetConstantStatic, 82, Int128, int32_t, int16_t, int8_t); OFFSET_BENCHMARK(BM_OffsetConstant, 82, Int128, int32_t, int16_t, int8_t); +OFFSET_BENCHMARK(BM_OffsetConstantIndirect, 82, Int128, int32_t, int16_t, + int8_t); + +OFFSET_BENCHMARK(BM_OffsetPartialHeadroom, 48, int8_t, int16_t, int32_t, + Int128); +OFFSET_BENCHMARK(BM_OffsetPartialStatic, 48, int8_t, int16_t, int32_t, Int128); +OFFSET_BENCHMARK(BM_OffsetPartial, 48, int8_t, int16_t, int32_t, Int128); + +OFFSET_BENCHMARK(BM_OffsetPartialHeadroom, 82, Int128, int32_t, int16_t, + int8_t); +OFFSET_BENCHMARK(BM_OffsetPartialStatic, 82, Int128, int32_t, int16_t, int8_t); +OFFSET_BENCHMARK(BM_OffsetPartial, 82, Int128, int32_t, int16_t, int8_t); + OFFSET_BENCHMARK(BM_OffsetVariableHeadroom, 48, int8_t, int16_t, int32_t, Int128); OFFSET_BENCHMARK(BM_OffsetVariable, 48, int8_t, int16_t, int32_t, Int128); + OFFSET_BENCHMARK(BM_OffsetVariableHeadroom, 82, Int128, int32_t, int16_t, int8_t); OFFSET_BENCHMARK(BM_OffsetVariable, 82, Int128, int32_t, int16_t, int8_t); + +OFFSET_BENCHMARK(BM_AllocSizeHeadroom, 192, int8_t, int16_t, int32_t, Int128); +OFFSET_BENCHMARK(BM_AllocSizeStatic, 192, int8_t, int16_t, int32_t, Int128); +OFFSET_BENCHMARK(BM_AllocSize, 192, int8_t, int16_t, int32_t, Int128); +OFFSET_BENCHMARK(BM_AllocSizeIndirect, 192, int8_t, int16_t, int32_t, Int128); + +OFFSET_BENCHMARK(BM_AllocSizeHeadroom, 91, Int128, int32_t, int16_t, int8_t); +OFFSET_BENCHMARK(BM_AllocSizeStatic, 91, Int128, int32_t, int16_t, int8_t); +OFFSET_BENCHMARK(BM_AllocSize, 91, Int128, int32_t, int16_t, int8_t); +OFFSET_BENCHMARK(BM_AllocSizeIndirect, 91, Int128, int32_t, int16_t, int8_t); + } // namespace } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/internal/layout_test.cc b/absl/container/internal/layout_test.cc index ae55cf7e..47fc9f33 100644 --- a/absl/container/internal/layout_test.cc +++ b/absl/container/internal/layout_test.cc @@ -68,9 +68,7 @@ struct alignas(8) Int128 { // int64_t is *not* 8-byte aligned on all platforms! struct alignas(8) Int64 { int64_t a; - friend bool operator==(Int64 lhs, Int64 rhs) { - return lhs.a == rhs.a; - } + friend bool operator==(Int64 lhs, Int64 rhs) { return lhs.a == rhs.a; } }; // Properties of types that this test relies on. @@ -271,6 +269,35 @@ TEST(Layout, Offsets) { } } +TEST(Layout, StaticOffsets) { + using L = Layout; + { + using SL = L::WithStaticSizes<>; + EXPECT_THAT(SL::Partial().Offsets(), ElementsAre(0)); + EXPECT_THAT(SL::Partial(5).Offsets(), ElementsAre(0, 8)); + EXPECT_THAT(SL::Partial(5, 3, 1).Offsets(), ElementsAre(0, 8, 24)); + EXPECT_THAT(SL(5, 3, 1).Offsets(), ElementsAre(0, 8, 24)); + } + { + using SL = L::WithStaticSizes<5>; + EXPECT_THAT(SL::Partial().Offsets(), ElementsAre(0, 8)); + EXPECT_THAT(SL::Partial(3).Offsets(), ElementsAre(0, 8, 24)); + EXPECT_THAT(SL::Partial(3, 1).Offsets(), ElementsAre(0, 8, 24)); + EXPECT_THAT(SL(3, 1).Offsets(), ElementsAre(0, 8, 24)); + } + { + using SL = L::WithStaticSizes<5, 3>; + EXPECT_THAT(SL::Partial().Offsets(), ElementsAre(0, 8, 24)); + EXPECT_THAT(SL::Partial(1).Offsets(), ElementsAre(0, 8, 24)); + EXPECT_THAT(SL(1).Offsets(), ElementsAre(0, 8, 24)); + } + { + using SL = L::WithStaticSizes<5, 3, 1>; + EXPECT_THAT(SL::Partial().Offsets(), ElementsAre(0, 8, 24)); + EXPECT_THAT(SL().Offsets(), ElementsAre(0, 8, 24)); + } +} + TEST(Layout, AllocSize) { { using L = Layout; @@ -295,6 +322,30 @@ TEST(Layout, AllocSize) { } } +TEST(Layout, StaticAllocSize) { + using L = Layout; + { + using SL = L::WithStaticSizes<>; + EXPECT_EQ(136, SL::Partial(3, 5, 7).AllocSize()); + EXPECT_EQ(136, SL(3, 5, 7).AllocSize()); + } + { + using SL = L::WithStaticSizes<3>; + EXPECT_EQ(136, SL::Partial(5, 7).AllocSize()); + EXPECT_EQ(136, SL(5, 7).AllocSize()); + } + { + using SL = L::WithStaticSizes<3, 5>; + EXPECT_EQ(136, SL::Partial(7).AllocSize()); + EXPECT_EQ(136, SL(7).AllocSize()); + } + { + using SL = L::WithStaticSizes<3, 5, 7>; + EXPECT_EQ(136, SL::Partial().AllocSize()); + EXPECT_EQ(136, SL().AllocSize()); + } +} + TEST(Layout, SizeByIndex) { { using L = Layout; @@ -370,6 +421,27 @@ TEST(Layout, Sizes) { } } +TEST(Layout, StaticSize) { + using L = Layout; + { + using SL = L::WithStaticSizes<>; + EXPECT_THAT(SL::Partial().Sizes(), ElementsAre()); + EXPECT_THAT(SL::Partial(3).Size<0>(), 3); + EXPECT_THAT(SL::Partial(3).Size(), 3); + EXPECT_THAT(SL::Partial(3).Sizes(), ElementsAre(3)); + EXPECT_THAT(SL::Partial(3, 5, 7).Size<0>(), 3); + EXPECT_THAT(SL::Partial(3, 5, 7).Size(), 3); + EXPECT_THAT(SL::Partial(3, 5, 7).Size<2>(), 7); + EXPECT_THAT(SL::Partial(3, 5, 7).Size(), 7); + EXPECT_THAT(SL::Partial(3, 5, 7).Sizes(), ElementsAre(3, 5, 7)); + EXPECT_THAT(SL(3, 5, 7).Size<0>(), 3); + EXPECT_THAT(SL(3, 5, 7).Size(), 3); + EXPECT_THAT(SL(3, 5, 7).Size<2>(), 7); + EXPECT_THAT(SL(3, 5, 7).Size(), 7); + EXPECT_THAT(SL(3, 5, 7).Sizes(), ElementsAre(3, 5, 7)); + } +} + TEST(Layout, PointerByIndex) { alignas(max_align_t) const unsigned char p[100] = {0}; { @@ -720,6 +792,61 @@ TEST(Layout, MutablePointers) { } } +TEST(Layout, StaticPointers) { + alignas(max_align_t) const unsigned char p[100] = {0}; + using L = Layout; + { + const auto x = L::WithStaticSizes<>::Partial(); + EXPECT_EQ(std::make_tuple(x.Pointer<0>(p)), + Type>(x.Pointers(p))); + } + { + const auto x = L::WithStaticSizes<>::Partial(1); + EXPECT_EQ(std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p)), + (Type>(x.Pointers(p)))); + } + { + const auto x = L::WithStaticSizes<1>::Partial(); + EXPECT_EQ(std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p)), + (Type>(x.Pointers(p)))); + } + { + const auto x = L::WithStaticSizes<>::Partial(1, 2, 3); + EXPECT_EQ( + std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p), x.Pointer<2>(p)), + (Type>( + x.Pointers(p)))); + } + { + const auto x = L::WithStaticSizes<1>::Partial(2, 3); + EXPECT_EQ( + std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p), x.Pointer<2>(p)), + (Type>( + x.Pointers(p)))); + } + { + const auto x = L::WithStaticSizes<1, 2>::Partial(3); + EXPECT_EQ( + std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p), x.Pointer<2>(p)), + (Type>( + x.Pointers(p)))); + } + { + const auto x = L::WithStaticSizes<1, 2, 3>::Partial(); + EXPECT_EQ( + std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p), x.Pointer<2>(p)), + (Type>( + x.Pointers(p)))); + } + { + const L::WithStaticSizes<1, 2, 3> x; + EXPECT_EQ( + std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p), x.Pointer<2>(p)), + (Type>( + x.Pointers(p)))); + } +} + TEST(Layout, SliceByIndexSize) { alignas(max_align_t) const unsigned char p[100] = {0}; { @@ -769,7 +896,6 @@ TEST(Layout, SliceByTypeSize) { EXPECT_EQ(7, L(3, 5, 7).Slice(p).size()); } } - TEST(Layout, MutableSliceByIndexSize) { alignas(max_align_t) unsigned char p[100] = {0}; { @@ -820,6 +946,39 @@ TEST(Layout, MutableSliceByTypeSize) { } } +TEST(Layout, StaticSliceSize) { + alignas(max_align_t) const unsigned char cp[100] = {0}; + alignas(max_align_t) unsigned char p[100] = {0}; + using L = Layout; + using SL = L::WithStaticSizes<3, 5>; + + EXPECT_EQ(3, SL::Partial().Slice<0>(cp).size()); + EXPECT_EQ(3, SL::Partial().Slice(cp).size()); + EXPECT_EQ(3, SL::Partial(7).Slice<0>(cp).size()); + EXPECT_EQ(3, SL::Partial(7).Slice(cp).size()); + + EXPECT_EQ(5, SL::Partial().Slice<1>(cp).size()); + EXPECT_EQ(5, SL::Partial().Slice(cp).size()); + EXPECT_EQ(5, SL::Partial(7).Slice<1>(cp).size()); + EXPECT_EQ(5, SL::Partial(7).Slice(cp).size()); + + EXPECT_EQ(7, SL::Partial(7).Slice<2>(cp).size()); + EXPECT_EQ(7, SL::Partial(7).Slice(cp).size()); + + EXPECT_EQ(3, SL::Partial().Slice<0>(p).size()); + EXPECT_EQ(3, SL::Partial().Slice(p).size()); + EXPECT_EQ(3, SL::Partial(7).Slice<0>(p).size()); + EXPECT_EQ(3, SL::Partial(7).Slice(p).size()); + + EXPECT_EQ(5, SL::Partial().Slice<1>(p).size()); + EXPECT_EQ(5, SL::Partial().Slice(p).size()); + EXPECT_EQ(5, SL::Partial(7).Slice<1>(p).size()); + EXPECT_EQ(5, SL::Partial(7).Slice(p).size()); + + EXPECT_EQ(7, SL::Partial(7).Slice<2>(p).size()); + EXPECT_EQ(7, SL::Partial(7).Slice(p).size()); +} + TEST(Layout, SliceByIndexData) { alignas(max_align_t) const unsigned char p[100] = {0}; { @@ -1230,6 +1389,39 @@ TEST(Layout, MutableSliceByTypeData) { } } +TEST(Layout, StaticSliceData) { + alignas(max_align_t) const unsigned char cp[100] = {0}; + alignas(max_align_t) unsigned char p[100] = {0}; + using L = Layout; + using SL = L::WithStaticSizes<3, 5>; + + EXPECT_EQ(0, Distance(cp, SL::Partial().Slice<0>(cp).data())); + EXPECT_EQ(0, Distance(cp, SL::Partial().Slice(cp).data())); + EXPECT_EQ(0, Distance(cp, SL::Partial(7).Slice<0>(cp).data())); + EXPECT_EQ(0, Distance(cp, SL::Partial(7).Slice(cp).data())); + + EXPECT_EQ(4, Distance(cp, SL::Partial().Slice<1>(cp).data())); + EXPECT_EQ(4, Distance(cp, SL::Partial().Slice(cp).data())); + EXPECT_EQ(4, Distance(cp, SL::Partial(7).Slice<1>(cp).data())); + EXPECT_EQ(4, Distance(cp, SL::Partial(7).Slice(cp).data())); + + EXPECT_EQ(24, Distance(cp, SL::Partial(7).Slice<2>(cp).data())); + EXPECT_EQ(24, Distance(cp, SL::Partial(7).Slice(cp).data())); + + EXPECT_EQ(0, Distance(p, SL::Partial().Slice<0>(p).data())); + EXPECT_EQ(0, Distance(p, SL::Partial().Slice(p).data())); + EXPECT_EQ(0, Distance(p, SL::Partial(7).Slice<0>(p).data())); + EXPECT_EQ(0, Distance(p, SL::Partial(7).Slice(p).data())); + + EXPECT_EQ(4, Distance(p, SL::Partial().Slice<1>(p).data())); + EXPECT_EQ(4, Distance(p, SL::Partial().Slice(p).data())); + EXPECT_EQ(4, Distance(p, SL::Partial(7).Slice<1>(p).data())); + EXPECT_EQ(4, Distance(p, SL::Partial(7).Slice(p).data())); + + EXPECT_EQ(24, Distance(p, SL::Partial(7).Slice<2>(p).data())); + EXPECT_EQ(24, Distance(p, SL::Partial(7).Slice(p).data())); +} + MATCHER_P(IsSameSlice, slice, "") { return arg.size() == slice.size() && arg.data() == slice.data(); } @@ -1339,6 +1531,43 @@ TEST(Layout, MutableSlices) { } } +TEST(Layout, StaticSlices) { + alignas(max_align_t) const unsigned char cp[100] = {0}; + alignas(max_align_t) unsigned char p[100] = {0}; + using SL = Layout::WithStaticSizes<1, 2>; + { + const auto x = SL::Partial(); + EXPECT_THAT( + (Type, Span>>( + x.Slices(cp))), + Tuple(IsSameSlice(x.Slice<0>(cp)), IsSameSlice(x.Slice<1>(cp)))); + EXPECT_THAT((Type, Span>>(x.Slices(p))), + Tuple(IsSameSlice(x.Slice<0>(p)), IsSameSlice(x.Slice<1>(p)))); + } + { + const auto x = SL::Partial(3); + EXPECT_THAT((Type, Span, + Span>>(x.Slices(cp))), + Tuple(IsSameSlice(x.Slice<0>(cp)), IsSameSlice(x.Slice<1>(cp)), + IsSameSlice(x.Slice<2>(cp)))); + EXPECT_THAT((Type, Span, Span>>( + x.Slices(p))), + Tuple(IsSameSlice(x.Slice<0>(p)), IsSameSlice(x.Slice<1>(p)), + IsSameSlice(x.Slice<2>(p)))); + } + { + const SL x(3); + EXPECT_THAT((Type, Span, + Span>>(x.Slices(cp))), + Tuple(IsSameSlice(x.Slice<0>(cp)), IsSameSlice(x.Slice<1>(cp)), + IsSameSlice(x.Slice<2>(cp)))); + EXPECT_THAT((Type, Span, Span>>( + x.Slices(p))), + Tuple(IsSameSlice(x.Slice<0>(p)), IsSameSlice(x.Slice<1>(p)), + IsSameSlice(x.Slice<2>(p)))); + } +} + TEST(Layout, UnalignedTypes) { constexpr Layout x(1, 2, 3); alignas(max_align_t) unsigned char p[x.AllocSize() + 1]; @@ -1377,6 +1606,36 @@ TEST(Layout, Alignment) { static_assert(Layout::Alignment() == 8, ""); static_assert(Layout::Alignment() == 8, ""); static_assert(Layout::Alignment() == 8, ""); + static_assert(Layout::Alignment() == 8, ""); + static_assert( + Layout>::WithStaticSizes<>::Alignment() == 64, ""); + static_assert( + Layout>::WithStaticSizes<2>::Alignment() == 64, ""); +} + +TEST(Layout, StaticAlignment) { + static_assert(Layout::WithStaticSizes<>::Alignment() == 1, ""); + static_assert(Layout::WithStaticSizes<0>::Alignment() == 1, ""); + static_assert(Layout::WithStaticSizes<7>::Alignment() == 1, ""); + static_assert(Layout::WithStaticSizes<>::Alignment() == 4, ""); + static_assert(Layout::WithStaticSizes<0>::Alignment() == 4, ""); + static_assert(Layout::WithStaticSizes<3>::Alignment() == 4, ""); + static_assert( + Layout>::WithStaticSizes<>::Alignment() == 64, ""); + static_assert( + Layout>::WithStaticSizes<0>::Alignment() == 64, ""); + static_assert( + Layout>::WithStaticSizes<2>::Alignment() == 64, ""); + static_assert( + Layout::WithStaticSizes<>::Alignment() == 8, ""); + static_assert( + Layout::WithStaticSizes<0, 0, 0>::Alignment() == + 8, + ""); + static_assert( + Layout::WithStaticSizes<1, 1, 1>::Alignment() == + 8, + ""); } TEST(Layout, ConstexprPartial) { @@ -1384,6 +1643,15 @@ TEST(Layout, ConstexprPartial) { constexpr Layout> x(1, 3); static_assert(x.Partial(1).template Offset<1>() == 2 * M, ""); } + +TEST(Layout, StaticConstexpr) { + constexpr size_t M = alignof(max_align_t); + using L = Layout>; + using SL = L::WithStaticSizes<1, 3>; + constexpr SL x; + static_assert(x.Offset<1>() == 2 * M, ""); +} + // [from, to) struct Region { size_t from; @@ -1458,6 +1726,41 @@ TEST(Layout, PoisonPadding) { } } +TEST(Layout, StaticPoisonPadding) { + using L = Layout; + using SL = L::WithStaticSizes<1, 2>; + + constexpr size_t n = L::Partial(1, 2, 3, 4).AllocSize(); + { + constexpr auto x = SL::Partial(); + alignas(max_align_t) const unsigned char c[n] = {}; + x.PoisonPadding(c); + EXPECT_EQ(x.Slices(c), x.Slices(c)); + ExpectPoisoned(c, {{1, 8}}); + } + { + constexpr auto x = SL::Partial(3); + alignas(max_align_t) const unsigned char c[n] = {}; + x.PoisonPadding(c); + EXPECT_EQ(x.Slices(c), x.Slices(c)); + ExpectPoisoned(c, {{1, 8}, {36, 40}}); + } + { + constexpr auto x = SL::Partial(3, 4); + alignas(max_align_t) const unsigned char c[n] = {}; + x.PoisonPadding(c); + EXPECT_EQ(x.Slices(c), x.Slices(c)); + ExpectPoisoned(c, {{1, 8}, {36, 40}}); + } + { + constexpr SL x(3, 4); + alignas(max_align_t) const unsigned char c[n] = {}; + x.PoisonPadding(c); + EXPECT_EQ(x.Slices(c), x.Slices(c)); + ExpectPoisoned(c, {{1, 8}, {36, 40}}); + } +} + TEST(Layout, DebugString) { { constexpr auto x = Layout::Partial(); @@ -1500,6 +1803,62 @@ TEST(Layout, DebugString) { } } +TEST(Layout, StaticDebugString) { + { + constexpr auto x = + Layout::WithStaticSizes<>::Partial(); + EXPECT_EQ("@0(1)", x.DebugString()); + } + { + constexpr auto x = + Layout::WithStaticSizes<>::Partial(1); + EXPECT_EQ("@0(1)[1]; @4(4)", x.DebugString()); + } + { + constexpr auto x = + Layout::WithStaticSizes<1>::Partial(); + EXPECT_EQ("@0(1)[1]; @4(4)", x.DebugString()); + } + { + constexpr auto x = + Layout::WithStaticSizes<>::Partial(1, + 2); + EXPECT_EQ("@0(1)[1]; @4(4)[2]; @12(1)", + x.DebugString()); + } + { + constexpr auto x = + Layout::WithStaticSizes<1>::Partial(2); + EXPECT_EQ("@0(1)[1]; @4(4)[2]; @12(1)", + x.DebugString()); + } + { + constexpr auto x = Layout::WithStaticSizes<1, 2>::Partial(); + EXPECT_EQ("@0(1)[1]; @4(4)[2]; @12(1)", + x.DebugString()); + } + { + constexpr auto x = Layout::WithStaticSizes<1, 2, 3, 4>::Partial(); + EXPECT_EQ( + "@0(1)[1]; @4(4)[2]; @12(1)[3]; " + "@16" + + Int128::Name() + "(16)[4]", + x.DebugString()); + } + { + constexpr Layout::WithStaticSizes<1, 2, 3, + 4> + x; + EXPECT_EQ( + "@0(1)[1]; @4(4)[2]; @12(1)[3]; " + "@16" + + Int128::Name() + "(16)[4]", + x.DebugString()); + } +} + TEST(Layout, CharTypes) { constexpr Layout x(1); alignas(max_align_t) char c[x.AllocSize()] = {}; @@ -1638,6 +1997,35 @@ TEST(CompactString, Works) { EXPECT_STREQ("hello", s.c_str()); } +// Same as the previous CompactString example, except we set the first array +// size to 1 statically, since we know it is always 1. This allows us to compute +// the offset of the character array at compile time. +class StaticCompactString { + public: + StaticCompactString(const char* s = "") { // NOLINT + const size_t size = strlen(s); + const SL layout(size + 1); + p_.reset(new unsigned char[layout.AllocSize()]); + layout.PoisonPadding(p_.get()); + *layout.Pointer(p_.get()) = size; + memcpy(layout.Pointer(p_.get()), s, size + 1); + } + + size_t size() const { return *SL::Partial().Pointer(p_.get()); } + + const char* c_str() const { return SL::Partial().Pointer(p_.get()); } + + private: + using SL = Layout::WithStaticSizes<1>; + std::unique_ptr p_; +}; + +TEST(StaticCompactString, Works) { + StaticCompactString s = "hello"; + EXPECT_EQ(5, s.size()); + EXPECT_STREQ("hello", s.c_str()); +} + } // namespace example } // namespace -- cgit v1.2.3 From 5e61a28e48e324b5cac13a6a1e3b6ba423ec2a6f Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Mon, 18 Mar 2024 11:34:18 -0700 Subject: Fix a typo in a comment. PiperOrigin-RevId: 616895950 Change-Id: I9dc9099e779df4b692496aa5ee5573ef0e7fd826 --- absl/container/internal/layout.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/layout.h b/absl/container/internal/layout.h index 7f2b83a2..72c14112 100644 --- a/absl/container/internal/layout.h +++ b/absl/container/internal/layout.h @@ -797,8 +797,8 @@ class LayoutWithStaticSizes // `-1`) or a variable with signed type is used, then it can generate a // warning about a potentially unsafe implicit cast. It would be great if we // could do this for Partial() too, but unfortunately as of C++23 there seems - // to be no way to define a function with a variable number of paramters of a - // certain type, a.k.a. homogenous function parameter packs. So we're forced + // to be no way to define a function with a variable number of parameters of a + // certain type, a.k.a. homogeneous function parameter packs. So we're forced // to choose between explicitly casting the arguments to size_t, which // suppresses all warnings, even potentially valid ones, or implicitly casting // them to size_t, which generates bogus warnings whenever literal arguments -- cgit v1.2.3 From d53b1e6619e11ebb51e1a29d0847166685307be0 Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Mon, 18 Mar 2024 13:40:13 -0700 Subject: Add template keyword to example comment for Layout::WithStaticSizes. Without this keyword, we can sometimes get cryptic compilation failures such as "error: expected ';' after alias declaration". PiperOrigin-RevId: 616933517 Change-Id: I2209f3899a4ac03c031217cec67de25bd376d355 --- absl/container/internal/layout.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/layout.h b/absl/container/internal/layout.h index 72c14112..d42651a2 100644 --- a/absl/container/internal/layout.h +++ b/absl/container/internal/layout.h @@ -85,9 +85,9 @@ // generate optimal code. To help the compiler do that in more cases, you can // specify the fixed sizes using `WithStaticSizes`. This ensures that all // computations that can be performed at compile time are indeed performed at -// compile time. E.g.: +// compile time. Note that sometimes the `template` keyword is needed. E.g.: // -// using SL = L::WithStaticSizes<1, 1>; +// using SL = L::template WithStaticSizes<1, 1>; // // void Use(unsigned char* p) { // // First, extract N and M. -- cgit v1.2.3 From 1980d7b98a0cae64432ccf9c4afebda5e21b2c5e Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Tue, 19 Mar 2024 12:07:34 -0700 Subject: Do hashtablez sampling on the first insertion into an empty SOO hashtable. When sampling triggers, we skip SOO and allocate a backing array. We must do this because the HashtablezInfoHandle is part of the heap allocation (along with the control bytes and slots). By default, we sample 1 in ~1024 hashtables when sampling is enabled. This will impact performance because (a) we won't benefit from SOO so we would have worse data locality (more cache/TLB misses), and (b) the backing array capacity will be 3 instead of 1 so (b.1) we skip the rehash after the second insertion and (b.2) we potentially waste up to two slots worth of memory. We also add an soo_capacity field to HashtablezInfo to allow for distinguishing which sampled tables may otherwise have been SOO - this will allow us to know approximately what fraction of tables are in SOO mode. PiperOrigin-RevId: 617252334 Change-Id: Ib48b7a4870bd74ea3ba923ed8f350a3b75dbb7d3 --- absl/container/BUILD.bazel | 2 + absl/container/CMakeLists.txt | 2 + absl/container/internal/hashtablez_sampler.cc | 20 ++- absl/container/internal/hashtablez_sampler.h | 25 ++- absl/container/internal/hashtablez_sampler_test.cc | 49 ++++-- absl/container/internal/raw_hash_set.h | 158 +++++++++++++----- absl/container/internal/raw_hash_set_test.cc | 184 ++++++++++++++++++--- 7 files changed, 348 insertions(+), 92 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/BUILD.bazel b/absl/container/BUILD.bazel index 0de45263..c1df8d83 100644 --- a/absl/container/BUILD.bazel +++ b/absl/container/BUILD.bazel @@ -701,8 +701,10 @@ cc_test( "//absl/base:config", "//absl/base:core_headers", "//absl/base:prefetch", + "//absl/functional:function_ref", "//absl/hash", "//absl/log", + "//absl/log:check", "//absl/memory", "//absl/meta:type_traits", "//absl/strings", diff --git a/absl/container/CMakeLists.txt b/absl/container/CMakeLists.txt index 4b08e6a3..7fa2e914 100644 --- a/absl/container/CMakeLists.txt +++ b/absl/container/CMakeLists.txt @@ -753,11 +753,13 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::base + absl::check absl::config absl::container_memory absl::core_headers absl::flat_hash_map absl::flat_hash_set + absl::function_ref absl::hash absl::hash_function_defaults absl::hash_policy_testing diff --git a/absl/container/internal/hashtablez_sampler.cc b/absl/container/internal/hashtablez_sampler.cc index b21beef0..e17ba14d 100644 --- a/absl/container/internal/hashtablez_sampler.cc +++ b/absl/container/internal/hashtablez_sampler.cc @@ -18,13 +18,18 @@ #include #include #include +#include +#include #include #include #include "absl/base/attributes.h" #include "absl/base/config.h" +#include "absl/base/internal/per_thread_tls.h" #include "absl/base/internal/raw_logging.h" +#include "absl/base/macros.h" #include "absl/base/no_destructor.h" +#include "absl/base/optimization.h" #include "absl/debugging/stacktrace.h" #include "absl/memory/memory.h" #include "absl/profiling/internal/exponential_biased.h" @@ -73,7 +78,8 @@ HashtablezInfo::HashtablezInfo() = default; HashtablezInfo::~HashtablezInfo() = default; void HashtablezInfo::PrepareForSampling(int64_t stride, - size_t inline_element_size_value) { + size_t inline_element_size_value, + uint16_t soo_capacity_value) { capacity.store(0, std::memory_order_relaxed); size.store(0, std::memory_order_relaxed); num_erases.store(0, std::memory_order_relaxed); @@ -93,6 +99,7 @@ void HashtablezInfo::PrepareForSampling(int64_t stride, depth = absl::GetStackTrace(stack, HashtablezInfo::kMaxStackDepth, /* skip_count= */ 0); inline_element_size = inline_element_size_value; + soo_capacity = soo_capacity_value; } static bool ShouldForceSampling() { @@ -116,12 +123,12 @@ static bool ShouldForceSampling() { } HashtablezInfo* SampleSlow(SamplingState& next_sample, - size_t inline_element_size) { + size_t inline_element_size, uint16_t soo_capacity) { if (ABSL_PREDICT_FALSE(ShouldForceSampling())) { next_sample.next_sample = 1; const int64_t old_stride = exchange(next_sample.sample_stride, 1); - HashtablezInfo* result = - GlobalHashtablezSampler().Register(old_stride, inline_element_size); + HashtablezInfo* result = GlobalHashtablezSampler().Register( + old_stride, inline_element_size, soo_capacity); return result; } @@ -151,10 +158,11 @@ HashtablezInfo* SampleSlow(SamplingState& next_sample, // that case. if (first) { if (ABSL_PREDICT_TRUE(--next_sample.next_sample > 0)) return nullptr; - return SampleSlow(next_sample, inline_element_size); + return SampleSlow(next_sample, inline_element_size, soo_capacity); } - return GlobalHashtablezSampler().Register(old_stride, inline_element_size); + return GlobalHashtablezSampler().Register(old_stride, inline_element_size, + soo_capacity); #endif } diff --git a/absl/container/internal/hashtablez_sampler.h b/absl/container/internal/hashtablez_sampler.h index e41ee2d7..4e11968d 100644 --- a/absl/container/internal/hashtablez_sampler.h +++ b/absl/container/internal/hashtablez_sampler.h @@ -40,15 +40,20 @@ #define ABSL_CONTAINER_INTERNAL_HASHTABLEZ_SAMPLER_H_ #include +#include +#include #include #include #include +#include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/internal/per_thread_tls.h" #include "absl/base/optimization.h" +#include "absl/base/thread_annotations.h" #include "absl/profiling/internal/sample_recorder.h" #include "absl/synchronization/mutex.h" +#include "absl/time/time.h" #include "absl/utility/utility.h" namespace absl { @@ -67,7 +72,8 @@ struct HashtablezInfo : public profiling_internal::Sample { // Puts the object into a clean state, fills in the logically `const` members, // blocking for any readers that are currently sampling the object. - void PrepareForSampling(int64_t stride, size_t inline_element_size_value) + void PrepareForSampling(int64_t stride, size_t inline_element_size_value, + uint16_t soo_capacity_value) ABSL_EXCLUSIVE_LOCKS_REQUIRED(init_mu); // These fields are mutated by the various Record* APIs and need to be @@ -91,8 +97,13 @@ struct HashtablezInfo : public profiling_internal::Sample { static constexpr int kMaxStackDepth = 64; absl::Time create_time; int32_t depth; + // The SOO capacity for this table in elements (not bytes). Note that sampled + // tables are never SOO because we need to store the infoz handle on the heap. + // Tables that would be SOO if not sampled should have: soo_capacity > 0 && + // size <= soo_capacity && max_reserve <= soo_capacity. + uint16_t soo_capacity; void* stack[kMaxStackDepth]; - size_t inline_element_size; // How big is the slot? + size_t inline_element_size; // How big is the slot in bytes? }; void RecordRehashSlow(HashtablezInfo* info, size_t total_probe_length); @@ -117,7 +128,7 @@ struct SamplingState { }; HashtablezInfo* SampleSlow(SamplingState& next_sample, - size_t inline_element_size); + size_t inline_element_size, uint16_t soo_capacity); void UnsampleSlow(HashtablezInfo* info); #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) @@ -204,16 +215,16 @@ class HashtablezInfoHandle { extern ABSL_PER_THREAD_TLS_KEYWORD SamplingState global_next_sample; #endif // defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) -// Returns an RAII sampling handle that manages registration and unregistation -// with the global sampler. +// Returns a sampling handle. inline HashtablezInfoHandle Sample( - size_t inline_element_size ABSL_ATTRIBUTE_UNUSED) { + ABSL_ATTRIBUTE_UNUSED size_t inline_element_size, + ABSL_ATTRIBUTE_UNUSED uint16_t soo_capacity) { #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) if (ABSL_PREDICT_TRUE(--global_next_sample.next_sample > 0)) { return HashtablezInfoHandle(nullptr); } return HashtablezInfoHandle( - SampleSlow(global_next_sample, inline_element_size)); + SampleSlow(global_next_sample, inline_element_size, soo_capacity)); #else return HashtablezInfoHandle(nullptr); #endif // !ABSL_PER_THREAD_TLS diff --git a/absl/container/internal/hashtablez_sampler_test.cc b/absl/container/internal/hashtablez_sampler_test.cc index 8ebb08da..a80f530d 100644 --- a/absl/container/internal/hashtablez_sampler_test.cc +++ b/absl/container/internal/hashtablez_sampler_test.cc @@ -15,8 +15,12 @@ #include "absl/container/internal/hashtablez_sampler.h" #include +#include +#include +#include #include #include +#include #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -67,7 +71,7 @@ std::vector GetSizes(HashtablezSampler* s) { HashtablezInfo* Register(HashtablezSampler* s, size_t size) { const int64_t test_stride = 123; const size_t test_element_size = 17; - auto* info = s->Register(test_stride, test_element_size); + auto* info = s->Register(test_stride, test_element_size, /*soo_capacity=*/0); assert(info != nullptr); info->size.store(size); return info; @@ -79,7 +83,8 @@ TEST(HashtablezInfoTest, PrepareForSampling) { const size_t test_element_size = 17; HashtablezInfo info; absl::MutexLock l(&info.init_mu); - info.PrepareForSampling(test_stride, test_element_size); + info.PrepareForSampling(test_stride, test_element_size, + /*soo_capacity_value=*/1); EXPECT_EQ(info.capacity.load(), 0); EXPECT_EQ(info.size.load(), 0); @@ -94,6 +99,7 @@ TEST(HashtablezInfoTest, PrepareForSampling) { EXPECT_GE(info.create_time, test_start); EXPECT_EQ(info.weight, test_stride); EXPECT_EQ(info.inline_element_size, test_element_size); + EXPECT_EQ(info.soo_capacity, 1); info.capacity.store(1, std::memory_order_relaxed); info.size.store(1, std::memory_order_relaxed); @@ -106,7 +112,8 @@ TEST(HashtablezInfoTest, PrepareForSampling) { info.max_reserve.store(1, std::memory_order_relaxed); info.create_time = test_start - absl::Hours(20); - info.PrepareForSampling(test_stride * 2, test_element_size); + info.PrepareForSampling(test_stride * 2, test_element_size, + /*soo_capacity_value=*/0); EXPECT_EQ(info.capacity.load(), 0); EXPECT_EQ(info.size.load(), 0); EXPECT_EQ(info.num_erases.load(), 0); @@ -120,6 +127,7 @@ TEST(HashtablezInfoTest, PrepareForSampling) { EXPECT_EQ(info.weight, 2 * test_stride); EXPECT_EQ(info.inline_element_size, test_element_size); EXPECT_GE(info.create_time, test_start); + EXPECT_EQ(info.soo_capacity, 0); } TEST(HashtablezInfoTest, RecordStorageChanged) { @@ -127,7 +135,8 @@ TEST(HashtablezInfoTest, RecordStorageChanged) { absl::MutexLock l(&info.init_mu); const int64_t test_stride = 21; const size_t test_element_size = 19; - info.PrepareForSampling(test_stride, test_element_size); + info.PrepareForSampling(test_stride, test_element_size, + /*soo_capacity_value=*/0); RecordStorageChangedSlow(&info, 17, 47); EXPECT_EQ(info.size.load(), 17); EXPECT_EQ(info.capacity.load(), 47); @@ -141,7 +150,8 @@ TEST(HashtablezInfoTest, RecordInsert) { absl::MutexLock l(&info.init_mu); const int64_t test_stride = 25; const size_t test_element_size = 23; - info.PrepareForSampling(test_stride, test_element_size); + info.PrepareForSampling(test_stride, test_element_size, + /*soo_capacity_value=*/0); EXPECT_EQ(info.max_probe_length.load(), 0); RecordInsertSlow(&info, 0x0000FF00, 6 * kProbeLength); EXPECT_EQ(info.max_probe_length.load(), 6); @@ -165,7 +175,8 @@ TEST(HashtablezInfoTest, RecordErase) { const size_t test_element_size = 29; HashtablezInfo info; absl::MutexLock l(&info.init_mu); - info.PrepareForSampling(test_stride, test_element_size); + info.PrepareForSampling(test_stride, test_element_size, + /*soo_capacity_value=*/1); EXPECT_EQ(info.num_erases.load(), 0); EXPECT_EQ(info.size.load(), 0); RecordInsertSlow(&info, 0x0000FF00, 6 * kProbeLength); @@ -174,6 +185,7 @@ TEST(HashtablezInfoTest, RecordErase) { EXPECT_EQ(info.size.load(), 0); EXPECT_EQ(info.num_erases.load(), 1); EXPECT_EQ(info.inline_element_size, test_element_size); + EXPECT_EQ(info.soo_capacity, 1); } TEST(HashtablezInfoTest, RecordRehash) { @@ -181,7 +193,8 @@ TEST(HashtablezInfoTest, RecordRehash) { const size_t test_element_size = 31; HashtablezInfo info; absl::MutexLock l(&info.init_mu); - info.PrepareForSampling(test_stride, test_element_size); + info.PrepareForSampling(test_stride, test_element_size, + /*soo_capacity_value=*/0); RecordInsertSlow(&info, 0x1, 0); RecordInsertSlow(&info, 0x2, kProbeLength); RecordInsertSlow(&info, 0x4, kProbeLength); @@ -201,6 +214,7 @@ TEST(HashtablezInfoTest, RecordRehash) { EXPECT_EQ(info.num_erases.load(), 0); EXPECT_EQ(info.num_rehashes.load(), 1); EXPECT_EQ(info.inline_element_size, test_element_size); + EXPECT_EQ(info.soo_capacity, 0); } TEST(HashtablezInfoTest, RecordReservation) { @@ -208,7 +222,8 @@ TEST(HashtablezInfoTest, RecordReservation) { absl::MutexLock l(&info.init_mu); const int64_t test_stride = 35; const size_t test_element_size = 33; - info.PrepareForSampling(test_stride, test_element_size); + info.PrepareForSampling(test_stride, test_element_size, + /*soo_capacity_value=*/0); RecordReservationSlow(&info, 3); EXPECT_EQ(info.max_reserve.load(), 3); @@ -229,7 +244,8 @@ TEST(HashtablezSamplerTest, SmallSampleParameter) { for (int i = 0; i < 1000; ++i) { SamplingState next_sample = {0, 0}; - HashtablezInfo* sample = SampleSlow(next_sample, test_element_size); + HashtablezInfo* sample = SampleSlow(next_sample, test_element_size, + /*soo_capacity=*/0); EXPECT_GT(next_sample.next_sample, 0); EXPECT_EQ(next_sample.next_sample, next_sample.sample_stride); EXPECT_NE(sample, nullptr); @@ -244,7 +260,8 @@ TEST(HashtablezSamplerTest, LargeSampleParameter) { for (int i = 0; i < 1000; ++i) { SamplingState next_sample = {0, 0}; - HashtablezInfo* sample = SampleSlow(next_sample, test_element_size); + HashtablezInfo* sample = SampleSlow(next_sample, test_element_size, + /*soo_capacity=*/0); EXPECT_GT(next_sample.next_sample, 0); EXPECT_EQ(next_sample.next_sample, next_sample.sample_stride); EXPECT_NE(sample, nullptr); @@ -260,7 +277,8 @@ TEST(HashtablezSamplerTest, Sample) { int64_t total = 0; double sample_rate = 0.0; for (int i = 0; i < 1000000; ++i) { - HashtablezInfoHandle h = Sample(test_element_size); + HashtablezInfoHandle h = Sample(test_element_size, + /*soo_capacity=*/0); ++total; if (h.IsSampled()) { ++num_sampled; @@ -275,7 +293,8 @@ TEST(HashtablezSamplerTest, Handle) { auto& sampler = GlobalHashtablezSampler(); const int64_t test_stride = 41; const size_t test_element_size = 39; - HashtablezInfoHandle h(sampler.Register(test_stride, test_element_size)); + HashtablezInfoHandle h(sampler.Register(test_stride, test_element_size, + /*soo_capacity=*/0)); auto* info = HashtablezInfoHandlePeer::GetInfo(&h); info->hashes_bitwise_and.store(0x12345678, std::memory_order_relaxed); @@ -358,11 +377,13 @@ TEST(HashtablezSamplerTest, MultiThreaded) { std::vector infoz; while (!stop.HasBeenNotified()) { if (infoz.empty()) { - infoz.push_back(sampler.Register(sampling_stride, elt_size)); + infoz.push_back(sampler.Register(sampling_stride, elt_size, + /*soo_capacity=*/0)); } switch (std::uniform_int_distribution<>(0, 2)(gen)) { case 0: { - infoz.push_back(sampler.Register(sampling_stride, elt_size)); + infoz.push_back(sampler.Register(sampling_stride, elt_size, + /*soo_capacity=*/0)); break; } case 1: { diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 258458b0..58188444 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -1775,17 +1775,48 @@ ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots( } } +template +constexpr bool ShouldSampleHashtablezInfo() { + // Folks with custom allocators often make unwarranted assumptions about the + // behavior of their classes vis-a-vis trivial destructability and what + // calls they will or won't make. Avoid sampling for people with custom + // allocators to get us out of this mess. This is not a hard guarantee but + // a workaround while we plan the exact guarantee we want to provide. + return std::is_same>::value; +} + +template +HashtablezInfoHandle SampleHashtablezInfo(size_t sizeof_slot, + size_t old_capacity, bool was_soo, + HashtablezInfoHandle forced_infoz, + CommonFields& c) { + if (forced_infoz.IsSampled()) return forced_infoz; + // In SOO, we sample on the first insertion so if this is an empty SOO case + // (e.g. when reserve is called), then we still need to sample. + if (kSooEnabled && was_soo && c.size() == 0) { + return Sample(sizeof_slot, SooCapacity()); + } + // For non-SOO cases, we sample whenever the capacity is increasing from zero + // to non-zero. + if (!kSooEnabled && old_capacity == 0) { + return Sample(sizeof_slot, 0); + } + return c.infoz(); +} + // Helper class to perform resize of the hash set. // // It contains special optimizations for small group resizes. // See GrowIntoSingleGroupShuffleControlBytes for details. class HashSetResizeHelper { public: - explicit HashSetResizeHelper(CommonFields& c, bool was_soo, bool had_soo_slot) + explicit HashSetResizeHelper(CommonFields& c, bool was_soo, bool had_soo_slot, + HashtablezInfoHandle forced_infoz) : old_capacity_(c.capacity()), had_infoz_(c.has_infoz()), was_soo_(was_soo), - had_soo_slot_(had_soo_slot) {} + had_soo_slot_(had_soo_slot), + forced_infoz_(forced_infoz) {} // Optimized for small groups version of `find_first_non_full`. // Beneficial only right after calling `raw_hash_set::resize`. @@ -1839,7 +1870,7 @@ class HashSetResizeHelper { // Reads `capacity` and updates all other fields based on the result of // the allocation. // - // It also may do the folowing actions: + // It also may do the following actions: // 1. initialize control bytes // 2. initialize slots // 3. deallocate old slots. @@ -1869,26 +1900,15 @@ class HashSetResizeHelper { // // Returns IsGrowingIntoSingleGroupApplicable result to avoid recomputation. template + bool SooEnabled, size_t AlignOfSlot> ABSL_ATTRIBUTE_NOINLINE bool InitializeSlots(CommonFields& c, Alloc alloc, ctrl_t soo_slot_h2) { assert(c.capacity()); - // Folks with custom allocators often make unwarranted assumptions about the - // behavior of their classes vis-a-vis trivial destructability and what - // calls they will or won't make. Avoid sampling for people with custom - // allocators to get us out of this mess. This is not a hard guarantee but - // a workaround while we plan the exact guarantee we want to provide. - const size_t sample_size = - (std::is_same>::value && - (was_soo_ || old_capacity_ == 0)) - ? SizeOfSlot - : 0; - // TODO(b/289225379): when SOO is enabled, we should still sample on first - // insertion and if Sample is non-null, then we should force a heap - // allocation. Note that we'll also have to force capacity of 3 so that - // is_soo() still works. HashtablezInfoHandle infoz = - sample_size > 0 ? Sample(sample_size) : c.infoz(); + ShouldSampleHashtablezInfo() + ? SampleHashtablezInfo(SizeOfSlot, old_capacity_, + was_soo_, forced_infoz_, c) + : HashtablezInfoHandle{}; const bool has_infoz = infoz.IsSampled(); RawHashSetLayout layout(c.capacity(), AlignOfSlot, has_infoz); @@ -1904,12 +1924,13 @@ class HashSetResizeHelper { const bool grow_single_group = IsGrowingIntoSingleGroupApplicable(old_capacity_, layout.capacity()); - if (was_soo_ && grow_single_group) { + if (SooEnabled && was_soo_ && grow_single_group) { InitControlBytesAfterSoo(c.control(), soo_slot_h2, layout.capacity()); if (TransferUsesMemcpy && had_soo_slot_) { TransferSlotAfterSoo(c, SizeOfSlot); } - } else if (old_capacity_ != 0 && grow_single_group) { + // SooEnabled implies that old_capacity_ != 0. + } else if ((SooEnabled || old_capacity_ != 0) && grow_single_group) { if (TransferUsesMemcpy) { GrowSizeIntoSingleGroupTransferable(c, SizeOfSlot); DeallocateOld(alloc, SizeOfSlot); @@ -1923,7 +1944,7 @@ class HashSetResizeHelper { c.set_has_infoz(has_infoz); if (has_infoz) { infoz.RecordStorageChanged(c.size(), layout.capacity()); - if (was_soo_ || grow_single_group || old_capacity_ == 0) { + if ((SooEnabled && was_soo_) || grow_single_group || old_capacity_ == 0) { infoz.RecordRehash(0); } c.set_infoz(infoz); @@ -2063,6 +2084,8 @@ class HashSetResizeHelper { bool had_infoz_; bool was_soo_; bool had_soo_slot_; + // Either null infoz or a pre-sampled forced infoz for SOO tables. + HashtablezInfoHandle forced_infoz_; }; inline void PrepareInsertCommon(CommonFields& common) { @@ -2545,6 +2568,8 @@ class raw_hash_set { assert(size == 1); common().set_full_soo(); emplace_at(soo_iterator(), *that.begin()); + const HashtablezInfoHandle infoz = try_sample_soo(); + if (infoz.IsSampled()) resize_with_soo_infoz(infoz); return; } assert(!that.is_soo()); @@ -2682,8 +2707,7 @@ class raw_hash_set { size_t size() const { return common().size(); } size_t capacity() const { const size_t cap = common().capacity(); - // Use local variables because compiler complains about using functions in - // assume. + // Compiler complains when using functions in assume so use local variables. ABSL_ATTRIBUTE_UNUSED static constexpr bool kEnabled = SooEnabled(); ABSL_ATTRIBUTE_UNUSED static constexpr size_t kCapacity = SooCapacity(); ABSL_ASSUME(!kEnabled || cap >= kCapacity); @@ -3051,7 +3075,17 @@ class raw_hash_set { SooEnabled()); return; } - if (SooEnabled() && size() <= SooCapacity()) { + if (fits_in_soo(size())) { + // When the table is already sampled, we keep it sampled. + if (infoz().IsSampled()) { + const size_t kInitialSampledCapacity = NextCapacity(SooCapacity()); + if (capacity() > kInitialSampledCapacity) { + resize(kInitialSampledCapacity); + } + // This asserts that we didn't lose sampling coverage in `resize`. + assert(infoz().IsSampled()); + return; + } alignas(slot_type) unsigned char slot_space[sizeof(slot_type)]; slot_type* tmp_slot = to_slot(slot_space); transfer(tmp_slot, begin().slot()); @@ -3322,6 +3356,15 @@ class raw_hash_set { } } + // Conditionally samples hashtablez for SOO tables. This should be called on + // insertion into an empty SOO table and in copy construction when the size + // can fit in SOO capacity. + inline HashtablezInfoHandle try_sample_soo() { + assert(is_soo()); + if (!ShouldSampleHashtablezInfo()) return HashtablezInfoHandle{}; + return Sample(sizeof(slot_type), SooCapacity()); + } + inline void destroy_slots() { assert(!is_soo()); if (PolicyTraits::template destroy_is_trivial()) return; @@ -3374,7 +3417,20 @@ class raw_hash_set { // HashSetResizeHelper::FindFirstNonFullAfterResize( // common(), old_capacity, hash) // can be called right after `resize`. - ABSL_ATTRIBUTE_NOINLINE void resize(size_t new_capacity) { + void resize(size_t new_capacity) { + resize_impl(new_capacity, HashtablezInfoHandle{}); + } + + // As above, except that we also accept a pre-sampled, forced infoz for + // SOO tables, since they need to switch from SOO to heap in order to + // store the infoz. + void resize_with_soo_infoz(HashtablezInfoHandle forced_infoz) { + assert(forced_infoz.IsSampled()); + resize_impl(NextCapacity(SooCapacity()), forced_infoz); + } + + ABSL_ATTRIBUTE_NOINLINE void resize_impl( + size_t new_capacity, HashtablezInfoHandle forced_infoz) { assert(IsValidCapacity(new_capacity)); assert(!fits_in_soo(new_capacity)); const bool was_soo = is_soo(); @@ -3382,7 +3438,8 @@ class raw_hash_set { const ctrl_t soo_slot_h2 = had_soo_slot ? static_cast(H2(hash_of(soo_slot()))) : ctrl_t::kEmpty; - HashSetResizeHelper resize_helper(common(), was_soo, had_soo_slot); + HashSetResizeHelper resize_helper(common(), was_soo, had_soo_slot, + forced_infoz); // Initialize HashSetResizeHelper::old_heap_or_soo_. We can't do this in // HashSetResizeHelper constructor because it can't transfer slots when // transfer_uses_memcpy is false. @@ -3400,7 +3457,7 @@ class raw_hash_set { const bool grow_single_group = resize_helper.InitializeSlots( + SooEnabled(), alignof(slot_type)>( common(), CharAlloc(alloc_ref()), soo_slot_h2); // In the SooEnabled() case, capacity is never 0 so we don't check. @@ -3621,26 +3678,29 @@ class raw_hash_set { return move_elements_allocs_unequal(std::move(that)); } - protected: - // Attempts to find `key` in the table; if it isn't found, returns an iterator - // where the value can be inserted into, with the control byte already set to - // `key`'s H2. Returns a bool indicating whether an insertion can take place. template - std::pair find_or_prepare_insert(const K& key) { - if (is_soo()) { - if (empty()) { + std::pair find_or_prepare_insert_soo(const K& key) { + if (empty()) { + const HashtablezInfoHandle infoz = try_sample_soo(); + if (infoz.IsSampled()) { + resize_with_soo_infoz(infoz); + } else { common().set_full_soo(); return {soo_iterator(), true}; } - if (PolicyTraits::apply(EqualElement{key, eq_ref()}, - PolicyTraits::element(soo_slot()))) { - return {soo_iterator(), false}; - } + } else if (PolicyTraits::apply(EqualElement{key, eq_ref()}, + PolicyTraits::element(soo_slot()))) { + return {soo_iterator(), false}; + } else { resize(NextCapacity(SooCapacity())); - const size_t index = - PrepareInsertAfterSoo(hash_ref()(key), sizeof(slot_type), common()); - return {iterator_at(index), true}; } + const size_t index = + PrepareInsertAfterSoo(hash_ref()(key), sizeof(slot_type), common()); + return {iterator_at(index), true}; + } + + template + std::pair find_or_prepare_insert_non_soo(const K& key) { prefetch_heap_block(); auto hash = hash_ref()(key); auto seq = probe(common(), hash); @@ -3660,6 +3720,16 @@ class raw_hash_set { return {iterator_at(prepare_insert(hash)), true}; } + protected: + // Attempts to find `key` in the table; if it isn't found, returns an iterator + // where the value can be inserted into, with the control byte already set to + // `key`'s H2. Returns a bool indicating whether an insertion can take place. + template + std::pair find_or_prepare_insert(const K& key) { + if (is_soo()) return find_or_prepare_insert_soo(key); + return find_or_prepare_insert_non_soo(key); + } + // Given the hash of a value not currently in the table, finds the next // viable slot index to insert it at. // @@ -3769,13 +3839,13 @@ class raw_hash_set { return static_cast(common().soo_data()); } const slot_type* soo_slot() const { - return reinterpret_cast(this)->soo_slot(); + return const_cast(this)->soo_slot(); } iterator soo_iterator() { return {SooControl(), soo_slot(), common().generation_ptr()}; } const_iterator soo_iterator() const { - return reinterpret_cast(this)->soo_iterator(); + return const_cast(this)->soo_iterator(); } HashtablezInfoHandle infoz() { assert(!is_soo()); diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 66baeb64..f00cef8c 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -52,7 +52,9 @@ #include "absl/container/internal/hashtable_debug.h" #include "absl/container/internal/hashtablez_sampler.h" #include "absl/container/internal/test_allocator.h" +#include "absl/functional/function_ref.h" #include "absl/hash/hash.h" +#include "absl/log/check.h" #include "absl/log/log.h" #include "absl/memory/memory.h" #include "absl/meta/type_traits.h" @@ -2035,6 +2037,9 @@ TYPED_TEST_P(SooTest, FindFullDeletedRegression) { } TYPED_TEST_P(SooTest, ReplacingDeletedSlotDoesNotRehash) { + // We need to disable hashtablez to avoid issues related to SOO and sampling. + SetHashtablezEnabled(false); + size_t n; { // Compute n such that n is the maximum number of elements before rehash. @@ -2388,6 +2393,9 @@ TYPED_TEST_P(SooTest, IterationOrderChangesOnRehash) { // Verify that pointers are invalidated as soon as a second element is inserted. // This prevents dependency on pointer stability on small tables. TYPED_TEST_P(SooTest, UnstablePointers) { + // We need to disable hashtablez to avoid issues related to SOO and sampling. + SetHashtablezEnabled(false); + TypeParam table; const auto addr = [&](int i) { @@ -2523,7 +2531,7 @@ TYPED_TEST_P(RawHashSamplerTest, Sample) { constexpr bool soo_enabled = std::is_same::value; // Enable the feature even if the prod default is off. SetHashtablezEnabled(true); - SetHashtablezSampleParameter(100); + SetHashtablezSampleParameter(100); // Sample ~1% of tables. auto& sampler = GlobalHashtablezSampler(); size_t start_size = 0; @@ -2557,25 +2565,26 @@ TYPED_TEST_P(RawHashSamplerTest, Sample) { absl::flat_hash_map observed_checksums; absl::flat_hash_map reservations; end_size += sampler.Iterate([&](const HashtablezInfo& info) { - if (preexisting_info.count(&info) == 0) { - observed_checksums[info.hashes_bitwise_xor.load( - std::memory_order_relaxed)]++; - reservations[info.max_reserve.load(std::memory_order_relaxed)]++; - } - EXPECT_EQ(info.inline_element_size, sizeof(typename TypeParam::value_type)); ++end_size; + if (preexisting_info.contains(&info)) return; + observed_checksums[info.hashes_bitwise_xor.load( + std::memory_order_relaxed)]++; + reservations[info.max_reserve.load(std::memory_order_relaxed)]++; + EXPECT_EQ(info.inline_element_size, sizeof(typename TypeParam::value_type)); + + if (soo_enabled) { + EXPECT_EQ(info.soo_capacity, SooCapacity()); + } else { + EXPECT_EQ(info.soo_capacity, 0); + } }); + // Expect that we sampled at the requested sampling rate of ~1%. EXPECT_NEAR((end_size - start_size) / static_cast(tables.size()), 0.01, 0.005); - if (soo_enabled) { - EXPECT_EQ(observed_checksums.size(), 9); - } else { - EXPECT_EQ(observed_checksums.size(), 5); - for (const auto& [_, count] : observed_checksums) { - EXPECT_NEAR((100 * count) / static_cast(tables.size()), 0.2, - 0.05); - } + EXPECT_EQ(observed_checksums.size(), 5); + for (const auto& [_, count] : observed_checksums) { + EXPECT_NEAR((100 * count) / static_cast(tables.size()), 0.2, 0.05); } EXPECT_EQ(reservations.size(), 10); @@ -2591,12 +2600,141 @@ TYPED_TEST_P(RawHashSamplerTest, Sample) { REGISTER_TYPED_TEST_SUITE_P(RawHashSamplerTest, Sample); using RawHashSamplerTestTypes = ::testing::Types; INSTANTIATE_TYPED_TEST_SUITE_P(My, RawHashSamplerTest, RawHashSamplerTestTypes); + +std::vector SampleSooMutation( + absl::FunctionRef mutate_table) { + // Enable the feature even if the prod default is off. + SetHashtablezEnabled(true); + SetHashtablezSampleParameter(100); // Sample ~1% of tables. + + auto& sampler = GlobalHashtablezSampler(); + size_t start_size = 0; + absl::flat_hash_set preexisting_info; + start_size += sampler.Iterate([&](const HashtablezInfo& info) { + preexisting_info.insert(&info); + ++start_size; + }); + + std::vector tables; + for (int i = 0; i < 1000000; ++i) { + tables.emplace_back(); + mutate_table(tables.back()); + } + size_t end_size = 0; + std::vector infos; + end_size += sampler.Iterate([&](const HashtablezInfo& info) { + ++end_size; + if (preexisting_info.contains(&info)) return; + infos.push_back(&info); + }); + + // Expect that we sampled at the requested sampling rate of ~1%. + EXPECT_NEAR((end_size - start_size) / static_cast(tables.size()), + 0.01, 0.005); + return infos; +} + +TEST(RawHashSamplerTest, SooTableInsertToEmpty) { + if (SooIntTable().capacity() != SooCapacity()) { + CHECK_LT(sizeof(void*), 8) << "missing SOO coverage"; + GTEST_SKIP() << "not SOO on this platform"; + } + std::vector infos = + SampleSooMutation([](SooIntTable& t) { t.insert(1); }); + + for (const HashtablezInfo* info : infos) { + ASSERT_EQ(info->inline_element_size, + sizeof(typename SooIntTable::value_type)); + ASSERT_EQ(info->soo_capacity, SooCapacity()); + ASSERT_EQ(info->capacity, NextCapacity(SooCapacity())); + ASSERT_EQ(info->size, 1); + ASSERT_EQ(info->max_reserve, 0); + ASSERT_EQ(info->num_erases, 0); + ASSERT_EQ(info->max_probe_length, 0); + ASSERT_EQ(info->total_probe_length, 0); + } +} + +TEST(RawHashSamplerTest, SooTableReserveToEmpty) { + if (SooIntTable().capacity() != SooCapacity()) { + CHECK_LT(sizeof(void*), 8) << "missing SOO coverage"; + GTEST_SKIP() << "not SOO on this platform"; + } + std::vector infos = + SampleSooMutation([](SooIntTable& t) { t.reserve(100); }); + + for (const HashtablezInfo* info : infos) { + ASSERT_EQ(info->inline_element_size, + sizeof(typename SooIntTable::value_type)); + ASSERT_EQ(info->soo_capacity, SooCapacity()); + ASSERT_GE(info->capacity, 100); + ASSERT_EQ(info->size, 0); + ASSERT_EQ(info->max_reserve, 100); + ASSERT_EQ(info->num_erases, 0); + ASSERT_EQ(info->max_probe_length, 0); + ASSERT_EQ(info->total_probe_length, 0); + } +} + +// This tests that reserve on a full SOO table doesn't incorrectly result in new +// (over-)sampling. +TEST(RawHashSamplerTest, SooTableReserveToFullSoo) { + if (SooIntTable().capacity() != SooCapacity()) { + CHECK_LT(sizeof(void*), 8) << "missing SOO coverage"; + GTEST_SKIP() << "not SOO on this platform"; + } + std::vector infos = + SampleSooMutation([](SooIntTable& t) { + t.insert(1); + t.reserve(100); + }); + + for (const HashtablezInfo* info : infos) { + ASSERT_EQ(info->inline_element_size, + sizeof(typename SooIntTable::value_type)); + ASSERT_EQ(info->soo_capacity, SooCapacity()); + ASSERT_GE(info->capacity, 100); + ASSERT_EQ(info->size, 1); + ASSERT_EQ(info->max_reserve, 100); + ASSERT_EQ(info->num_erases, 0); + ASSERT_EQ(info->max_probe_length, 0); + ASSERT_EQ(info->total_probe_length, 0); + } +} + +// This tests that rehash(0) on a sampled table with size that fits in SOO +// doesn't incorrectly result in losing sampling. +TEST(RawHashSamplerTest, SooTableRehashShrinkWhenSizeFitsInSoo) { + if (SooIntTable().capacity() != SooCapacity()) { + CHECK_LT(sizeof(void*), 8) << "missing SOO coverage"; + GTEST_SKIP() << "not SOO on this platform"; + } + std::vector infos = + SampleSooMutation([](SooIntTable& t) { + t.reserve(100); + t.insert(1); + EXPECT_GE(t.capacity(), 100); + t.rehash(0); + }); + + for (const HashtablezInfo* info : infos) { + ASSERT_EQ(info->inline_element_size, + sizeof(typename SooIntTable::value_type)); + ASSERT_EQ(info->soo_capacity, SooCapacity()); + ASSERT_EQ(info->capacity, NextCapacity(SooCapacity())); + ASSERT_EQ(info->size, 1); + ASSERT_EQ(info->max_reserve, 100); + ASSERT_EQ(info->num_erases, 0); + ASSERT_EQ(info->max_probe_length, 0); + ASSERT_EQ(info->total_probe_length, 0); + } +} #endif // ABSL_INTERNAL_HASHTABLEZ_SAMPLE TEST(RawHashSamplerTest, DoNotSampleCustomAllocators) { // Enable the feature even if the prod default is off. SetHashtablezEnabled(true); - SetHashtablezSampleParameter(100); + SetHashtablezSampleParameter(100); // Sample ~1% of tables. auto& sampler = GlobalHashtablezSampler(); size_t start_size = 0; @@ -2974,7 +3112,7 @@ TYPED_TEST_P(SooTable, Basic) { bool frozen = true; TypeParam t{FreezableAlloc(&frozen)}; if (t.capacity() != SooCapacity()) { - EXPECT_LT(sizeof(void*), 8); + CHECK_LT(sizeof(void*), 8) << "missing SOO coverage"; GTEST_SKIP() << "not SOO on this platform"; } @@ -3006,14 +3144,18 @@ using FreezableSooTableTypes = FreezableSizedValueSooTable<16>>; INSTANTIATE_TYPED_TEST_SUITE_P(My, SooTable, FreezableSooTableTypes); -TEST(Table, RehashToSoo) { +TEST(Table, RehashToSooUnsampled) { SooIntTable t; if (t.capacity() != SooCapacity()) { - EXPECT_LT(sizeof(void*), 8); + CHECK_LT(sizeof(void*), 8) << "missing SOO coverage"; GTEST_SKIP() << "not SOO on this platform"; } - t.reserve(10); + // We disable hashtablez sampling for this test to ensure that the table isn't + // sampled. When the table is sampled, it won't rehash down to SOO. + SetHashtablezEnabled(false); + + t.reserve(100); t.insert(0); EXPECT_EQ(*t.begin(), 0); @@ -3026,7 +3168,7 @@ TEST(Table, RehashToSoo) { EXPECT_EQ(t.find(1), t.end()); } -TEST(Table, ResizeToNonSoo) { +TEST(Table, ReserveToNonSoo) { for (int reserve_capacity : {8, 100000}) { SooIntTable t; t.insert(0); -- cgit v1.2.3 From 85166c912d2a8cfb4eafb78b66a37facf15ae2e5 Mon Sep 17 00:00:00 2001 From: Dino Radakovic Date: Wed, 20 Mar 2024 11:20:41 -0700 Subject: `layout_benchmark`: Replace leftover comment with intended call to MyAlign PiperOrigin-RevId: 617573381 Change-Id: I0ddab2ab7cf68651424b3cf385b484d27106dd59 --- absl/container/internal/layout_benchmark.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/layout_benchmark.cc b/absl/container/internal/layout_benchmark.cc index a8fbfa7b..d6f26697 100644 --- a/absl/container/internal/layout_benchmark.cc +++ b/absl/container/internal/layout_benchmark.cc @@ -77,8 +77,7 @@ size_t PartialOffset(size_t k); template <> size_t PartialOffset(size_t k) { constexpr size_t o = MyAlign(MyAlign(3 * 1, 2) + 5 * 2, 4); - // return Align(o + k * 4, 8); - return (o + k * 4 + 7) & ~7U; + return MyAlign(o + k * 4, 8); } template <> -- cgit v1.2.3 From 1cd7128b5697045562dc1bed4ad17c819d4ab166 Mon Sep 17 00:00:00 2001 From: Dino Radakovic Date: Thu, 21 Mar 2024 08:41:48 -0700 Subject: `layout`: Delete outdated comments about ElementType alias not being used because of MSVC Code below those comments does use ElementType. PiperOrigin-RevId: 617854731 Change-Id: I7996b8cbaa2fb65855a801f634a57d821408b1f3 --- absl/container/internal/layout.h | 6 ------ 1 file changed, 6 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/layout.h b/absl/container/internal/layout.h index d42651a2..384929af 100644 --- a/absl/container/internal/layout.h +++ b/absl/container/internal/layout.h @@ -542,9 +542,6 @@ class LayoutImpl< // std::tie(ints, doubles) = x.Pointers(p); // // Requires: `p` is aligned to `Alignment()`. - // - // Note: We're not using ElementType alias here because it does not compile - // under MSVC. template auto Pointers(Char* p) const { return std::tuple>*...>( @@ -599,9 +596,6 @@ class LayoutImpl< // // Requires: `p` is aligned to `Alignment()`. // - // Note: We're not using ElementType alias here because it does not compile - // under MSVC. - // // Note: We mark the parameter as unused because GCC detects it is not used // when `SizeSeq` is empty [-Werror=unused-but-set-parameter]. template -- cgit v1.2.3 From 5036f0b69ccfaa9f74a7895b50748247a7aee124 Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Thu, 21 Mar 2024 09:57:45 -0700 Subject: Use Layout::WithStaticSizes in btree. PiperOrigin-RevId: 617877687 Change-Id: I29c52f9288f099255c4adb7c1f9fa8831ac55b05 --- absl/container/internal/btree.h | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/btree.h b/absl/container/internal/btree.h index fd7860da..00a9cb13 100644 --- a/absl/container/internal/btree.h +++ b/absl/container/internal/btree.h @@ -580,14 +580,12 @@ class btree_node { using layout_type = absl::container_internal::Layout; + using leaf_layout_type = typename layout_type::template WithStaticSizes< + /*parent*/ 1, + /*generation*/ BtreeGenerationsEnabled() ? 1 : 0, + /*position, start, finish, max_count*/ 4>; constexpr static size_type SizeWithNSlots(size_type n) { - return layout_type( - /*parent*/ 1, - /*generation*/ BtreeGenerationsEnabled() ? 1 : 0, - /*position, start, finish, max_count*/ 4, - /*slots*/ n, - /*children*/ 0) - .AllocSize(); + return leaf_layout_type(/*slots*/ n, /*children*/ 0).AllocSize(); } // A lower bound for the overhead of fields other than slots in a leaf node. constexpr static size_type MinimumOverhead() { @@ -619,27 +617,22 @@ class btree_node { constexpr static size_type kNodeSlots = kNodeTargetSlots >= kMinNodeSlots ? kNodeTargetSlots : kMinNodeSlots; + using internal_layout_type = typename layout_type::template WithStaticSizes< + /*parent*/ 1, + /*generation*/ BtreeGenerationsEnabled() ? 1 : 0, + /*position, start, finish, max_count*/ 4, /*slots*/ kNodeSlots, + /*children*/ kNodeSlots + 1>; + // The node is internal (i.e. is not a leaf node) if and only if `max_count` // has this value. constexpr static field_type kInternalNodeMaxCount = 0; - constexpr static layout_type Layout(const size_type slot_count, - const size_type child_count) { - return layout_type( - /*parent*/ 1, - /*generation*/ BtreeGenerationsEnabled() ? 1 : 0, - /*position, start, finish, max_count*/ 4, - /*slots*/ slot_count, - /*children*/ child_count); - } // Leaves can have less than kNodeSlots values. - constexpr static layout_type LeafLayout( + constexpr static leaf_layout_type LeafLayout( const size_type slot_count = kNodeSlots) { - return Layout(slot_count, 0); - } - constexpr static layout_type InternalLayout() { - return Layout(kNodeSlots, kNodeSlots + 1); + return leaf_layout_type(slot_count, 0); } + constexpr static auto InternalLayout() { return internal_layout_type(); } constexpr static size_type LeafSize(const size_type slot_count = kNodeSlots) { return LeafLayout(slot_count).AllocSize(); } -- cgit v1.2.3 From 06e119066162d7dfa21c4481446e580772ff51f1 Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Mon, 25 Mar 2024 09:47:22 -0700 Subject: Fix ClangTidy warnings in btree.h. PiperOrigin-RevId: 618872032 Change-Id: I9fdfadff906494eb64cee976c02a1fff57923c79 --- absl/container/BUILD.bazel | 2 +- absl/container/CMakeLists.txt | 4 ++-- absl/container/internal/btree.h | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/BUILD.bazel b/absl/container/BUILD.bazel index c1df8d83..78ad60e0 100644 --- a/absl/container/BUILD.bazel +++ b/absl/container/BUILD.bazel @@ -1004,6 +1004,7 @@ cc_library( ":compressed_tuple", ":container_memory", ":layout", + "//absl/base:config", "//absl/base:core_headers", "//absl/base:raw_logging_internal", "//absl/base:throw_delegate", @@ -1012,7 +1013,6 @@ cc_library( "//absl/strings", "//absl/strings:cord", "//absl/types:compare", - "//absl/utility", ], ) diff --git a/absl/container/CMakeLists.txt b/absl/container/CMakeLists.txt index 7fa2e914..04055816 100644 --- a/absl/container/CMakeLists.txt +++ b/absl/container/CMakeLists.txt @@ -27,10 +27,11 @@ absl_cc_library( LINKOPTS ${ABSL_DEFAULT_LINKOPTS} DEPS - absl::container_common absl::common_policy_traits absl::compare absl::compressed_tuple + absl::config + absl::container_common absl::container_memory absl::cord absl::core_headers @@ -40,7 +41,6 @@ absl_cc_library( absl::strings absl::throw_delegate absl::type_traits - absl::utility ) # Internal-only target, do not depend on directly. diff --git a/absl/container/internal/btree.h b/absl/container/internal/btree.h index 00a9cb13..4ffc44e7 100644 --- a/absl/container/internal/btree.h +++ b/absl/container/internal/btree.h @@ -53,11 +53,11 @@ #include #include #include -#include #include #include #include +#include "absl/base/config.h" #include "absl/base/internal/raw_logging.h" #include "absl/base/macros.h" #include "absl/container/internal/common.h" @@ -70,7 +70,6 @@ #include "absl/strings/cord.h" #include "absl/strings/string_view.h" #include "absl/types/compare.h" -#include "absl/utility/utility.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -475,7 +474,7 @@ struct SearchResult { // useful information. template struct SearchResult { - SearchResult() {} + SearchResult() = default; explicit SearchResult(V v) : value(v) {} SearchResult(V v, MatchKind /*match*/) : value(v) {} -- cgit v1.2.3 From 48abb9fe0eeaf0149f0351acb00201f07e79f293 Mon Sep 17 00:00:00 2001 From: Chris Kennelly Date: Mon, 25 Mar 2024 13:16:00 -0700 Subject: Record sizeof(key_type), sizeof(value_type) in hashtable profiles. This can identify situations where flat_hash_* is suboptimal for large elements. PiperOrigin-RevId: 618937993 Change-Id: I2bde069bc3526b14ad1718ba6f50467002aeed16 --- absl/container/internal/hashtablez_sampler.cc | 14 +++- absl/container/internal/hashtablez_sampler.h | 13 +++- absl/container/internal/hashtablez_sampler_test.cc | 89 ++++++++++++++++++++-- absl/container/internal/raw_hash_set.h | 22 ++++-- absl/container/internal/raw_hash_set_test.cc | 2 + 5 files changed, 117 insertions(+), 23 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/hashtablez_sampler.cc b/absl/container/internal/hashtablez_sampler.cc index e17ba14d..fd21d966 100644 --- a/absl/container/internal/hashtablez_sampler.cc +++ b/absl/container/internal/hashtablez_sampler.cc @@ -79,6 +79,8 @@ HashtablezInfo::~HashtablezInfo() = default; void HashtablezInfo::PrepareForSampling(int64_t stride, size_t inline_element_size_value, + size_t key_size_value, + size_t value_size_value, uint16_t soo_capacity_value) { capacity.store(0, std::memory_order_relaxed); size.store(0, std::memory_order_relaxed); @@ -99,6 +101,8 @@ void HashtablezInfo::PrepareForSampling(int64_t stride, depth = absl::GetStackTrace(stack, HashtablezInfo::kMaxStackDepth, /* skip_count= */ 0); inline_element_size = inline_element_size_value; + key_size = key_size_value; + value_size = value_size_value; soo_capacity = soo_capacity_value; } @@ -123,12 +127,13 @@ static bool ShouldForceSampling() { } HashtablezInfo* SampleSlow(SamplingState& next_sample, - size_t inline_element_size, uint16_t soo_capacity) { + size_t inline_element_size, size_t key_size, + size_t value_size, uint16_t soo_capacity) { if (ABSL_PREDICT_FALSE(ShouldForceSampling())) { next_sample.next_sample = 1; const int64_t old_stride = exchange(next_sample.sample_stride, 1); HashtablezInfo* result = GlobalHashtablezSampler().Register( - old_stride, inline_element_size, soo_capacity); + old_stride, inline_element_size, key_size, value_size, soo_capacity); return result; } @@ -158,11 +163,12 @@ HashtablezInfo* SampleSlow(SamplingState& next_sample, // that case. if (first) { if (ABSL_PREDICT_TRUE(--next_sample.next_sample > 0)) return nullptr; - return SampleSlow(next_sample, inline_element_size, soo_capacity); + return SampleSlow(next_sample, inline_element_size, key_size, value_size, + soo_capacity); } return GlobalHashtablezSampler().Register(old_stride, inline_element_size, - soo_capacity); + key_size, value_size, soo_capacity); #endif } diff --git a/absl/container/internal/hashtablez_sampler.h b/absl/container/internal/hashtablez_sampler.h index 4e11968d..d74acf8c 100644 --- a/absl/container/internal/hashtablez_sampler.h +++ b/absl/container/internal/hashtablez_sampler.h @@ -73,6 +73,7 @@ struct HashtablezInfo : public profiling_internal::Sample { // Puts the object into a clean state, fills in the logically `const` members, // blocking for any readers that are currently sampling the object. void PrepareForSampling(int64_t stride, size_t inline_element_size_value, + size_t key_size, size_t value_size, uint16_t soo_capacity_value) ABSL_EXCLUSIVE_LOCKS_REQUIRED(init_mu); @@ -104,6 +105,8 @@ struct HashtablezInfo : public profiling_internal::Sample { uint16_t soo_capacity; void* stack[kMaxStackDepth]; size_t inline_element_size; // How big is the slot in bytes? + size_t key_size; // sizeof(key_type) + size_t value_size; // sizeof(value_type) }; void RecordRehashSlow(HashtablezInfo* info, size_t total_probe_length); @@ -128,7 +131,8 @@ struct SamplingState { }; HashtablezInfo* SampleSlow(SamplingState& next_sample, - size_t inline_element_size, uint16_t soo_capacity); + size_t inline_element_size, size_t key_size, + size_t value_size, uint16_t soo_capacity); void UnsampleSlow(HashtablezInfo* info); #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) @@ -218,13 +222,16 @@ extern ABSL_PER_THREAD_TLS_KEYWORD SamplingState global_next_sample; // Returns a sampling handle. inline HashtablezInfoHandle Sample( ABSL_ATTRIBUTE_UNUSED size_t inline_element_size, + ABSL_ATTRIBUTE_UNUSED size_t key_size, + ABSL_ATTRIBUTE_UNUSED size_t value_size, ABSL_ATTRIBUTE_UNUSED uint16_t soo_capacity) { #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) if (ABSL_PREDICT_TRUE(--global_next_sample.next_sample > 0)) { return HashtablezInfoHandle(nullptr); } - return HashtablezInfoHandle( - SampleSlow(global_next_sample, inline_element_size, soo_capacity)); + return HashtablezInfoHandle(SampleSlow(global_next_sample, + inline_element_size, key_size, + value_size, soo_capacity)); #else return HashtablezInfoHandle(nullptr); #endif // !ABSL_PER_THREAD_TLS diff --git a/absl/container/internal/hashtablez_sampler_test.cc b/absl/container/internal/hashtablez_sampler_test.cc index a80f530d..24d3bc48 100644 --- a/absl/container/internal/hashtablez_sampler_test.cc +++ b/absl/container/internal/hashtablez_sampler_test.cc @@ -71,7 +71,11 @@ std::vector GetSizes(HashtablezSampler* s) { HashtablezInfo* Register(HashtablezSampler* s, size_t size) { const int64_t test_stride = 123; const size_t test_element_size = 17; - auto* info = s->Register(test_stride, test_element_size, /*soo_capacity=*/0); + const size_t test_key_size = 3; + const size_t test_value_size = 5; + auto* info = + s->Register(test_stride, test_element_size, /*key_size=*/test_key_size, + /*value_size=*/test_value_size, /*soo_capacity=*/0); assert(info != nullptr); info->size.store(size); return info; @@ -81,9 +85,14 @@ TEST(HashtablezInfoTest, PrepareForSampling) { absl::Time test_start = absl::Now(); const int64_t test_stride = 123; const size_t test_element_size = 17; + const size_t test_key_size = 15; + const size_t test_value_size = 13; + HashtablezInfo info; absl::MutexLock l(&info.init_mu); info.PrepareForSampling(test_stride, test_element_size, + /*key_size=*/test_key_size, + /*value_size=*/test_value_size, /*soo_capacity_value=*/1); EXPECT_EQ(info.capacity.load(), 0); @@ -99,6 +108,8 @@ TEST(HashtablezInfoTest, PrepareForSampling) { EXPECT_GE(info.create_time, test_start); EXPECT_EQ(info.weight, test_stride); EXPECT_EQ(info.inline_element_size, test_element_size); + EXPECT_EQ(info.key_size, test_key_size); + EXPECT_EQ(info.value_size, test_value_size); EXPECT_EQ(info.soo_capacity, 1); info.capacity.store(1, std::memory_order_relaxed); @@ -113,6 +124,8 @@ TEST(HashtablezInfoTest, PrepareForSampling) { info.create_time = test_start - absl::Hours(20); info.PrepareForSampling(test_stride * 2, test_element_size, + /*key_size=*/test_key_size, + /*value_size=*/test_value_size, /*soo_capacity_value=*/0); EXPECT_EQ(info.capacity.load(), 0); EXPECT_EQ(info.size.load(), 0); @@ -126,6 +139,8 @@ TEST(HashtablezInfoTest, PrepareForSampling) { EXPECT_EQ(info.max_reserve.load(), 0); EXPECT_EQ(info.weight, 2 * test_stride); EXPECT_EQ(info.inline_element_size, test_element_size); + EXPECT_EQ(info.key_size, test_key_size); + EXPECT_EQ(info.value_size, test_value_size); EXPECT_GE(info.create_time, test_start); EXPECT_EQ(info.soo_capacity, 0); } @@ -135,7 +150,12 @@ TEST(HashtablezInfoTest, RecordStorageChanged) { absl::MutexLock l(&info.init_mu); const int64_t test_stride = 21; const size_t test_element_size = 19; + const size_t test_key_size = 17; + const size_t test_value_size = 15; + info.PrepareForSampling(test_stride, test_element_size, + /*key_size=*/test_key_size, + /*value_size=*/test_value_size, /*soo_capacity_value=*/0); RecordStorageChangedSlow(&info, 17, 47); EXPECT_EQ(info.size.load(), 17); @@ -150,7 +170,12 @@ TEST(HashtablezInfoTest, RecordInsert) { absl::MutexLock l(&info.init_mu); const int64_t test_stride = 25; const size_t test_element_size = 23; + const size_t test_key_size = 21; + const size_t test_value_size = 19; + info.PrepareForSampling(test_stride, test_element_size, + /*key_size=*/test_key_size, + /*value_size=*/test_value_size, /*soo_capacity_value=*/0); EXPECT_EQ(info.max_probe_length.load(), 0); RecordInsertSlow(&info, 0x0000FF00, 6 * kProbeLength); @@ -173,9 +198,14 @@ TEST(HashtablezInfoTest, RecordInsert) { TEST(HashtablezInfoTest, RecordErase) { const int64_t test_stride = 31; const size_t test_element_size = 29; + const size_t test_key_size = 27; + const size_t test_value_size = 25; + HashtablezInfo info; absl::MutexLock l(&info.init_mu); info.PrepareForSampling(test_stride, test_element_size, + /*key_size=*/test_key_size, + /*value_size=*/test_value_size, /*soo_capacity_value=*/1); EXPECT_EQ(info.num_erases.load(), 0); EXPECT_EQ(info.size.load(), 0); @@ -185,15 +215,22 @@ TEST(HashtablezInfoTest, RecordErase) { EXPECT_EQ(info.size.load(), 0); EXPECT_EQ(info.num_erases.load(), 1); EXPECT_EQ(info.inline_element_size, test_element_size); + EXPECT_EQ(info.key_size, test_key_size); + EXPECT_EQ(info.value_size, test_value_size); EXPECT_EQ(info.soo_capacity, 1); } TEST(HashtablezInfoTest, RecordRehash) { const int64_t test_stride = 33; const size_t test_element_size = 31; + const size_t test_key_size = 29; + const size_t test_value_size = 27; HashtablezInfo info; absl::MutexLock l(&info.init_mu); info.PrepareForSampling(test_stride, test_element_size, + /*key_size=*/test_key_size, + /*value_size=*/test_value_size, + /*soo_capacity_value=*/0); RecordInsertSlow(&info, 0x1, 0); RecordInsertSlow(&info, 0x2, kProbeLength); @@ -214,6 +251,8 @@ TEST(HashtablezInfoTest, RecordRehash) { EXPECT_EQ(info.num_erases.load(), 0); EXPECT_EQ(info.num_rehashes.load(), 1); EXPECT_EQ(info.inline_element_size, test_element_size); + EXPECT_EQ(info.key_size, test_key_size); + EXPECT_EQ(info.value_size, test_value_size); EXPECT_EQ(info.soo_capacity, 0); } @@ -222,7 +261,13 @@ TEST(HashtablezInfoTest, RecordReservation) { absl::MutexLock l(&info.init_mu); const int64_t test_stride = 35; const size_t test_element_size = 33; + const size_t test_key_size = 31; + const size_t test_value_size = 29; + info.PrepareForSampling(test_stride, test_element_size, + /*key_size=*/test_key_size, + /*value_size=*/test_value_size, + /*soo_capacity_value=*/0); RecordReservationSlow(&info, 3); EXPECT_EQ(info.max_reserve.load(), 3); @@ -239,13 +284,19 @@ TEST(HashtablezInfoTest, RecordReservation) { #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) TEST(HashtablezSamplerTest, SmallSampleParameter) { const size_t test_element_size = 31; + const size_t test_key_size = 33; + const size_t test_value_size = 35; + SetHashtablezEnabled(true); SetHashtablezSampleParameter(100); for (int i = 0; i < 1000; ++i) { SamplingState next_sample = {0, 0}; - HashtablezInfo* sample = SampleSlow(next_sample, test_element_size, - /*soo_capacity=*/0); + HashtablezInfo* sample = + SampleSlow(next_sample, test_element_size, + /*key_size=*/test_key_size, /*value_size=*/test_value_size, + + /*soo_capacity=*/0); EXPECT_GT(next_sample.next_sample, 0); EXPECT_EQ(next_sample.next_sample, next_sample.sample_stride); EXPECT_NE(sample, nullptr); @@ -255,13 +306,17 @@ TEST(HashtablezSamplerTest, SmallSampleParameter) { TEST(HashtablezSamplerTest, LargeSampleParameter) { const size_t test_element_size = 31; + const size_t test_key_size = 33; + const size_t test_value_size = 35; SetHashtablezEnabled(true); SetHashtablezSampleParameter(std::numeric_limits::max()); for (int i = 0; i < 1000; ++i) { SamplingState next_sample = {0, 0}; - HashtablezInfo* sample = SampleSlow(next_sample, test_element_size, - /*soo_capacity=*/0); + HashtablezInfo* sample = + SampleSlow(next_sample, test_element_size, + /*key_size=*/test_key_size, /*value_size=*/test_value_size, + /*soo_capacity=*/0); EXPECT_GT(next_sample.next_sample, 0); EXPECT_EQ(next_sample.next_sample, next_sample.sample_stride); EXPECT_NE(sample, nullptr); @@ -271,14 +326,20 @@ TEST(HashtablezSamplerTest, LargeSampleParameter) { TEST(HashtablezSamplerTest, Sample) { const size_t test_element_size = 31; + const size_t test_key_size = 33; + const size_t test_value_size = 35; SetHashtablezEnabled(true); SetHashtablezSampleParameter(100); int64_t num_sampled = 0; int64_t total = 0; double sample_rate = 0.0; for (int i = 0; i < 1000000; ++i) { - HashtablezInfoHandle h = Sample(test_element_size, - /*soo_capacity=*/0); + HashtablezInfoHandle h = + Sample(test_element_size, + /*key_size=*/test_key_size, /*value_size=*/test_value_size, + + /*soo_capacity=*/0); + ++total; if (h.IsSampled()) { ++num_sampled; @@ -293,7 +354,11 @@ TEST(HashtablezSamplerTest, Handle) { auto& sampler = GlobalHashtablezSampler(); const int64_t test_stride = 41; const size_t test_element_size = 39; + const size_t test_key_size = 37; + const size_t test_value_size = 35; HashtablezInfoHandle h(sampler.Register(test_stride, test_element_size, + /*key_size=*/test_key_size, + /*value_size=*/test_value_size, /*soo_capacity=*/0)); auto* info = HashtablezInfoHandlePeer::GetInfo(&h); info->hashes_bitwise_and.store(0x12345678, std::memory_order_relaxed); @@ -370,7 +435,10 @@ TEST(HashtablezSamplerTest, MultiThreaded) { for (int i = 0; i < 10; ++i) { const int64_t sampling_stride = 11 + i % 3; const size_t elt_size = 10 + i % 2; - pool.Schedule([&sampler, &stop, sampling_stride, elt_size]() { + const size_t key_size = 12 + i % 4; + const size_t value_size = 13 + i % 5; + pool.Schedule([&sampler, &stop, sampling_stride, elt_size, key_size, + value_size]() { std::random_device rd; std::mt19937 gen(rd()); @@ -378,11 +446,16 @@ TEST(HashtablezSamplerTest, MultiThreaded) { while (!stop.HasBeenNotified()) { if (infoz.empty()) { infoz.push_back(sampler.Register(sampling_stride, elt_size, + /*key_size=*/key_size, + /*value_size=*/value_size, /*soo_capacity=*/0)); } switch (std::uniform_int_distribution<>(0, 2)(gen)) { case 0: { infoz.push_back(sampler.Register(sampling_stride, elt_size, + /*key_size=*/key_size, + /*value_size=*/value_size, + /*soo_capacity=*/0)); break; } diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 58188444..2b0337ef 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -1786,7 +1786,8 @@ constexpr bool ShouldSampleHashtablezInfo() { } template -HashtablezInfoHandle SampleHashtablezInfo(size_t sizeof_slot, +HashtablezInfoHandle SampleHashtablezInfo(size_t sizeof_slot, size_t sizeof_key, + size_t sizeof_value, size_t old_capacity, bool was_soo, HashtablezInfoHandle forced_infoz, CommonFields& c) { @@ -1794,12 +1795,12 @@ HashtablezInfoHandle SampleHashtablezInfo(size_t sizeof_slot, // In SOO, we sample on the first insertion so if this is an empty SOO case // (e.g. when reserve is called), then we still need to sample. if (kSooEnabled && was_soo && c.size() == 0) { - return Sample(sizeof_slot, SooCapacity()); + return Sample(sizeof_slot, sizeof_key, sizeof_value, SooCapacity()); } // For non-SOO cases, we sample whenever the capacity is increasing from zero // to non-zero. if (!kSooEnabled && old_capacity == 0) { - return Sample(sizeof_slot, 0); + return Sample(sizeof_slot, sizeof_key, sizeof_value, 0); } return c.infoz(); } @@ -1902,12 +1903,15 @@ class HashSetResizeHelper { template ABSL_ATTRIBUTE_NOINLINE bool InitializeSlots(CommonFields& c, Alloc alloc, - ctrl_t soo_slot_h2) { + ctrl_t soo_slot_h2, + size_t key_size, + size_t value_size) { assert(c.capacity()); HashtablezInfoHandle infoz = ShouldSampleHashtablezInfo() - ? SampleHashtablezInfo(SizeOfSlot, old_capacity_, - was_soo_, forced_infoz_, c) + ? SampleHashtablezInfo(SizeOfSlot, key_size, value_size, + old_capacity_, was_soo_, + forced_infoz_, c) : HashtablezInfoHandle{}; const bool has_infoz = infoz.IsSampled(); @@ -3362,7 +3366,8 @@ class raw_hash_set { inline HashtablezInfoHandle try_sample_soo() { assert(is_soo()); if (!ShouldSampleHashtablezInfo()) return HashtablezInfoHandle{}; - return Sample(sizeof(slot_type), SooCapacity()); + return Sample(sizeof(slot_type), sizeof(key_type), sizeof(value_type), + SooCapacity()); } inline void destroy_slots() { @@ -3458,7 +3463,8 @@ class raw_hash_set { resize_helper.InitializeSlots( - common(), CharAlloc(alloc_ref()), soo_slot_h2); + common(), CharAlloc(alloc_ref()), soo_slot_h2, sizeof(key_type), + sizeof(value_type)); // In the SooEnabled() case, capacity is never 0 so we don't check. if (!SooEnabled() && resize_helper.old_capacity() == 0) { diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index f00cef8c..ca6656c8 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -2571,6 +2571,8 @@ TYPED_TEST_P(RawHashSamplerTest, Sample) { std::memory_order_relaxed)]++; reservations[info.max_reserve.load(std::memory_order_relaxed)]++; EXPECT_EQ(info.inline_element_size, sizeof(typename TypeParam::value_type)); + EXPECT_EQ(info.key_size, sizeof(typename TypeParam::key_type)); + EXPECT_EQ(info.value_size, sizeof(typename TypeParam::value_type)); if (soo_enabled) { EXPECT_EQ(info.soo_capacity, SooCapacity()); -- cgit v1.2.3 From ad5499a290fd98de54ee54dcf8120f8d287640ce Mon Sep 17 00:00:00 2001 From: Vitaly Goldshteyn Date: Mon, 25 Mar 2024 15:01:22 -0700 Subject: Add `BM_EraseIf` benchmark. PiperOrigin-RevId: 618970135 Change-Id: Ifb9d0b425904d5cb37d80ec28ab7845957209313 --- absl/container/BUILD.bazel | 1 + absl/container/internal/raw_hash_set_benchmark.cc | 60 +++++++++++++++++++++++ 2 files changed, 61 insertions(+) (limited to 'absl/container/internal') diff --git a/absl/container/BUILD.bazel b/absl/container/BUILD.bazel index 78ad60e0..12e27c21 100644 --- a/absl/container/BUILD.bazel +++ b/absl/container/BUILD.bazel @@ -726,6 +726,7 @@ cc_binary( ":hash_function_defaults", ":raw_hash_set", "//absl/base:raw_logging_internal", + "//absl/random", "//absl/strings:str_format", "@com_github_google_benchmark//:benchmark_main", ], diff --git a/absl/container/internal/raw_hash_set_benchmark.cc b/absl/container/internal/raw_hash_set_benchmark.cc index 0fa9c712..05f62d2f 100644 --- a/absl/container/internal/raw_hash_set_benchmark.cc +++ b/absl/container/internal/raw_hash_set_benchmark.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -27,6 +28,7 @@ #include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_function_defaults.h" #include "absl/container/internal/raw_hash_set.h" +#include "absl/random/random.h" #include "absl/strings/str_format.h" #include "benchmark/benchmark.h" @@ -575,6 +577,64 @@ void BM_Resize(benchmark::State& state) { } BENCHMARK(BM_Resize); +void BM_EraseIf(benchmark::State& state) { + int64_t num_elements = state.range(0); + size_t num_erased = static_cast(state.range(1)); + + constexpr size_t kRepetitions = 64; + + absl::BitGen rng; + + std::vector> keys(kRepetitions); + std::vector tables; + std::vector threshold; + for (auto& k : keys) { + tables.push_back(IntTable()); + auto& table = tables.back(); + for (int64_t i = 0; i < num_elements; i++) { + // We use random keys to reduce noise. + k.push_back(absl::Uniform(rng, 0, ~int64_t{})); + if (!table.insert(k.back()).second) { + k.pop_back(); + --i; // duplicated value, retrying + } + } + std::sort(k.begin(), k.end()); + threshold.push_back(k[num_erased]); + } + + while (state.KeepRunningBatch(static_cast(kRepetitions) * + std::max(num_elements, int64_t{1}))) { + benchmark::DoNotOptimize(tables); + for (size_t t_id = 0; t_id < kRepetitions; t_id++) { + auto& table = tables[t_id]; + benchmark::DoNotOptimize(num_erased); + auto pred = [t = threshold[t_id]](int64_t key) { return key < t; }; + benchmark::DoNotOptimize(pred); + benchmark::DoNotOptimize(table); + absl::container_internal::EraseIf(pred, &table); + } + state.PauseTiming(); + for (size_t t_id = 0; t_id < kRepetitions; t_id++) { + auto& k = keys[t_id]; + auto& table = tables[t_id]; + for (size_t i = 0; i < num_erased; i++) { + table.insert(k[i]); + } + } + state.ResumeTiming(); + } +} + +BENCHMARK(BM_EraseIf) + ->ArgNames({"num_elements", "num_erased"}) + ->ArgPair(10, 0) + ->ArgPair(1000, 0) + ->ArgPair(10, 5) + ->ArgPair(1000, 500) + ->ArgPair(10, 10) + ->ArgPair(1000, 1000); + } // namespace } // namespace container_internal ABSL_NAMESPACE_END -- cgit v1.2.3 From 68ce303da1920522a27e5d8e2732b3e50d3aae57 Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Tue, 26 Mar 2024 12:04:23 -0700 Subject: Respect `NDEBUG_SANITIZER` Often code needs to know that it's built with sanitizers. There are two common use for such information: 1. Work around incompatibility with sanitizers 2. Use sanitizers for more aggressive bug detection With the current `ABSL_HAVE_*_SANITIZER` we can't distinguish this two cased, and we didn't need that before. Now user can define `NDEBUG_SANITIZER` to ask code like this to avoid unnecessary checks. I am not 100% sure that `NDEBUG` is not enough. However relying on `NDEBUG` today will relax many tests, which runs in NDEBUG mode only. So new `NDEBUG_SANITIZER` is safer approach. PiperOrigin-RevId: 619268413 Change-Id: I58185cd6886593a3742b8424deccdec366c2a35a --- absl/container/internal/btree.h | 7 ++++--- absl/container/internal/raw_hash_set.h | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/btree.h b/absl/container/internal/btree.h index 4ffc44e7..689e71a5 100644 --- a/absl/container/internal/btree.h +++ b/absl/container/internal/btree.h @@ -77,9 +77,10 @@ namespace container_internal { #ifdef ABSL_BTREE_ENABLE_GENERATIONS #error ABSL_BTREE_ENABLE_GENERATIONS cannot be directly set -#elif defined(ABSL_HAVE_ADDRESS_SANITIZER) || \ - defined(ABSL_HAVE_HWADDRESS_SANITIZER) || \ - defined(ABSL_HAVE_MEMORY_SANITIZER) +#elif (defined(ABSL_HAVE_ADDRESS_SANITIZER) || \ + defined(ABSL_HAVE_HWADDRESS_SANITIZER) || \ + defined(ABSL_HAVE_MEMORY_SANITIZER)) && \ + !defined(NDEBUG_SANITIZER) // If defined, performance is important. // When compiled in sanitizer mode, we add generation integers to the nodes and // iterators. When iterators are used, we validate that the container has not // been mutated since the iterator was constructed. diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 2b0337ef..da2ee1c5 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -240,9 +240,10 @@ namespace container_internal { #ifdef ABSL_SWISSTABLE_ENABLE_GENERATIONS #error ABSL_SWISSTABLE_ENABLE_GENERATIONS cannot be directly set -#elif defined(ABSL_HAVE_ADDRESS_SANITIZER) || \ - defined(ABSL_HAVE_HWADDRESS_SANITIZER) || \ - defined(ABSL_HAVE_MEMORY_SANITIZER) +#elif (defined(ABSL_HAVE_ADDRESS_SANITIZER) || \ + defined(ABSL_HAVE_HWADDRESS_SANITIZER) || \ + defined(ABSL_HAVE_MEMORY_SANITIZER)) && \ + !defined(NDEBUG_SANITIZER) // If defined, performance is important. // When compiled in sanitizer mode, we add generation integers to the backing // array and iterators. In the backing array, we store the generation between // the control bytes and the slots. When iterators are dereferenced, we assert -- cgit v1.2.3 From 6f0c500453dd911018bd5cfb46999cee5e3e2cf8 Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Tue, 26 Mar 2024 13:34:56 -0700 Subject: Refactor the GCC unintialized memory warning suppression in raw_hash_set.h. Motivation: there are new uninitialized memory warnings when we enable small object optimization. PiperOrigin-RevId: 619295212 Change-Id: If10762bab0e43c9619fc03f6d1eef5b8836bbf9a --- absl/container/internal/raw_hash_set.h | 88 ++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 36 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index da2ee1c5..af75e7fb 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -1131,24 +1131,27 @@ struct soo_tag_t {}; // Sentinel type to indicate SOO CommonFields construction with full size. struct full_soo_tag_t {}; +// Suppress erroneous uninitialized memory errors on GCC. For example, GCC +// thinks that the call to slot_array() in find_or_prepare_insert() is reading +// uninitialized memory, but slot_array is only called there when the table is +// non-empty and this memory is initialized when the table is non-empty. +#if !defined(__clang__) && defined(__GNUC__) +#define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED(x) \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wmaybe-uninitialized\"") \ + _Pragma("GCC diagnostic ignored \"-Wuninitialized\"") x; \ + _Pragma("GCC diagnostic pop") +#define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(x) \ + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED(return x) +#else +#define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED(x) x +#define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(x) return x +#endif + // This allows us to work around an uninitialized memory warning when // constructing begin() iterators in empty hashtables. union MaybeInitializedPtr { - void* get() const { - // Suppress erroneous uninitialized memory errors on GCC. GCC thinks that - // the call to slot_array() in find_or_prepare_insert() is reading - // uninitialized memory, but slot_array is only called there when the table - // is non-empty and this memory is initialized when the table is non-empty. -#if !defined(__clang__) && defined(__GNUC__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#pragma GCC diagnostic ignored "-Wuninitialized" -#endif - return p; -#if !defined(__clang__) && defined(__GNUC__) -#pragma GCC diagnostic pop -#endif - } + void* get() const { ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(p); } void set(void* ptr) { p = ptr; } void* p; @@ -1180,6 +1183,25 @@ union HeapOrSoo { HeapOrSoo() = default; explicit HeapOrSoo(ctrl_t* c) : heap(c) {} + ctrl_t*& control() { + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap.control); + } + ctrl_t* control() const { + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap.control); + } + MaybeInitializedPtr& slot_array() { + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap.slot_array); + } + MaybeInitializedPtr slot_array() const { + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap.slot_array); + } + void* get_soo_data() { + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(soo_data); + } + const void* get_soo_data() const { + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(soo_data); + } + HeapPtrs heap; unsigned char soo_data[sizeof(HeapPtrs)]; }; @@ -1208,14 +1230,14 @@ class CommonFields : public CommonFieldsGenerationInfo { } // The inline data for SOO is written on top of control_/slots_. - const void* soo_data() const { return heap_or_soo_.soo_data; } - void* soo_data() { return heap_or_soo_.soo_data; } + const void* soo_data() const { return heap_or_soo_.get_soo_data(); } + void* soo_data() { return heap_or_soo_.get_soo_data(); } HeapOrSoo heap_or_soo() const { return heap_or_soo_; } const HeapOrSoo& heap_or_soo_ref() const { return heap_or_soo_; } - ctrl_t* control() const { return heap_or_soo_.heap.control; } - void set_control(ctrl_t* c) { heap_or_soo_.heap.control = c; } + ctrl_t* control() const { return heap_or_soo_.control(); } + void set_control(ctrl_t* c) { heap_or_soo_.control() = c; } void* backing_array_start() const { // growth_left (and maybe infoz) is stored before control bytes. assert(reinterpret_cast(control()) % alignof(size_t) == 0); @@ -1223,19 +1245,9 @@ class CommonFields : public CommonFieldsGenerationInfo { } // Note: we can't use slots() because Qt defines "slots" as a macro. - void* slot_array() const { return heap_or_soo_.heap.slot_array.get(); } - MaybeInitializedPtr slots_union() const { - // Suppress erroneous uninitialized memory errors on GCC. -#if !defined(__clang__) && defined(__GNUC__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif - return heap_or_soo_.heap.slot_array; -#if !defined(__clang__) && defined(__GNUC__) -#pragma GCC diagnostic pop -#endif - } - void set_slots(void* s) { heap_or_soo_.heap.slot_array.set(s); } + void* slot_array() const { return heap_or_soo_.slot_array().get(); } + MaybeInitializedPtr slots_union() const { return heap_or_soo_.slot_array(); } + void set_slots(void* s) { heap_or_soo_.slot_array().set(s); } // The number of filled slots. size_t size() const { return size_ >> HasInfozShift(); } @@ -1850,14 +1862,14 @@ class HashSetResizeHelper { } HeapOrSoo& old_heap_or_soo() { return old_heap_or_soo_; } - void* old_soo_data() { return old_heap_or_soo_.soo_data; } + void* old_soo_data() { return old_heap_or_soo_.get_soo_data(); } ctrl_t* old_ctrl() const { assert(!was_soo_); - return old_heap_or_soo_.heap.control; + return old_heap_or_soo_.control(); } void* old_slots() const { assert(!was_soo_); - return old_heap_or_soo_.heap.slot_array.get(); + return old_heap_or_soo_.slot_array().get(); } size_t old_capacity() const { return old_capacity_; } @@ -3393,7 +3405,9 @@ class raw_hash_set { inline void destructor_impl() { if (capacity() == 0) return; if (is_soo()) { - if (!empty()) destroy(soo_slot()); + if (!empty()) { + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED(destroy(soo_slot())); + } return; } destroy_slots(); @@ -3977,5 +3991,7 @@ ABSL_NAMESPACE_END } // namespace absl #undef ABSL_SWISSTABLE_ENABLE_GENERATIONS +#undef ABSL_SWISSTABLE_IGNORE_UNINITIALIZED +#undef ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN #endif // ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ -- cgit v1.2.3 From 1ccc2eb35ed685a5640cb80a26be4df535b9c7b9 Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Tue, 26 Mar 2024 14:20:28 -0700 Subject: Enable small object optimization in swisstable. See [implementation commit](https://github.com/abseil/abseil-cpp/commit/1449c9a106b090f61441ba245c781d7d2f89000c) for design details. PiperOrigin-RevId: 619309882 Change-Id: I093c00365dda2268be86ba3d21421b6ffb59a5ce --- absl/container/internal/hash_policy_traits.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/hash_policy_traits.h b/absl/container/internal/hash_policy_traits.h index ec08794a..ad835d6f 100644 --- a/absl/container/internal/hash_policy_traits.h +++ b/absl/container/internal/hash_policy_traits.h @@ -168,7 +168,7 @@ struct hash_policy_traits : common_policy_traits { #endif } - // Whether small object optimization is enabled. False by default. + // Whether small object optimization is enabled. True by default. static constexpr bool soo_enabled() { return soo_enabled_impl(Rank1{}); } private: @@ -197,7 +197,7 @@ struct hash_policy_traits : common_policy_traits { return P::soo_enabled(); } - static constexpr bool soo_enabled_impl(Rank0) { return false; } + static constexpr bool soo_enabled_impl(Rank0) { return true; } }; } // namespace container_internal -- cgit v1.2.3 From b70ad841380f60e8e825019bdc1e63f7c071843f Mon Sep 17 00:00:00 2001 From: Vitaly Goldshteyn Date: Tue, 26 Mar 2024 21:54:27 -0700 Subject: Introduce GrowthInfo with tests, but without usage. The motivation is to use presence of kDeleted slots for optimizing InsertMiss for tables without kDeleted slots. PiperOrigin-RevId: 619411282 Change-Id: Icc30606374aba7ce60b41f93ee8d44894e1b8aa5 --- absl/container/internal/raw_hash_set.h | 82 ++++++++++++++++++++++++++++ absl/container/internal/raw_hash_set_test.cc | 72 ++++++++++++++++++++++++ 2 files changed, 154 insertions(+) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index af75e7fb..7db1c304 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -1054,6 +1054,88 @@ using CommonFieldsGenerationInfo = CommonFieldsGenerationInfoDisabled; using HashSetIteratorGenerationInfo = HashSetIteratorGenerationInfoDisabled; #endif +// Stored the information regarding number of slots we can still fill +// without needing to rehash. +// +// We want to ensure sufficient number of kEmpty slots in the table in order +// to keep probe sequences relatively short. kEmpty slot in the probe group +// is required to stop probing. +// +// Tombstones (kDeleted slots) are not included in the growth capacity, +// because we'd like to rehash when the table is filled with tombstones and/or +// full slots. +// +// GrowthInfo also stores a bit that encodes whether table may have any +// kDeleted slots. +// Most of the tables (>95%) have no kDeleted slots, so some functions can +// be more efficient with this information. +// +// Callers can also force a rehash via the standard `rehash(0)`, +// which will recompute this value as a side-effect. +// +// See also `CapacityToGrowth()`. +class GrowthInfo { + public: + // Leaves data member uninitialized. + GrowthInfo() = default; + + // Initializes the GrowthInfo assuming we can grow `growth_left` elements + // and there are no kDeleted slots in the table. + void InitGrowthLeftNoDeleted(size_t growth_left) { + growth_left_info_ = growth_left; + } + + // Overwrites single full slot with an empty slot. + void OverwriteFullAsEmpty() { ++growth_left_info_; } + + // Overwrites single empty slot with a full slot. + void OverwriteEmptyAsFull() { + assert(GetGrowthLeft() > 0); + --growth_left_info_; + } + + // Overwrites several empty slots with full slots. + void OverwriteManyEmptyAsFull(size_t cnt) { + assert(GetGrowthLeft() >= cnt); + growth_left_info_ -= cnt; + } + + // Overwrites specified control element with full slot. + void OverwriteControlAsFull(ctrl_t ctrl) { + assert(GetGrowthLeft() >= static_cast(IsEmpty(ctrl))); + growth_left_info_ -= static_cast(IsEmpty(ctrl)); + } + + // Overwrites single full slot with a deleted slot. + void OverwriteFullAsDeleted() { growth_left_info_ |= kDeletedBit; } + + // Returns true if table satisfies two properties: + // 1. Guaranteed to have no kDeleted slots. + // 2. There is a place for at least one element to grow. + bool HasNoDeletedAndGrowthLeft() const { + return static_cast>(growth_left_info_) > 0; + } + + // Returns true if table guaranteed to have no k + bool HasNoDeleted() const { + return static_cast>(growth_left_info_) >= 0; + } + + // Returns the number of elements left to grow. + size_t GetGrowthLeft() const { + return growth_left_info_ & kGrowthLeftMask; + } + + private: + static constexpr size_t kGrowthLeftMask = ((~size_t{}) >> 1); + static constexpr size_t kDeletedBit = ~kGrowthLeftMask; + // Topmost bit signal whenever there are deleted slots. + size_t growth_left_info_; +}; + +static_assert(sizeof(GrowthInfo) == sizeof(size_t), ""); +static_assert(alignof(GrowthInfo) == alignof(size_t), ""); + // Returns whether `n` is a valid capacity (i.e., number of slots). // // A valid capacity is a non-zero integer `2^m - 1`. diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index ca6656c8..876894f4 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -92,6 +92,78 @@ using ::testing::UnorderedElementsAre; // Convenience function to static cast to ctrl_t. ctrl_t CtrlT(int i) { return static_cast(i); } +TEST(GrowthInfoTest, GetGrowthLeft) { + GrowthInfo gi; + gi.InitGrowthLeftNoDeleted(5); + EXPECT_EQ(gi.GetGrowthLeft(), 5); + gi.OverwriteFullAsDeleted(); + EXPECT_EQ(gi.GetGrowthLeft(), 5); +} + +TEST(GrowthInfoTest, HasNoDeleted) { + GrowthInfo gi; + gi.InitGrowthLeftNoDeleted(5); + EXPECT_TRUE(gi.HasNoDeleted()); + gi.OverwriteFullAsDeleted(); + EXPECT_FALSE(gi.HasNoDeleted()); + // After reinitialization we have no deleted slots. + gi.InitGrowthLeftNoDeleted(5); + EXPECT_TRUE(gi.HasNoDeleted()); +} + +TEST(GrowthInfoTest, HasNoDeletedAndGrowthLeft) { + GrowthInfo gi; + gi.InitGrowthLeftNoDeleted(5); + EXPECT_TRUE(gi.HasNoDeletedAndGrowthLeft()); + gi.OverwriteFullAsDeleted(); + EXPECT_FALSE(gi.HasNoDeletedAndGrowthLeft()); + gi.InitGrowthLeftNoDeleted(0); + EXPECT_FALSE(gi.HasNoDeletedAndGrowthLeft()); + gi.OverwriteFullAsDeleted(); + EXPECT_FALSE(gi.HasNoDeletedAndGrowthLeft()); + // After reinitialization we have no deleted slots. + gi.InitGrowthLeftNoDeleted(5); + EXPECT_TRUE(gi.HasNoDeletedAndGrowthLeft()); +} + +TEST(GrowthInfoTest, OverwriteFullAsEmpty) { + GrowthInfo gi; + gi.InitGrowthLeftNoDeleted(5); + gi.OverwriteFullAsEmpty(); + EXPECT_EQ(gi.GetGrowthLeft(), 6); + gi.OverwriteFullAsDeleted(); + EXPECT_EQ(gi.GetGrowthLeft(), 6); + gi.OverwriteFullAsEmpty(); + EXPECT_EQ(gi.GetGrowthLeft(), 7); + EXPECT_FALSE(gi.HasNoDeleted()); +} + +TEST(GrowthInfoTest, OverwriteEmptyAsFull) { + GrowthInfo gi; + gi.InitGrowthLeftNoDeleted(5); + gi.OverwriteEmptyAsFull(); + EXPECT_EQ(gi.GetGrowthLeft(), 4); + gi.OverwriteFullAsDeleted(); + EXPECT_EQ(gi.GetGrowthLeft(), 4); + gi.OverwriteEmptyAsFull(); + EXPECT_EQ(gi.GetGrowthLeft(), 3); + EXPECT_FALSE(gi.HasNoDeleted()); +} + +TEST(GrowthInfoTest, OverwriteControlAsFull) { + GrowthInfo gi; + gi.InitGrowthLeftNoDeleted(5); + gi.OverwriteControlAsFull(ctrl_t::kEmpty); + EXPECT_EQ(gi.GetGrowthLeft(), 4); + gi.OverwriteControlAsFull(ctrl_t::kDeleted); + EXPECT_EQ(gi.GetGrowthLeft(), 4); + gi.OverwriteFullAsDeleted(); + gi.OverwriteControlAsFull(ctrl_t::kDeleted); + // We do not count number of deleted, so the bit sticks till the next rehash. + EXPECT_FALSE(gi.HasNoDeletedAndGrowthLeft()); + EXPECT_FALSE(gi.HasNoDeleted()); +} + TEST(Util, NormalizeCapacity) { EXPECT_EQ(1, NormalizeCapacity(0)); EXPECT_EQ(1, NormalizeCapacity(1)); -- cgit v1.2.3 From ff0a0f2d5428e8cc6a79cb9000d9cd3f9a0a67b5 Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Wed, 27 Mar 2024 11:33:27 -0700 Subject: Disable small object optimization while debugging some failing tests. PiperOrigin-RevId: 619598530 Change-Id: Ie4b808a3b826db8c271c81914c7a88d2c6216eb2 --- absl/container/internal/hash_policy_traits.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/hash_policy_traits.h b/absl/container/internal/hash_policy_traits.h index ad835d6f..ec08794a 100644 --- a/absl/container/internal/hash_policy_traits.h +++ b/absl/container/internal/hash_policy_traits.h @@ -168,7 +168,7 @@ struct hash_policy_traits : common_policy_traits { #endif } - // Whether small object optimization is enabled. True by default. + // Whether small object optimization is enabled. False by default. static constexpr bool soo_enabled() { return soo_enabled_impl(Rank1{}); } private: @@ -197,7 +197,7 @@ struct hash_policy_traits : common_policy_traits { return P::soo_enabled(); } - static constexpr bool soo_enabled_impl(Rank0) { return true; } + static constexpr bool soo_enabled_impl(Rank0) { return false; } }; } // namespace container_internal -- cgit v1.2.3 From 52715dbd30e19bda452f686c496379fe20660345 Mon Sep 17 00:00:00 2001 From: Vitaly Goldshteyn Date: Wed, 27 Mar 2024 14:05:18 -0700 Subject: Use GrowthInfo without applying any optimizations based on it. PiperOrigin-RevId: 619649335 Change-Id: I8b3380816418a363fb6686db7966248cb530c491 --- absl/container/internal/raw_hash_set.cc | 9 +++--- absl/container/internal/raw_hash_set.h | 47 +++++++++++++++------------- absl/container/internal/raw_hash_set_test.cc | 21 +++++++++++++ 3 files changed, 52 insertions(+), 25 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.cc b/absl/container/internal/raw_hash_set.cc index f0f840d1..c23c1f3b 100644 --- a/absl/container/internal/raw_hash_set.cc +++ b/absl/container/internal/raw_hash_set.cc @@ -33,11 +33,11 @@ namespace container_internal { // Represents a control byte corresponding to a full slot with arbitrary hash. constexpr ctrl_t ZeroCtrlT() { return static_cast(0); } -// We have space for `growth_left` before a single block of control bytes. A +// We have space for `growth_info` before a single block of control bytes. A // single block of empty control bytes for tables without any slots allocated. // This enables removing a branch in the hot path of find(). In order to ensure // that the control bytes are aligned to 16, we have 16 bytes before the control -// bytes even though growth_left only needs 8. +// bytes even though growth_info only needs 8. alignas(16) ABSL_CONST_INIT ABSL_DLL const ctrl_t kEmptyGroup[32] = { ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), @@ -133,7 +133,7 @@ size_t PrepareInsertAfterSoo(size_t hash, size_t slot_size, assert(HashSetResizeHelper::SooSlotIndex() == 1); PrepareInsertCommon(common); const size_t offset = H1(hash, common.control()) & 2; - common.set_growth_left(common.growth_left() - 1); + common.growth_info().OverwriteEmptyAsFull(); SetCtrlInSingleGroupTable(common, offset, H2(hash), slot_size); common.infoz().RecordInsert(hash, /*distance_from_desired=*/0); return offset; @@ -274,10 +274,11 @@ void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size) { if (WasNeverFull(c, index)) { SetCtrl(c, index, ctrl_t::kEmpty, slot_size); - c.set_growth_left(c.growth_left() + 1); + c.growth_info().OverwriteFullAsEmpty(); return; } + c.growth_info().OverwriteFullAsDeleted(); SetCtrl(c, index, ctrl_t::kDeleted, slot_size); } diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 7db1c304..b99b4a4b 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -1154,9 +1154,9 @@ constexpr size_t NumControlBytes(size_t capacity) { } // Computes the offset from the start of the backing allocation of control. -// infoz and growth_left are stored at the beginning of the backing array. +// infoz and growth_info are stored at the beginning of the backing array. inline static size_t ControlOffset(bool has_infoz) { - return (has_infoz ? sizeof(HashtablezInfoHandle) : 0) + sizeof(size_t); + return (has_infoz ? sizeof(HashtablezInfoHandle) : 0) + sizeof(GrowthInfo); } // Helper class for computing offsets and allocation size of hash set fields. @@ -1249,7 +1249,7 @@ struct HeapPtrs { // This contains `capacity + 1 + NumClonedBytes()` entries, even // when the table is empty (hence EmptyGroup). // - // Note that growth_left is stored immediately before this pointer. + // Note that growth_info is stored immediately before this pointer. // May be uninitialized for SOO tables. ctrl_t* control; @@ -1321,7 +1321,7 @@ class CommonFields : public CommonFieldsGenerationInfo { ctrl_t* control() const { return heap_or_soo_.control(); } void set_control(ctrl_t* c) { heap_or_soo_.control() = c; } void* backing_array_start() const { - // growth_left (and maybe infoz) is stored before control bytes. + // growth_info (and maybe infoz) is stored before control bytes. assert(reinterpret_cast(control()) % alignof(size_t) == 0); return control() - ControlOffset(has_infoz()); } @@ -1362,17 +1362,15 @@ class CommonFields : public CommonFieldsGenerationInfo { // The number of slots we can still fill without needing to rehash. // This is stored in the heap allocation before the control bytes. - // TODO(b/289225379): experiment with moving growth_left back inline to + // TODO(b/289225379): experiment with moving growth_info back inline to // increase room for SOO. - size_t growth_left() const { - const size_t* gl_ptr = reinterpret_cast(control()) - 1; - assert(reinterpret_cast(gl_ptr) % alignof(size_t) == 0); + GrowthInfo& growth_info() { + auto* gl_ptr = reinterpret_cast(control()) - 1; + assert(reinterpret_cast(gl_ptr) % alignof(GrowthInfo) == 0); return *gl_ptr; } - void set_growth_left(size_t gl) { - size_t* gl_ptr = reinterpret_cast(control()) - 1; - assert(reinterpret_cast(gl_ptr) % alignof(size_t) == 0); - *gl_ptr = gl; + GrowthInfo growth_info() const { + return const_cast(this)->growth_info(); } bool has_infoz() const { @@ -1759,7 +1757,8 @@ extern template FindInfo find_first_non_full(const CommonFields&, size_t); FindInfo find_first_non_full_outofline(const CommonFields&, size_t); inline void ResetGrowthLeft(CommonFields& common) { - common.set_growth_left(CapacityToGrowth(common.capacity()) - common.size()); + common.growth_info().InitGrowthLeftNoDeleted( + CapacityToGrowth(common.capacity()) - common.size()); } // Sets `ctrl` to `{kEmpty, kSentinel, ..., kEmpty}`, marking the entire @@ -1818,9 +1817,9 @@ inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, h2_t h, SetCtrlInSingleGroupTable(c, i, static_cast(h), slot_size); } -// growth_left (which is a size_t) is stored with the backing array. +// growth_info (which is a size_t) is stored with the backing array. constexpr size_t BackingArrayAlignment(size_t align_of_slot) { - return (std::max)(align_of_slot, alignof(size_t)); + return (std::max)(align_of_slot, alignof(GrowthInfo)); } // Returns the address of the ith slot in slots where each slot occupies @@ -2717,7 +2716,7 @@ class raw_hash_set { infoz().RecordStorageChanged(size, cap); } common().set_size(size); - set_growth_left(growth_left() - size); + growth_info().OverwriteManyEmptyAsFull(size); } ABSL_ATTRIBUTE_NOINLINE raw_hash_set(raw_hash_set&& that) noexcept( @@ -3847,7 +3846,8 @@ class raw_hash_set { resize(growth_left() > 0 ? cap : NextCapacity(cap)); } FindInfo target; - if (!rehash_for_bug_detection && ABSL_PREDICT_FALSE(growth_left() == 0)) { + if (!rehash_for_bug_detection && + ABSL_PREDICT_FALSE(growth_left() == 0)) { const size_t old_capacity = capacity(); rehash_and_grow_if_necessary(); // NOTE: It is safe to use `FindFirstNonFullAfterResize` after @@ -3863,7 +3863,7 @@ class raw_hash_set { target = find_first_non_full(common(), hash); } PrepareInsertCommon(common()); - set_growth_left(growth_left() - IsEmpty(control()[target.offset])); + growth_info().OverwriteControlAsFull(control()[target.offset]); SetCtrl(common(), target.offset, H2(hash), sizeof(slot_type)); infoz().RecordInsert(hash, target.probe_length); return target.offset; @@ -3909,11 +3909,16 @@ class raw_hash_set { // See `CapacityToGrowth()`. size_t growth_left() const { assert(!is_soo()); - return common().growth_left(); + return growth_info().GetGrowthLeft(); + } + + GrowthInfo& growth_info() { + assert(!is_soo()); + return common().growth_info(); } - void set_growth_left(size_t gl) { + GrowthInfo growth_info() const { assert(!is_soo()); - return common().set_growth_left(gl); + return common().growth_info(); } // Prefetch the heap-allocated memory region to resolve potential TLB and diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 876894f4..c4e05d60 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -1825,6 +1825,27 @@ TEST(Table, EraseInsertProbing) { EXPECT_THAT(t, UnorderedElementsAre(1, 10, 3, 11, 12)); } +TEST(Table, GrowthInfoDeletedBit) { + BadTable t; + EXPECT_TRUE( + RawHashSetTestOnlyAccess::GetCommon(t).growth_info().HasNoDeleted()); + int64_t init_count = static_cast( + CapacityToGrowth(NormalizeCapacity(Group::kWidth + 1))); + for (int64_t i = 0; i < init_count; ++i) { + t.insert(i); + } + EXPECT_TRUE( + RawHashSetTestOnlyAccess::GetCommon(t).growth_info().HasNoDeleted()); + t.erase(0); + EXPECT_EQ(RawHashSetTestOnlyAccess::CountTombstones(t), 1); + EXPECT_FALSE( + RawHashSetTestOnlyAccess::GetCommon(t).growth_info().HasNoDeleted()); + t.rehash(0); + EXPECT_EQ(RawHashSetTestOnlyAccess::CountTombstones(t), 0); + EXPECT_TRUE( + RawHashSetTestOnlyAccess::GetCommon(t).growth_info().HasNoDeleted()); +} + TYPED_TEST_P(SooTest, Clear) { TypeParam t; EXPECT_TRUE(t.find(0) == t.end()); -- cgit v1.2.3 From 41136ed173b64fbe4ef55838bcc24c6b81dead5e Mon Sep 17 00:00:00 2001 From: Vitaly Goldshteyn Date: Wed, 27 Mar 2024 15:37:09 -0700 Subject: Optimize InsertMiss for tables without kDeleted slots. This CL contains two optimizations that were measured together. 1) InsertMiss (i. e. successful insert) optimization: The idea that in case there is no kDeleted, we already know 99% of the information. So we are finding the position to insert with 2 asm instructions (or 3 in case of ARM or portable) and passing that as a hint into `prepare_insert`. `prepare_insert` is out of the line in order to minimize effect on InsertHit (the most important case). `prepare_insert` may use the hint in case we still have growth and no kDeleted is guaranteed. In case of kDeleted, we still call `find_first_non_full` in order to potentially find kDeleted slot earlier. We may consider different ways to do it faster for kDeleted later. 2) `find_first_non_full` optimization: After optimization #1 `find_first_non_full` is used: 1. during resize and copy 2. right after resize 3. during DropDeletedWithoutResize 3. in InsertMiss for tables with kDeleted In cases 1-3 the table is quite sparse. 1. After resize it is 7/16 sparse 2. During resize it is 7/16 maximum, but during first inserts it is much sparser. 3. During copy it may be up to 7/8, but at the beginning it is way sparser. 4. During DropDeletedWithoutResize it is 25/32 sparse, but at the beginning it is way sparser. The only case, where the table is not known to be sparse, with `find_first_non_full` usage is a table with deleted elements. But according to hashz, we have <3% such tables. Adding an extra branch wouldn't hurt much there. PiperOrigin-RevId: 619681885 Change-Id: Id3e2852cc6d85f6c8f90982d7aeb14799696bf39 --- absl/container/internal/raw_hash_set.h | 104 +++++++++++++++++++++++---------- 1 file changed, 72 insertions(+), 32 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index b99b4a4b..1d2e2d14 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -571,6 +571,16 @@ inline bool IsEmptyGeneration(const GenerationType* generation) { bool ShouldInsertBackwardsForDebug(size_t capacity, size_t hash, const ctrl_t* ctrl); +ABSL_ATTRIBUTE_ALWAYS_INLINE inline bool ShouldInsertBackwards( + ABSL_ATTRIBUTE_UNUSED size_t capacity, ABSL_ATTRIBUTE_UNUSED size_t hash, + ABSL_ATTRIBUTE_UNUSED const ctrl_t* ctrl) { +#if defined(NDEBUG) + return false; +#else + return ShouldInsertBackwardsForDebug(capacity, hash, ctrl); +#endif +} + // Returns insert position for the given mask. // We want to add entropy even when ASLR is not enabled. // In debug build we will randomly insert in either the front or back of @@ -1057,8 +1067,8 @@ using HashSetIteratorGenerationInfo = HashSetIteratorGenerationInfoDisabled; // Stored the information regarding number of slots we can still fill // without needing to rehash. // -// We want to ensure sufficient number of kEmpty slots in the table in order -// to keep probe sequences relatively short. kEmpty slot in the probe group +// We want to ensure sufficient number of empty slots in the table in order +// to keep probe sequences relatively short. Empty slot in the probe group // is required to stop probing. // // Tombstones (kDeleted slots) are not included in the growth capacity, @@ -1066,8 +1076,8 @@ using HashSetIteratorGenerationInfo = HashSetIteratorGenerationInfoDisabled; // full slots. // // GrowthInfo also stores a bit that encodes whether table may have any -// kDeleted slots. -// Most of the tables (>95%) have no kDeleted slots, so some functions can +// deleted slots. +// Most of the tables (>95%) have no deleted slots, so some functions can // be more efficient with this information. // // Callers can also force a rehash via the standard `rehash(0)`, @@ -1734,6 +1744,10 @@ template inline FindInfo find_first_non_full(const CommonFields& common, size_t hash) { auto seq = probe(common, hash); const ctrl_t* ctrl = common.control(); + if (IsEmptyOrDeleted(ctrl[seq.offset()]) && + !ShouldInsertBackwards(common.capacity(), hash, ctrl)) { + return {seq.offset(), /*probe_length=*/0}; + } while (true) { GroupFullEmptyOrDeleted g{ctrl + seq.offset()}; auto mask = g.MaskEmptyOrDeleted(); @@ -3815,11 +3829,17 @@ class raw_hash_set { PolicyTraits::element(slot_array() + seq.offset(i))))) return {iterator_at(seq.offset(i)), false}; } - if (ABSL_PREDICT_TRUE(g.MaskEmpty())) break; + auto mask_empty = g.MaskEmpty(); + if (ABSL_PREDICT_TRUE(mask_empty)) { + size_t target = seq.offset( + GetInsertionOffset(mask_empty, capacity(), hash, control())); + return { + iterator_at(prepare_insert(hash, FindInfo{target, seq.index()})), + true}; + } seq.next(); assert(seq.index() <= capacity() && "full table!"); } - return {iterator_at(prepare_insert(hash)), true}; } protected: @@ -3832,35 +3852,55 @@ class raw_hash_set { return find_or_prepare_insert_non_soo(key); } - // Given the hash of a value not currently in the table, finds the next - // viable slot index to insert it at. + // Given the hash of a value not currently in the table and the first empty + // slot in the probe sequence, finds the viable slot index to insert it at. + // + // In case there's no space left, the table can be resized or rehashed + // (see rehash_and_grow_if_necessary). + // + // In case of absence of deleted slots and positive growth_left, element can + // be inserted in the provided `target` position. + // + // When table has deleted slots (according to GrowthInfo), target position + // will be searched one more time using `find_first_non_full`. // // REQUIRES: At least one non-full slot available. - size_t prepare_insert(size_t hash) ABSL_ATTRIBUTE_NOINLINE { + // REQUIRES: `target` is a valid empty position to insert. + size_t prepare_insert(size_t hash, FindInfo target) ABSL_ATTRIBUTE_NOINLINE { assert(!is_soo()); - const bool rehash_for_bug_detection = - common().should_rehash_for_bug_detection_on_insert(); - if (rehash_for_bug_detection) { - // Move to a different heap allocation in order to detect bugs. - const size_t cap = capacity(); - resize(growth_left() > 0 ? cap : NextCapacity(cap)); - } - FindInfo target; - if (!rehash_for_bug_detection && - ABSL_PREDICT_FALSE(growth_left() == 0)) { - const size_t old_capacity = capacity(); - rehash_and_grow_if_necessary(); - // NOTE: It is safe to use `FindFirstNonFullAfterResize` after - // `rehash_and_grow_if_necessary`, whether capacity changes or not. - // `rehash_and_grow_if_necessary` may *not* call `resize` - // and perform `drop_deletes_without_resize` instead. But this - // could happen only on big tables and will not change capacity. - // For big tables `FindFirstNonFullAfterResize` will always - // fallback to normal `find_first_non_full`. - target = HashSetResizeHelper::FindFirstNonFullAfterResize( - common(), old_capacity, hash); - } else { - target = find_first_non_full(common(), hash); + // When there are no deleted slots in the table + // and growth_left is positive, we can insert at the first + // empty slot in the probe sequence (target). + bool use_target_hint = false; + // Optimization is disabled on enabled generations. + // We have to rehash even sparse tables randomly in such mode. +#ifndef ABSL_SWISSTABLE_ENABLE_GENERATIONS + use_target_hint = growth_info().HasNoDeletedAndGrowthLeft(); +#endif + if (ABSL_PREDICT_FALSE(!use_target_hint)) { + const bool rehash_for_bug_detection = + common().should_rehash_for_bug_detection_on_insert(); + if (rehash_for_bug_detection) { + // Move to a different heap allocation in order to detect bugs. + const size_t cap = capacity(); + resize(growth_left() > 0 ? cap : NextCapacity(cap)); + } + if (!rehash_for_bug_detection && + ABSL_PREDICT_FALSE(growth_left() == 0)) { + const size_t old_capacity = capacity(); + rehash_and_grow_if_necessary(); + // NOTE: It is safe to use `FindFirstNonFullAfterResize` after + // `rehash_and_grow_if_necessary`, whether capacity changes or not. + // `rehash_and_grow_if_necessary` may *not* call `resize` + // and perform `drop_deletes_without_resize` instead. But this + // could happen only on big tables and will not change capacity. + // For big tables `FindFirstNonFullAfterResize` will always + // fallback to normal `find_first_non_full`. + target = HashSetResizeHelper::FindFirstNonFullAfterResize( + common(), old_capacity, hash); + } else { + target = find_first_non_full(common(), hash); + } } PrepareInsertCommon(common()); growth_info().OverwriteControlAsFull(control()[target.offset]); -- cgit v1.2.3 From 160d390608db56f11cbd29bd6b8938661dee525c Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Thu, 28 Mar 2024 10:34:11 -0700 Subject: Roll forward: enable small object optimization in swisstable. PiperOrigin-RevId: 619984581 Change-Id: I68fc9d6e9dd447bdccdbfd270073e11865f85965 --- absl/container/internal/hash_policy_traits.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/hash_policy_traits.h b/absl/container/internal/hash_policy_traits.h index ec08794a..ad835d6f 100644 --- a/absl/container/internal/hash_policy_traits.h +++ b/absl/container/internal/hash_policy_traits.h @@ -168,7 +168,7 @@ struct hash_policy_traits : common_policy_traits { #endif } - // Whether small object optimization is enabled. False by default. + // Whether small object optimization is enabled. True by default. static constexpr bool soo_enabled() { return soo_enabled_impl(Rank1{}); } private: @@ -197,7 +197,7 @@ struct hash_policy_traits : common_policy_traits { return P::soo_enabled(); } - static constexpr bool soo_enabled_impl(Rank0) { return false; } + static constexpr bool soo_enabled_impl(Rank0) { return true; } }; } // namespace container_internal -- cgit v1.2.3 From fbd5fa1781ce6151770557a90d018f71e64e45d0 Mon Sep 17 00:00:00 2001 From: Vitaly Goldshteyn Date: Tue, 2 Apr 2024 12:20:51 -0700 Subject: Fix bug in BM_EraseIf. PiperOrigin-RevId: 621258501 Change-Id: Id094f3f0d0bc4a9fa8f3d1f90cfcd4c53beeb776 --- absl/container/internal/raw_hash_set_benchmark.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set_benchmark.cc b/absl/container/internal/raw_hash_set_benchmark.cc index 05f62d2f..424b72cf 100644 --- a/absl/container/internal/raw_hash_set_benchmark.cc +++ b/absl/container/internal/raw_hash_set_benchmark.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -593,14 +594,17 @@ void BM_EraseIf(benchmark::State& state) { auto& table = tables.back(); for (int64_t i = 0; i < num_elements; i++) { // We use random keys to reduce noise. - k.push_back(absl::Uniform(rng, 0, ~int64_t{})); + k.push_back( + absl::Uniform(rng, 0, std::numeric_limits::max())); if (!table.insert(k.back()).second) { k.pop_back(); --i; // duplicated value, retrying } } std::sort(k.begin(), k.end()); - threshold.push_back(k[num_erased]); + threshold.push_back(static_cast(num_erased) < num_elements + ? k[num_erased] + : std::numeric_limits::max()); } while (state.KeepRunningBatch(static_cast(kRepetitions) * -- cgit v1.2.3 From 9a61b00dde4031f17ed4fa4bdc0e0e9ad8859846 Mon Sep 17 00:00:00 2001 From: Lawrence Wolf-Sonkin Date: Thu, 18 Apr 2024 09:15:45 -0700 Subject: [absl] Re-use the existing `std::type_identity` backfill instead of redefining it again * Specifically, using `absl::internal::type_identity_t` instead of a reimplementation thereof (`NoTypeDeduction`) in the `absl::InlinedVector` code PiperOrigin-RevId: 626055714 Change-Id: I3f5a9a1c25480bc4431edbcc4784e6bc8d257f8d --- absl/container/BUILD.bazel | 1 + absl/container/CMakeLists.txt | 1 + absl/container/internal/inlined_vector.h | 13 ++----------- 3 files changed, 4 insertions(+), 11 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/BUILD.bazel b/absl/container/BUILD.bazel index 12e27c21..2eaece69 100644 --- a/absl/container/BUILD.bazel +++ b/absl/container/BUILD.bazel @@ -126,6 +126,7 @@ cc_library( linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ ":compressed_tuple", + "//absl/base:base_internal", "//absl/base:config", "//absl/base:core_headers", "//absl/memory", diff --git a/absl/container/CMakeLists.txt b/absl/container/CMakeLists.txt index 04055816..576e83e2 100644 --- a/absl/container/CMakeLists.txt +++ b/absl/container/CMakeLists.txt @@ -176,6 +176,7 @@ absl_cc_library( COPTS ${ABSL_DEFAULT_COPTS} DEPS + absl::base_internal absl::compressed_tuple absl::config absl::core_headers diff --git a/absl/container/internal/inlined_vector.h b/absl/container/internal/inlined_vector.h index a1575328..2f24e461 100644 --- a/absl/container/internal/inlined_vector.h +++ b/absl/container/internal/inlined_vector.h @@ -27,6 +27,7 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" +#include "absl/base/internal/identity.h" #include "absl/base/macros.h" #include "absl/container/internal/compressed_tuple.h" #include "absl/memory/memory.h" @@ -82,16 +83,6 @@ using IsMoveAssignOk = std::is_move_assignable>; template using IsSwapOk = absl::type_traits_internal::IsSwappable>; -template -struct TypeIdentity { - using type = T; -}; - -// Used for function arguments in template functions to prevent ADL by forcing -// callers to explicitly specify the template parameter. -template -using NoTypeDeduction = typename TypeIdentity::type; - template >::value> struct DestroyAdapter; @@ -139,7 +130,7 @@ struct MallocAdapter { }; template -void ConstructElements(NoTypeDeduction& allocator, +void ConstructElements(absl::internal::type_identity_t& allocator, Pointer construct_first, ValueAdapter& values, SizeType construct_size) { for (SizeType i = 0; i < construct_size; ++i) { -- cgit v1.2.3 From 6ab5b0aad86dc08d257f6b567611c231c6b8ac31 Mon Sep 17 00:00:00 2001 From: Vitaly Goldshteyn Date: Mon, 20 May 2024 11:57:11 -0700 Subject: Move `prepare_insert` out of the line as type erased `PrepareInsertNonSoo`. This significantly reduces binary size of big binaries and creates a single hot function instead of many cold. That is decreasing cash misses during code execution. We also avoid size related computations for tables with no deleted slots, when resize is necessary. PiperOrigin-RevId: 635527119 Change-Id: I763b135f1f6089051e62e348a07b33536af265ab --- absl/container/internal/compressed_tuple_test.cc | 28 +++ absl/container/internal/raw_hash_set.cc | 168 +++++++++++++++- absl/container/internal/raw_hash_set.h | 244 ++++++++--------------- absl/container/internal/raw_hash_set_test.cc | 16 ++ 4 files changed, 295 insertions(+), 161 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/compressed_tuple_test.cc b/absl/container/internal/compressed_tuple_test.cc index 49818fb8..3cd9e18b 100644 --- a/absl/container/internal/compressed_tuple_test.cc +++ b/absl/container/internal/compressed_tuple_test.cc @@ -15,8 +15,11 @@ #include "absl/container/internal/compressed_tuple.h" #include +#include #include +#include #include +#include #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -55,6 +58,7 @@ namespace { using absl::test_internal::CopyableMovableInstance; using absl::test_internal::InstanceTracker; +using ::testing::Each; TEST(CompressedTupleTest, Sizeof) { EXPECT_EQ(sizeof(int), sizeof(CompressedTuple)); @@ -71,6 +75,30 @@ TEST(CompressedTupleTest, Sizeof) { sizeof(CompressedTuple, NotEmpty, Empty<1>>)); } +TEST(CompressedTupleTest, PointerToEmpty) { + auto to_void_ptrs = [](const auto&... objs) { + return std::vector{static_cast(&objs)...}; + }; + { + using Tuple = CompressedTuple>; + EXPECT_EQ(sizeof(int), sizeof(Tuple)); + Tuple t; + EXPECT_THAT(to_void_ptrs(t.get<1>()), Each(&t)); + } + { + using Tuple = CompressedTuple, Empty<1>>; + EXPECT_EQ(sizeof(int), sizeof(Tuple)); + Tuple t; + EXPECT_THAT(to_void_ptrs(t.get<1>(), t.get<2>()), Each(&t)); + } + { + using Tuple = CompressedTuple, Empty<1>, Empty<2>>; + EXPECT_EQ(sizeof(int), sizeof(Tuple)); + Tuple t; + EXPECT_THAT(to_void_ptrs(t.get<1>(), t.get<2>(), t.get<3>()), Each(&t)); + } +} + TEST(CompressedTupleTest, OneMoveOnRValueConstructionTemp) { InstanceTracker tracker; CompressedTuple x1(CopyableMovableInstance(1)); diff --git a/absl/container/internal/raw_hash_set.cc b/absl/container/internal/raw_hash_set.cc index c23c1f3b..9d399a1b 100644 --- a/absl/container/internal/raw_hash_set.cc +++ b/absl/container/internal/raw_hash_set.cc @@ -23,7 +23,9 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/dynamic_annotations.h" +#include "absl/base/optimization.h" #include "absl/container/internal/container_memory.h" +#include "absl/container/internal/hashtablez_sampler.h" #include "absl/hash/hash.h" namespace absl { @@ -157,6 +159,8 @@ FindInfo find_first_non_full_outofline(const CommonFields& common, return find_first_non_full(common, hash); } +namespace { + // Returns the address of the slot just after slot assuming each slot has the // specified size. static inline void* NextSlot(void* slot, size_t slot_size) { @@ -169,8 +173,22 @@ static inline void* PrevSlot(void* slot, size_t slot_size) { return reinterpret_cast(reinterpret_cast(slot) - slot_size); } -void DropDeletesWithoutResize(CommonFields& common, const void* hash_fn, - const PolicyFunctions& policy, void* tmp_space) { +// Finds guaranteed to exists empty slot from the given position. +// NOTE: this function is almost never triggered inside of the +// DropDeletesWithoutResize, so we keep it simple. +// The table is rather sparse, so empty slot will be found very quickly. +size_t FindEmptySlot(size_t start, size_t end, const ctrl_t* ctrl) { + for (size_t i = start; i < end; ++i) { + if (IsEmpty(ctrl[i])) { + return i; + } + } + assert(false && "no empty slot"); + return ~size_t{}; +} + +void DropDeletesWithoutResize(CommonFields& common, + const PolicyFunctions& policy) { void* set = &common; void* slot_array = common.slot_array(); const size_t capacity = common.capacity(); @@ -194,15 +212,26 @@ void DropDeletesWithoutResize(CommonFields& common, const void* hash_fn, // repeat procedure for current slot with moved from element (target) ctrl_t* ctrl = common.control(); ConvertDeletedToEmptyAndFullToDeleted(ctrl, capacity); + const void* hash_fn = policy.hash_fn(common); auto hasher = policy.hash_slot; auto transfer = policy.transfer; const size_t slot_size = policy.slot_size; size_t total_probe_length = 0; void* slot_ptr = SlotAddress(slot_array, 0, slot_size); + + // The index of an empty slot that can be used as temporary memory for + // the swap operation. + constexpr size_t kUnknownId = ~size_t{}; + size_t tmp_space_id = kUnknownId; + for (size_t i = 0; i != capacity; ++i, slot_ptr = NextSlot(slot_ptr, slot_size)) { assert(slot_ptr == SlotAddress(slot_array, i, slot_size)); + if (IsEmpty(ctrl[i])) { + tmp_space_id = i; + continue; + } if (!IsDeleted(ctrl[i])) continue; const size_t hash = (*hasher)(hash_fn, slot_ptr); const FindInfo target = find_first_non_full(common, hash); @@ -231,16 +260,26 @@ void DropDeletesWithoutResize(CommonFields& common, const void* hash_fn, SetCtrl(common, new_i, H2(hash), slot_size); (*transfer)(set, new_slot_ptr, slot_ptr); SetCtrl(common, i, ctrl_t::kEmpty, slot_size); + // Initialize or change empty space id. + tmp_space_id = i; } else { assert(IsDeleted(ctrl[new_i])); SetCtrl(common, new_i, H2(hash), slot_size); // Until we are done rehashing, DELETED marks previously FULL slots. + if (tmp_space_id == kUnknownId) { + tmp_space_id = FindEmptySlot(i + 1, capacity, ctrl); + } + void* tmp_space = SlotAddress(slot_array, tmp_space_id, slot_size); + SanitizerUnpoisonMemoryRegion(tmp_space, slot_size); + // Swap i and new_i elements. (*transfer)(set, tmp_space, new_slot_ptr); (*transfer)(set, new_slot_ptr, slot_ptr); (*transfer)(set, slot_ptr, tmp_space); + SanitizerPoisonMemoryRegion(tmp_space, slot_size); + // repeat the processing of the ith slot --i; slot_ptr = PrevSlot(slot_ptr, slot_size); @@ -267,6 +306,8 @@ static bool WasNeverFull(CommonFields& c, size_t index) { Group::kWidth; } +} // namespace + void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size) { assert(IsFull(c.control()[index]) && "erasing a dangling iterator"); c.decrement_size(); @@ -424,6 +465,129 @@ void HashSetResizeHelper::TransferSlotAfterSoo(CommonFields& c, PoisonSingleGroupEmptySlots(c, slot_size); } +namespace { + +// Called whenever the table needs to vacate empty slots either by removing +// tombstones via rehash or growth. +ABSL_ATTRIBUTE_NOINLINE +FindInfo FindInsertPositionWithGrowthOrRehash(CommonFields& common, size_t hash, + const PolicyFunctions& policy) { + const size_t cap = common.capacity(); + if (cap > Group::kWidth && + // Do these calculations in 64-bit to avoid overflow. + common.size() * uint64_t{32} <= cap * uint64_t{25}) { + // Squash DELETED without growing if there is enough capacity. + // + // Rehash in place if the current size is <= 25/32 of capacity. + // Rationale for such a high factor: 1) DropDeletesWithoutResize() is + // faster than resize, and 2) it takes quite a bit of work to add + // tombstones. In the worst case, seems to take approximately 4 + // insert/erase pairs to create a single tombstone and so if we are + // rehashing because of tombstones, we can afford to rehash-in-place as + // long as we are reclaiming at least 1/8 the capacity without doing more + // than 2X the work. (Where "work" is defined to be size() for rehashing + // or rehashing in place, and 1 for an insert or erase.) But rehashing in + // place is faster per operation than inserting or even doubling the size + // of the table, so we actually afford to reclaim even less space from a + // resize-in-place. The decision is to rehash in place if we can reclaim + // at about 1/8th of the usable capacity (specifically 3/28 of the + // capacity) which means that the total cost of rehashing will be a small + // fraction of the total work. + // + // Here is output of an experiment using the BM_CacheInSteadyState + // benchmark running the old case (where we rehash-in-place only if we can + // reclaim at least 7/16*capacity) vs. this code (which rehashes in place + // if we can recover 3/32*capacity). + // + // Note that although in the worst-case number of rehashes jumped up from + // 15 to 190, but the number of operations per second is almost the same. + // + // Abridged output of running BM_CacheInSteadyState benchmark from + // raw_hash_set_benchmark. N is the number of insert/erase operations. + // + // | OLD (recover >= 7/16 | NEW (recover >= 3/32) + // size | N/s LoadFactor NRehashes | N/s LoadFactor NRehashes + // 448 | 145284 0.44 18 | 140118 0.44 19 + // 493 | 152546 0.24 11 | 151417 0.48 28 + // 538 | 151439 0.26 11 | 151152 0.53 38 + // 583 | 151765 0.28 11 | 150572 0.57 50 + // 628 | 150241 0.31 11 | 150853 0.61 66 + // 672 | 149602 0.33 12 | 150110 0.66 90 + // 717 | 149998 0.35 12 | 149531 0.70 129 + // 762 | 149836 0.37 13 | 148559 0.74 190 + // 807 | 149736 0.39 14 | 151107 0.39 14 + // 852 | 150204 0.42 15 | 151019 0.42 15 + DropDeletesWithoutResize(common, policy); + } else { + // Otherwise grow the container. + policy.resize(common, NextCapacity(cap), HashtablezInfoHandle{}); + } + // This function is typically called with tables containing deleted slots. + // The table will be big and `FindFirstNonFullAfterResize` will always + // fallback to `find_first_non_full`. So using `find_first_non_full` directly. + return find_first_non_full(common, hash); +} + +} // namespace + +const void* GetHashRefForEmptyHasher(const CommonFields& common) { + // Empty base optimization typically make the empty base class address to be + // the same as the first address of the derived class object. + // But we generally assume that for empty hasher we can return any valid + // pointer. + return &common; +} + +size_t PrepareInsertNonSoo(CommonFields& common, size_t hash, FindInfo target, + const PolicyFunctions& policy) { + // When there are no deleted slots in the table + // and growth_left is positive, we can insert at the first + // empty slot in the probe sequence (target). + const bool use_target_hint = + // Optimization is disabled when generations are enabled. + // We have to rehash even sparse tables randomly in such mode. + !SwisstableGenerationsEnabled() && + common.growth_info().HasNoDeletedAndGrowthLeft(); + if (ABSL_PREDICT_FALSE(!use_target_hint)) { + // Notes about optimized mode when generations are disabled: + // We do not enter this branch if table has no deleted slots + // and growth_left is positive. + // We enter this branch in the following cases listed in decreasing + // frequency: + // 1. Table without deleted slots (>95% cases) that needs to be resized. + // 2. Table with deleted slots that has space for the inserting element. + // 3. Table with deleted slots that needs to be rehashed or resized. + if (ABSL_PREDICT_TRUE(common.growth_info().HasNoGrowthLeftAndNoDeleted())) { + const size_t old_capacity = common.capacity(); + policy.resize(common, NextCapacity(old_capacity), HashtablezInfoHandle{}); + target = HashSetResizeHelper::FindFirstNonFullAfterResize( + common, old_capacity, hash); + } else { + // Note: the table may have no deleted slots here when generations + // are enabled. + const bool rehash_for_bug_detection = + common.should_rehash_for_bug_detection_on_insert(); + if (rehash_for_bug_detection) { + // Move to a different heap allocation in order to detect bugs. + const size_t cap = common.capacity(); + policy.resize(common, + common.growth_left() > 0 ? cap : NextCapacity(cap), + HashtablezInfoHandle{}); + } + if (ABSL_PREDICT_TRUE(common.growth_left() > 0)) { + target = find_first_non_full(common, hash); + } else { + target = FindInsertPositionWithGrowthOrRehash(common, hash, policy); + } + } + } + PrepareInsertCommon(common); + common.growth_info().OverwriteControlAsFull(common.control()[target.offset]); + SetCtrl(common, target.offset, H2(hash), policy.slot_size); + common.infoz().RecordInsert(hash, target.probe_length); + return target.offset; +} + } // namespace container_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 1d2e2d14..a64133a7 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -1126,6 +1126,13 @@ class GrowthInfo { return static_cast>(growth_left_info_) > 0; } + // Returns true if the table satisfies two properties: + // 1. Guaranteed to have no kDeleted slots. + // 2. There is no growth left. + bool HasNoGrowthLeftAndNoDeleted() const { + return growth_left_info_ == 0; + } + // Returns true if table guaranteed to have no k bool HasNoDeleted() const { return static_cast>(growth_left_info_) >= 0; @@ -1374,6 +1381,8 @@ class CommonFields : public CommonFieldsGenerationInfo { // This is stored in the heap allocation before the control bytes. // TODO(b/289225379): experiment with moving growth_info back inline to // increase room for SOO. + size_t growth_left() const { return growth_info().GetGrowthLeft(); } + GrowthInfo& growth_info() { auto* gl_ptr = reinterpret_cast(control()) - 1; assert(reinterpret_cast(gl_ptr) % alignof(GrowthInfo) == 0); @@ -1933,8 +1942,7 @@ class HashSetResizeHelper { // will be no performance benefit. // It has implicit assumption that `resize` will call // `GrowSizeIntoSingleGroup*` in case `IsGrowingIntoSingleGroupApplicable`. - // Falls back to `find_first_non_full` in case of big groups, so it is - // safe to use after `rehash_and_grow_if_necessary`. + // Falls back to `find_first_non_full` in case of big groups. static FindInfo FindFirstNonFullAfterResize(const CommonFields& c, size_t old_capacity, size_t hash) { @@ -2216,14 +2224,22 @@ size_t PrepareInsertAfterSoo(size_t hash, size_t slot_size, struct PolicyFunctions { size_t slot_size; + // Returns the pointer to the hash function stored in the set. + const void* (*hash_fn)(const CommonFields& common); + // Returns the hash of the pointed-to slot. size_t (*hash_slot)(const void* hash_fn, void* slot); - // Transfer the contents of src_slot to dst_slot. + // Transfers the contents of src_slot to dst_slot. void (*transfer)(void* set, void* dst_slot, void* src_slot); - // Deallocate the backing store from common. + // Deallocates the backing store from common. void (*dealloc)(CommonFields& common, const PolicyFunctions& policy); + + // Resizes set to the new capacity. + // Arguments are used as in raw_hash_set::resize_impl. + void (*resize)(CommonFields& common, size_t new_capacity, + HashtablezInfoHandle forced_infoz); }; // ClearBackingArray clears the backing array, either modifying it in place, @@ -2261,9 +2277,26 @@ ABSL_ATTRIBUTE_NOINLINE void TransferRelocatable(void*, void* dst, void* src) { memcpy(dst, src, SizeOfSlot); } -// Type-erased version of raw_hash_set::drop_deletes_without_resize. -void DropDeletesWithoutResize(CommonFields& common, const void* hash_fn, - const PolicyFunctions& policy, void* tmp_space); +// Type erased raw_hash_set::get_hash_ref_fn for the empty hash function case. +const void* GetHashRefForEmptyHasher(const CommonFields& common); + +// Given the hash of a value not currently in the table and the first empty +// slot in the probe sequence, finds a viable slot index to insert it at. +// +// In case there's no space left, the table can be resized or rehashed +// (for tables with deleted slots, see FindInsertPositionWithGrowthOrRehash). +// +// In the case of absence of deleted slots and positive growth_left, the element +// can be inserted in the provided `target` position. +// +// When the table has deleted slots (according to GrowthInfo), the target +// position will be searched one more time using `find_first_non_full`. +// +// REQUIRES: Table is not SOO. +// REQUIRES: At least one non-full slot available. +// REQUIRES: `target` is a valid empty position to insert. +size_t PrepareInsertNonSoo(CommonFields& common, size_t hash, FindInfo target, + const PolicyFunctions& policy); // A SwissTable. // @@ -3533,7 +3566,7 @@ class raw_hash_set { // common(), old_capacity, hash) // can be called right after `resize`. void resize(size_t new_capacity) { - resize_impl(new_capacity, HashtablezInfoHandle{}); + raw_hash_set::resize_impl(common(), new_capacity, HashtablezInfoHandle{}); } // As above, except that we also accept a pre-sampled, forced infoz for @@ -3541,19 +3574,24 @@ class raw_hash_set { // store the infoz. void resize_with_soo_infoz(HashtablezInfoHandle forced_infoz) { assert(forced_infoz.IsSampled()); - resize_impl(NextCapacity(SooCapacity()), forced_infoz); + raw_hash_set::resize_impl(common(), NextCapacity(SooCapacity()), + forced_infoz); } - ABSL_ATTRIBUTE_NOINLINE void resize_impl( - size_t new_capacity, HashtablezInfoHandle forced_infoz) { + // Resizes set to the new capacity. + // It is a static function in order to use its pointer in GetPolicyFunctions. + ABSL_ATTRIBUTE_NOINLINE static void resize_impl( + CommonFields& common, size_t new_capacity, + HashtablezInfoHandle forced_infoz) { + raw_hash_set* set = reinterpret_cast(&common); assert(IsValidCapacity(new_capacity)); - assert(!fits_in_soo(new_capacity)); - const bool was_soo = is_soo(); - const bool had_soo_slot = was_soo && !empty(); + assert(!set->fits_in_soo(new_capacity)); + const bool was_soo = set->is_soo(); + const bool had_soo_slot = was_soo && !set->empty(); const ctrl_t soo_slot_h2 = - had_soo_slot ? static_cast(H2(hash_of(soo_slot()))) + had_soo_slot ? static_cast(H2(set->hash_of(set->soo_slot()))) : ctrl_t::kEmpty; - HashSetResizeHelper resize_helper(common(), was_soo, had_soo_slot, + HashSetResizeHelper resize_helper(common, was_soo, had_soo_slot, forced_infoz); // Initialize HashSetResizeHelper::old_heap_or_soo_. We can't do this in // HashSetResizeHelper constructor because it can't transfer slots when @@ -3561,11 +3599,12 @@ class raw_hash_set { // TODO(b/289225379): try to handle more of the SOO cases inside // InitializeSlots. See comment on cl/555990034 snapshot #63. if (PolicyTraits::transfer_uses_memcpy() || !had_soo_slot) { - resize_helper.old_heap_or_soo() = common().heap_or_soo(); + resize_helper.old_heap_or_soo() = common.heap_or_soo(); } else { - transfer(to_slot(resize_helper.old_soo_data()), soo_slot()); + set->transfer(set->to_slot(resize_helper.old_soo_data()), + set->soo_slot()); } - common().set_capacity(new_capacity); + common.set_capacity(new_capacity); // Note that `InitializeSlots` does different number initialization steps // depending on the values of `transfer_uses_memcpy` and capacities. // Refer to the comment in `InitializeSlots` for more details. @@ -3573,7 +3612,7 @@ class raw_hash_set { resize_helper.InitializeSlots( - common(), CharAlloc(alloc_ref()), soo_slot_h2, sizeof(key_type), + common, CharAlloc(set->alloc_ref()), soo_slot_h2, sizeof(key_type), sizeof(value_type)); // In the SooEnabled() case, capacity is never 0 so we don't check. @@ -3585,30 +3624,30 @@ class raw_hash_set { // Nothing more to do in this case. if (was_soo && !had_soo_slot) return; - slot_type* new_slots = slot_array(); + slot_type* new_slots = set->slot_array(); if (grow_single_group) { if (PolicyTraits::transfer_uses_memcpy()) { // InitializeSlots did all the work. return; } if (was_soo) { - transfer(new_slots + resize_helper.SooSlotIndex(), - to_slot(resize_helper.old_soo_data())); + set->transfer(new_slots + resize_helper.SooSlotIndex(), + to_slot(resize_helper.old_soo_data())); return; } else { // We want GrowSizeIntoSingleGroup to be called here in order to make // InitializeSlots not depend on PolicyTraits. - resize_helper.GrowSizeIntoSingleGroup(common(), - alloc_ref()); + resize_helper.GrowSizeIntoSingleGroup(common, + set->alloc_ref()); } } else { // InitializeSlots prepares control bytes to correspond to empty table. const auto insert_slot = [&](slot_type* slot) { - size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, + size_t hash = PolicyTraits::apply(HashElement{set->hash_ref()}, PolicyTraits::element(slot)); - auto target = find_first_non_full(common(), hash); - SetCtrl(common(), target.offset, H2(hash), sizeof(slot_type)); - transfer(new_slots + target.offset, slot); + auto target = find_first_non_full(common, hash); + SetCtrl(common, target.offset, H2(hash), sizeof(slot_type)); + set->transfer(new_slots + target.offset, slot); return target.probe_length; }; if (was_soo) { @@ -3623,80 +3662,13 @@ class raw_hash_set { total_probe_length += insert_slot(old_slots + i); } } - infoz().RecordRehash(total_probe_length); + common.infoz().RecordRehash(total_probe_length); } } - resize_helper.DeallocateOld(CharAlloc(alloc_ref()), + resize_helper.DeallocateOld(CharAlloc(set->alloc_ref()), sizeof(slot_type)); } - // Prunes control bytes to remove as many tombstones as possible. - // - // See the comment on `rehash_and_grow_if_necessary()`. - inline void drop_deletes_without_resize() { - // Stack-allocate space for swapping elements. - alignas(slot_type) unsigned char tmp[sizeof(slot_type)]; - DropDeletesWithoutResize(common(), &hash_ref(), GetPolicyFunctions(), tmp); - } - - // Called whenever the table *might* need to conditionally grow. - // - // This function is an optimization opportunity to perform a rehash even when - // growth is unnecessary, because vacating tombstones is beneficial for - // performance in the long-run. - void rehash_and_grow_if_necessary() { - const size_t cap = capacity(); - if (cap > Group::kWidth && - // Do these calculations in 64-bit to avoid overflow. - size() * uint64_t{32} <= cap * uint64_t{25}) { - // Squash DELETED without growing if there is enough capacity. - // - // Rehash in place if the current size is <= 25/32 of capacity. - // Rationale for such a high factor: 1) drop_deletes_without_resize() is - // faster than resize, and 2) it takes quite a bit of work to add - // tombstones. In the worst case, seems to take approximately 4 - // insert/erase pairs to create a single tombstone and so if we are - // rehashing because of tombstones, we can afford to rehash-in-place as - // long as we are reclaiming at least 1/8 the capacity without doing more - // than 2X the work. (Where "work" is defined to be size() for rehashing - // or rehashing in place, and 1 for an insert or erase.) But rehashing in - // place is faster per operation than inserting or even doubling the size - // of the table, so we actually afford to reclaim even less space from a - // resize-in-place. The decision is to rehash in place if we can reclaim - // at about 1/8th of the usable capacity (specifically 3/28 of the - // capacity) which means that the total cost of rehashing will be a small - // fraction of the total work. - // - // Here is output of an experiment using the BM_CacheInSteadyState - // benchmark running the old case (where we rehash-in-place only if we can - // reclaim at least 7/16*capacity) vs. this code (which rehashes in place - // if we can recover 3/32*capacity). - // - // Note that although in the worst-case number of rehashes jumped up from - // 15 to 190, but the number of operations per second is almost the same. - // - // Abridged output of running BM_CacheInSteadyState benchmark from - // raw_hash_set_benchmark. N is the number of insert/erase operations. - // - // | OLD (recover >= 7/16 | NEW (recover >= 3/32) - // size | N/s LoadFactor NRehashes | N/s LoadFactor NRehashes - // 448 | 145284 0.44 18 | 140118 0.44 19 - // 493 | 152546 0.24 11 | 151417 0.48 28 - // 538 | 151439 0.26 11 | 151152 0.53 38 - // 583 | 151765 0.28 11 | 150572 0.57 50 - // 628 | 150241 0.31 11 | 150853 0.61 66 - // 672 | 149602 0.33 12 | 150110 0.66 90 - // 717 | 149998 0.35 12 | 149531 0.70 129 - // 762 | 149836 0.37 13 | 148559 0.74 190 - // 807 | 149736 0.39 14 | 151107 0.39 14 - // 852 | 150204 0.42 15 | 151019 0.42 15 - drop_deletes_without_resize(); - } else { - // Otherwise grow the container. - resize(NextCapacity(cap)); - } - } - // Casting directly from e.g. char* to slot_type* can cause compilation errors // on objective-C. This function converts to void* first, avoiding the issue. static slot_type* to_slot(void* buf) { @@ -3817,6 +3789,7 @@ class raw_hash_set { template std::pair find_or_prepare_insert_non_soo(const K& key) { + assert(!is_soo()); prefetch_heap_block(); auto hash = hash_ref()(key); auto seq = probe(common(), hash); @@ -3833,9 +3806,10 @@ class raw_hash_set { if (ABSL_PREDICT_TRUE(mask_empty)) { size_t target = seq.offset( GetInsertionOffset(mask_empty, capacity(), hash, control())); - return { - iterator_at(prepare_insert(hash, FindInfo{target, seq.index()})), - true}; + return {iterator_at(PrepareInsertNonSoo(common(), hash, + FindInfo{target, seq.index()}, + GetPolicyFunctions())), + true}; } seq.next(); assert(seq.index() <= capacity() && "full table!"); @@ -3852,63 +3826,6 @@ class raw_hash_set { return find_or_prepare_insert_non_soo(key); } - // Given the hash of a value not currently in the table and the first empty - // slot in the probe sequence, finds the viable slot index to insert it at. - // - // In case there's no space left, the table can be resized or rehashed - // (see rehash_and_grow_if_necessary). - // - // In case of absence of deleted slots and positive growth_left, element can - // be inserted in the provided `target` position. - // - // When table has deleted slots (according to GrowthInfo), target position - // will be searched one more time using `find_first_non_full`. - // - // REQUIRES: At least one non-full slot available. - // REQUIRES: `target` is a valid empty position to insert. - size_t prepare_insert(size_t hash, FindInfo target) ABSL_ATTRIBUTE_NOINLINE { - assert(!is_soo()); - // When there are no deleted slots in the table - // and growth_left is positive, we can insert at the first - // empty slot in the probe sequence (target). - bool use_target_hint = false; - // Optimization is disabled on enabled generations. - // We have to rehash even sparse tables randomly in such mode. -#ifndef ABSL_SWISSTABLE_ENABLE_GENERATIONS - use_target_hint = growth_info().HasNoDeletedAndGrowthLeft(); -#endif - if (ABSL_PREDICT_FALSE(!use_target_hint)) { - const bool rehash_for_bug_detection = - common().should_rehash_for_bug_detection_on_insert(); - if (rehash_for_bug_detection) { - // Move to a different heap allocation in order to detect bugs. - const size_t cap = capacity(); - resize(growth_left() > 0 ? cap : NextCapacity(cap)); - } - if (!rehash_for_bug_detection && - ABSL_PREDICT_FALSE(growth_left() == 0)) { - const size_t old_capacity = capacity(); - rehash_and_grow_if_necessary(); - // NOTE: It is safe to use `FindFirstNonFullAfterResize` after - // `rehash_and_grow_if_necessary`, whether capacity changes or not. - // `rehash_and_grow_if_necessary` may *not* call `resize` - // and perform `drop_deletes_without_resize` instead. But this - // could happen only on big tables and will not change capacity. - // For big tables `FindFirstNonFullAfterResize` will always - // fallback to normal `find_first_non_full`. - target = HashSetResizeHelper::FindFirstNonFullAfterResize( - common(), old_capacity, hash); - } else { - target = find_first_non_full(common(), hash); - } - } - PrepareInsertCommon(common()); - growth_info().OverwriteControlAsFull(control()[target.offset]); - SetCtrl(common(), target.offset, H2(hash), sizeof(slot_type)); - infoz().RecordInsert(hash, target.probe_length); - return target.offset; - } - // Constructs the value in the space pointed by the iterator. This only works // after an unsuccessful find_or_prepare_insert() and before any other // modifications happen in the raw_hash_set. @@ -3949,7 +3866,7 @@ class raw_hash_set { // See `CapacityToGrowth()`. size_t growth_left() const { assert(!is_soo()); - return growth_info().GetGrowthLeft(); + return common().growth_left(); } GrowthInfo& growth_info() { @@ -4009,6 +3926,10 @@ class raw_hash_set { return settings_.template get<3>(); } + static const void* get_hash_ref_fn(const CommonFields& common) { + auto* h = reinterpret_cast(&common); + return &h->hash_ref(); + } static void transfer_slot_fn(void* set, void* dst, void* src) { auto* h = static_cast(set); h->transfer(static_cast(dst), static_cast(src)); @@ -4030,6 +3951,10 @@ class raw_hash_set { static const PolicyFunctions& GetPolicyFunctions() { static constexpr PolicyFunctions value = { sizeof(slot_type), + // TODO(b/328722020): try to type erase + // for standard layout and alignof(Hash) <= alignof(CommonFields). + std::is_empty::value ? &GetHashRefForEmptyHasher + : &raw_hash_set::get_hash_ref_fn, PolicyTraits::template get_hash_slot_fn(), PolicyTraits::transfer_uses_memcpy() ? TransferRelocatable @@ -4037,6 +3962,7 @@ class raw_hash_set { (std::is_same>::value ? &DeallocateStandard : &raw_hash_set::dealloc_fn), + &raw_hash_set::resize_impl, }; return value; } diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index c4e05d60..10f793ef 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -126,6 +126,22 @@ TEST(GrowthInfoTest, HasNoDeletedAndGrowthLeft) { EXPECT_TRUE(gi.HasNoDeletedAndGrowthLeft()); } +TEST(GrowthInfoTest, HasNoGrowthLeftAndNoDeleted) { + GrowthInfo gi; + gi.InitGrowthLeftNoDeleted(1); + EXPECT_FALSE(gi.HasNoGrowthLeftAndNoDeleted()); + gi.OverwriteEmptyAsFull(); + EXPECT_TRUE(gi.HasNoGrowthLeftAndNoDeleted()); + gi.OverwriteFullAsDeleted(); + EXPECT_FALSE(gi.HasNoGrowthLeftAndNoDeleted()); + gi.OverwriteFullAsEmpty(); + EXPECT_FALSE(gi.HasNoGrowthLeftAndNoDeleted()); + gi.InitGrowthLeftNoDeleted(0); + EXPECT_TRUE(gi.HasNoGrowthLeftAndNoDeleted()); + gi.OverwriteFullAsEmpty(); + EXPECT_FALSE(gi.HasNoGrowthLeftAndNoDeleted()); +} + TEST(GrowthInfoTest, OverwriteFullAsEmpty) { GrowthInfo gi; gi.InitGrowthLeftNoDeleted(5); -- cgit v1.2.3 From 1a31b81c0a467c1c8e229b9fc172a4eb0db5bd85 Mon Sep 17 00:00:00 2001 From: Paul Rigge Date: Wed, 22 May 2024 10:39:28 -0700 Subject: Rework casting in raw_hash_set's IsFull(). PiperOrigin-RevId: 636218177 Change-Id: I9f58ccbb468fcc0c44ef12162415f7b721a745bf --- absl/container/internal/raw_hash_set.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index a64133a7..1f677a4e 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -623,7 +623,12 @@ inline h2_t H2(size_t hash) { return hash & 0x7F; } // Helpers for checking the state of a control byte. inline bool IsEmpty(ctrl_t c) { return c == ctrl_t::kEmpty; } -inline bool IsFull(ctrl_t c) { return c >= static_cast(0); } +inline bool IsFull(ctrl_t c) { + // Cast `c` to the underlying type instead of casting `0` to `ctrl_t` as `0` + // is not a value in the enum. Both ways are equivalent, but this way makes + // linters happier. + return static_cast>(c) >= 0; +} inline bool IsDeleted(ctrl_t c) { return c == ctrl_t::kDeleted; } inline bool IsEmptyOrDeleted(ctrl_t c) { return c < ctrl_t::kSentinel; } -- cgit v1.2.3 From 40b2776ef9a2f0cff6e7c6fc4ab79c1ebd87c554 Mon Sep 17 00:00:00 2001 From: Connal de Souza Date: Tue, 28 May 2024 16:52:45 -0700 Subject: Optimize GrowIntoSingleGroupShuffleControlBytes. This implementation is designed to avoid needing to copy to an intermediate buffer and then read from it again, which is an expensive Read-after-Write hazard. PiperOrigin-RevId: 638071429 Change-Id: I390b4d38b8c1bd7fffba3d403baba6f1511555b0 --- absl/container/internal/raw_hash_set.cc | 150 +++++++++++++++++++++----------- 1 file changed, 100 insertions(+), 50 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.cc b/absl/container/internal/raw_hash_set.cc index 9d399a1b..1cae0381 100644 --- a/absl/container/internal/raw_hash_set.cc +++ b/absl/container/internal/raw_hash_set.cc @@ -23,6 +23,7 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/dynamic_annotations.h" +#include "absl/base/internal/endian.h" #include "absl/base/optimization.h" #include "absl/container/internal/container_memory.h" #include "absl/container/internal/hashtablez_sampler.h" @@ -342,77 +343,126 @@ void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy, } void HashSetResizeHelper::GrowIntoSingleGroupShuffleControlBytes( - ctrl_t* new_ctrl, size_t new_capacity) const { + ctrl_t* __restrict new_ctrl, size_t new_capacity) const { assert(is_single_group(new_capacity)); constexpr size_t kHalfWidth = Group::kWidth / 2; + constexpr size_t kQuarterWidth = Group::kWidth / 4; assert(old_capacity_ < kHalfWidth); + static_assert(sizeof(uint64_t) >= kHalfWidth, + "Group size is too large. The ctrl bytes for half a group must " + "fit into a uint64_t for this implementation."); + static_assert(sizeof(uint64_t) <= Group::kWidth, + "Group size is too small. The ctrl bytes for a group must " + "cover a uint64_t for this implementation."); const size_t half_old_capacity = old_capacity_ / 2; // NOTE: operations are done with compile time known size = kHalfWidth. // Compiler optimizes that into single ASM operation. - // Copy second half of bytes to the beginning. - // We potentially copy more bytes in order to have compile time known size. - // Mirrored bytes from the old_ctrl() will also be copied. - // In case of old_capacity_ == 3, we will copy 1st element twice. + // Load the bytes from half_old_capacity + 1. This contains the last half of + // old_ctrl bytes, followed by the sentinel byte, and then the first half of + // the cloned bytes. This effectively shuffles the control bytes. + uint64_t copied_bytes = 0; + copied_bytes = + absl::little_endian::Load64(old_ctrl() + half_old_capacity + 1); + + // We change the sentinel byte to kEmpty before storing to both the start of + // the new_ctrl, and past the end of the new_ctrl later for the new cloned + // bytes. Note that this is faster than setting the sentinel byte to kEmpty + // after the copy directly in new_ctrl because we are limited on store + // bandwidth. + constexpr uint64_t kEmptyXorSentinel = + static_cast(ctrl_t::kEmpty) ^ + static_cast(ctrl_t::kSentinel); + const uint64_t mask_convert_old_sentinel_to_empty = + kEmptyXorSentinel << (half_old_capacity * 8); + copied_bytes ^= mask_convert_old_sentinel_to_empty; + + // Copy second half of bytes to the beginning. This correctly sets the bytes + // [0, old_capacity]. We potentially copy more bytes in order to have compile + // time known size. Mirrored bytes from the old_ctrl() will also be copied. In + // case of old_capacity_ == 3, we will copy 1st element twice. // Examples: + // (old capacity = 1) // old_ctrl = 0S0EEEEEEE... - // new_ctrl = S0EEEEEEEE... + // new_ctrl = E0EEEEEE??... // - // old_ctrl = 01S01EEEEE... - // new_ctrl = 1S01EEEEEE... + // (old capacity = 3) + // old_ctrl = 012S012EEEEE... + // new_ctrl = 12E012EE????... // + // (old capacity = 7) // old_ctrl = 0123456S0123456EE... - // new_ctrl = 456S0123?????????... - std::memcpy(new_ctrl, old_ctrl() + half_old_capacity + 1, kHalfWidth); - // Clean up copied kSentinel from old_ctrl. - new_ctrl[half_old_capacity] = ctrl_t::kEmpty; - - // Clean up damaged or uninitialized bytes. - - // Clean bytes after the intended size of the copy. - // Example: - // new_ctrl = 1E01EEEEEEE???? - // *new_ctrl= 1E0EEEEEEEE???? - // position / - std::memset(new_ctrl + old_capacity_ + 1, static_cast(ctrl_t::kEmpty), - kHalfWidth); - // Clean non-mirrored bytes that are not initialized. - // For small old_capacity that may be inside of mirrored bytes zone. + // new_ctrl = 456E0123?????????... + absl::little_endian::Store64(new_ctrl, copied_bytes); + + // Set the space [old_capacity + 1, new_capacity] to empty as these bytes will + // not be written again. This is safe because + // NumControlBytes = new_capacity + kWidth and new_capacity >= + // old_capacity+1. // Examples: - // new_ctrl = 1E0EEEEEEEE??????????.... - // *new_ctrl= 1E0EEEEEEEEEEEEE?????.... - // position / + // (old_capacity = 3, new_capacity = 15) + // new_ctrl = 12E012EE?????????????...?? + // *new_ctrl = 12E0EEEEEEEEEEEEEEEE?...?? + // position / S // - // new_ctrl = 456E0123???????????... - // *new_ctrl= 456E0123EEEEEEEE???... - // position / - std::memset(new_ctrl + kHalfWidth, static_cast(ctrl_t::kEmpty), - kHalfWidth); - // Clean last mirrored bytes that are not initialized - // and will not be overwritten by mirroring. + // (old_capacity = 7, new_capacity = 15) + // new_ctrl = 456E0123?????????????????...?? + // *new_ctrl = 456E0123EEEEEEEEEEEEEEEE?...?? + // position / S + std::memset(new_ctrl + old_capacity_ + 1, static_cast(ctrl_t::kEmpty), + Group::kWidth); + + // Set the last kHalfWidth bytes to empty, to ensure the bytes all the way to + // the end are initialized. // Examples: - // new_ctrl = 1E0EEEEEEEEEEEEE???????? - // *new_ctrl= 1E0EEEEEEEEEEEEEEEEEEEEE - // position S / + // new_ctrl = 12E0EEEEEEEEEEEEEEEE?...??????? + // *new_ctrl = 12E0EEEEEEEEEEEEEEEE???EEEEEEEE + // position S / // - // new_ctrl = 456E0123EEEEEEEE??????????????? - // *new_ctrl= 456E0123EEEEEEEE???????EEEEEEEE - // position S / - std::memset(new_ctrl + new_capacity + kHalfWidth, + // new_ctrl = 456E0123EEEEEEEEEEEEEEEE??????? + // *new_ctrl = 456E0123EEEEEEEEEEEEEEEEEEEEEEE + // position S / + std::memset(new_ctrl + NumControlBytes(new_capacity) - kHalfWidth, static_cast(ctrl_t::kEmpty), kHalfWidth); - // Create mirrored bytes. old_capacity_ < kHalfWidth - // Example: - // new_ctrl = 456E0123EEEEEEEE???????EEEEEEEE - // *new_ctrl= 456E0123EEEEEEEE456E0123EEEEEEE - // position S/ - ctrl_t g[kHalfWidth]; - std::memcpy(g, new_ctrl, kHalfWidth); - std::memcpy(new_ctrl + new_capacity + 1, g, kHalfWidth); + // Copy the first bytes to the end (starting at new_capacity +1) to set the + // cloned bytes. Note that we use the already copied bytes from old_ctrl here + // rather than copying from new_ctrl to avoid a Read-after-Write hazard, since + // new_ctrl was just written to. The first old_capacity-1 bytes are set + // correctly. Then there may be up to old_capacity bytes that need to be + // overwritten, and any remaining bytes will be correctly set to empty. This + // sets [new_capacity + 1, new_capacity +1 + old_capacity] correctly. + // Examples: + // new_ctrl = 12E0EEEEEEEEEEEEEEEE?...??????? + // *new_ctrl = 12E0EEEEEEEEEEEE12E012EEEEEEEEE + // position S/ + // + // new_ctrl = 456E0123EEEEEEEE?...???EEEEEEEE + // *new_ctrl = 456E0123EEEEEEEE456E0123EEEEEEE + // position S/ + absl::little_endian::Store64(new_ctrl + new_capacity + 1, copied_bytes); + + // Set The remaining bytes at the end past the cloned bytes to empty. The + // incorrectly set bytes are [new_capacity + old_capacity + 2, + // min(new_capacity + 1 + kHalfWidth, new_capacity + old_capacity + 2 + + // half_old_capacity)]. Taking the difference, we need to set min(kHalfWidth - + // (old_capacity + 1), half_old_capacity)]. Since old_capacity < kHalfWidth, + // half_old_capacity < kQuarterWidth, so we set kQuarterWidth beginning at + // new_capacity + old_capacity + 2 to kEmpty. + // Examples: + // new_ctrl = 12E0EEEEEEEEEEEE12E012EEEEEEEEE + // *new_ctrl = 12E0EEEEEEEEEEEE12E0EEEEEEEEEEE + // position S / + // + // new_ctrl = 456E0123EEEEEEEE456E0123EEEEEEE + // *new_ctrl = 456E0123EEEEEEEE456E0123EEEEEEE (no change) + // position S / + std::memset(new_ctrl + new_capacity + old_capacity_ + 2, + static_cast(ctrl_t::kEmpty), kQuarterWidth); - // Finally set sentinel to its place. + // Finally, we set the new sentinel byte. new_ctrl[new_capacity] = ctrl_t::kSentinel; } -- cgit v1.2.3 From 65dfbf2b51f0d152b7beb9c93186992d347b37a5 Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Mon, 3 Jun 2024 13:27:55 -0700 Subject: Clarify function comment for `erase` by stating that this idiom only works for "some" standard containers. If you use this idiom with `std::vector` or `absl::btree_map` you can end up either skipping elements or dereferencing an invalid iterator. PiperOrigin-RevId: 639892758 Change-Id: Ic5c213667b4b1e8c39813ee237aaffe320a8eb27 --- absl/container/internal/raw_hash_set.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 1f677a4e..9ef9a6be 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -3105,7 +3105,7 @@ class raw_hash_set { // this method returns void to reduce algorithmic complexity to O(1). The // iterator is invalidated, so any increment should be done before calling // erase. In order to erase while iterating across a map, use the following - // idiom (which also works for standard containers): + // idiom (which also works for some standard containers): // // for (auto it = m.begin(), end = m.end(); it != end;) { // // `erase()` will invalidate `it`, so advance `it` first. -- cgit v1.2.3 From 9645a2fb848516b3dec22afb60f4a7a860fe35d1 Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Tue, 4 Jun 2024 10:49:37 -0700 Subject: Remove redundant check of is_soo() while prefetching heap blocks. PiperOrigin-RevId: 640208455 Change-Id: I4c2b7d3f1adad2078e8a5f805574f71c4d7743d4 --- absl/container/internal/raw_hash_set.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 9ef9a6be..2ea07e40 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -3887,7 +3887,7 @@ class raw_hash_set { // cache misses. This is intended to overlap with execution of calculating the // hash for a key. void prefetch_heap_block() const { - if (is_soo()) return; + assert(!is_soo()); #if ABSL_HAVE_BUILTIN(__builtin_prefetch) || defined(__GNUC__) __builtin_prefetch(control(), 0, 1); #endif -- cgit v1.2.3 From 29bd16cb925bd762ac7ca90e26ad044200ae0711 Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Tue, 4 Jun 2024 19:18:06 -0700 Subject: Remove redundant check of is_soo() while prefetching heap blocks. PiperOrigin-RevId: 640359514 Change-Id: Ic321d23cad283425c686536ae7be04a490a950d5 --- absl/container/internal/raw_hash_set.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 2ea07e40..9ef9a6be 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -3887,7 +3887,7 @@ class raw_hash_set { // cache misses. This is intended to overlap with execution of calculating the // hash for a key. void prefetch_heap_block() const { - assert(!is_soo()); + if (is_soo()) return; #if ABSL_HAVE_BUILTIN(__builtin_prefetch) || defined(__GNUC__) __builtin_prefetch(control(), 0, 1); #endif -- cgit v1.2.3 From 9e72bd674675db044ef1de81ff542539d26fcb1b Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Wed, 5 Jun 2024 12:58:03 -0700 Subject: Remove redundant check of is_soo() while prefetching heap blocks. PiperOrigin-RevId: 640620462 Change-Id: I72c0a7f0f549404ad8310b8cebd7dd491341390b --- absl/container/internal/raw_hash_set.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 9ef9a6be..2ea07e40 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -3887,7 +3887,7 @@ class raw_hash_set { // cache misses. This is intended to overlap with execution of calculating the // hash for a key. void prefetch_heap_block() const { - if (is_soo()) return; + assert(!is_soo()); #if ABSL_HAVE_BUILTIN(__builtin_prefetch) || defined(__GNUC__) __builtin_prefetch(control(), 0, 1); #endif -- cgit v1.2.3 From 66ef711d6846771b8e725e377de37bedae6c1527 Mon Sep 17 00:00:00 2001 From: Evan Brown Date: Thu, 6 Jun 2024 11:12:29 -0700 Subject: Add validation that hash/eq functors are consistent, meaning that `eq(k1, k2) -> hash(k1) == hash(k2)`. Also add missing includes/dependencies in flat_hash_map_test. PiperOrigin-RevId: 640959222 Change-Id: I8d99544af05e97310045e6149f6ef6f7c82e552d --- absl/container/BUILD.bazel | 2 ++ absl/container/CMakeLists.txt | 2 ++ absl/container/flat_hash_map_test.cc | 36 ++++++++++++++++++++ absl/container/internal/raw_hash_set.h | 49 ++++++++++++++++++++++++++++ absl/container/internal/raw_hash_set_test.cc | 34 +++++++++++++++++++ 5 files changed, 123 insertions(+) (limited to 'absl/container/internal') diff --git a/absl/container/BUILD.bazel b/absl/container/BUILD.bazel index 859163f8..2e32b4dd 100644 --- a/absl/container/BUILD.bazel +++ b/absl/container/BUILD.bazel @@ -265,11 +265,13 @@ cc_test( deps = [ ":flat_hash_map", ":hash_generator_testing", + ":hash_policy_testing", ":test_allocator", ":unordered_map_constructor_test", ":unordered_map_lookup_test", ":unordered_map_members_test", ":unordered_map_modifiers_test", + "//absl/base:config", "//absl/log:check", "//absl/meta:type_traits", "//absl/types:any", diff --git a/absl/container/CMakeLists.txt b/absl/container/CMakeLists.txt index b1f5f9d8..a1023def 100644 --- a/absl/container/CMakeLists.txt +++ b/absl/container/CMakeLists.txt @@ -306,8 +306,10 @@ absl_cc_test( DEPS absl::any absl::check + absl::config absl::flat_hash_map absl::hash_generator_testing + absl::hash_policy_testing absl::test_allocator absl::type_traits absl::unordered_map_constructor_test diff --git a/absl/container/flat_hash_map_test.cc b/absl/container/flat_hash_map_test.cc index 8ef1a62b..eaa86925 100644 --- a/absl/container/flat_hash_map_test.cc +++ b/absl/container/flat_hash_map_test.cc @@ -16,12 +16,16 @@ #include #include +#include #include #include #include +#include "gmock/gmock.h" #include "gtest/gtest.h" +#include "absl/base/config.h" #include "absl/container/internal/hash_generator_testing.h" +#include "absl/container/internal/hash_policy_testing.h" #include "absl/container/internal/test_allocator.h" #include "absl/container/internal/unordered_map_constructor_test.h" #include "absl/container/internal/unordered_map_lookup_test.h" @@ -363,6 +367,38 @@ TEST(FlatHashMap, FlatHashMapPolicyDestroyReturnsTrue) { std::allocator>(nullptr, nullptr))())); } +struct InconsistentHashEqType { + InconsistentHashEqType(int v1, int v2) : v1(v1), v2(v2) {} + template + friend H AbslHashValue(H h, InconsistentHashEqType t) { + return H::combine(std::move(h), t.v1); + } + bool operator==(InconsistentHashEqType t) const { return v2 == t.v2; } + int v1, v2; +}; + +TEST(Iterator, InconsistentHashEqFunctorsValidation) { + if (!IsAssertEnabled()) GTEST_SKIP() << "Assertions not enabled."; + + absl::flat_hash_map m; + for (int i = 0; i < 10; ++i) m[{i, i}] = 1; + // We need to insert multiple times to guarantee that we get the assertion + // because it's possible for the hash to collide with the inserted element + // that has v2==0. In those cases, the new element won't be inserted. + auto insert_conflicting_elems = [&] { + for (int i = 100; i < 20000; ++i) { + EXPECT_EQ((m[{i, 0}]), 1); + } + }; + + const char* crash_message = "hash/eq functors are inconsistent."; +#if defined(__arm__) || defined(__aarch64__) + // On ARM, the crash message is garbled so don't expect a specific message. + crash_message = ""; +#endif + EXPECT_DEATH_IF_SUPPORTED(insert_conflicting_elems(), crash_message); +} + } // namespace } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 2ea07e40..f494fb1c 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -3317,11 +3317,13 @@ class raw_hash_set { template iterator find(const key_arg& key, size_t hash) ABSL_ATTRIBUTE_LIFETIME_BOUND { + AssertHashEqConsistent(key); if (is_soo()) return find_soo(key); return find_non_soo(key, hash); } template iterator find(const key_arg& key) ABSL_ATTRIBUTE_LIFETIME_BOUND { + AssertHashEqConsistent(key); if (is_soo()) return find_soo(key); prefetch_heap_block(); return find_non_soo(key, hash_ref()(key)); @@ -3822,11 +3824,58 @@ class raw_hash_set { } protected: + // Asserts that hash and equal functors provided by the user are consistent, + // meaning that `eq(k1, k2)` implies `hash(k1)==hash(k2)`. + template + void AssertHashEqConsistent(ABSL_ATTRIBUTE_UNUSED const K& key) { +#ifndef NDEBUG + if (empty()) return; + + const size_t hash_of_arg = hash_ref()(key); + const auto assert_consistent = [&](const ctrl_t*, slot_type* slot) { + const value_type& element = PolicyTraits::element(slot); + const bool is_key_equal = + PolicyTraits::apply(EqualElement{key, eq_ref()}, element); + if (!is_key_equal) return; + + const size_t hash_of_slot = + PolicyTraits::apply(HashElement{hash_ref()}, element); + const bool is_hash_equal = hash_of_arg == hash_of_slot; + if (!is_hash_equal) { + // In this case, we're going to crash. Do a couple of other checks for + // idempotence issues. Recalculating hash/eq here is also convenient for + // debugging with gdb/lldb. + const size_t once_more_hash_arg = hash_ref()(key); + assert(hash_of_arg == once_more_hash_arg && "hash is not idempotent."); + const size_t once_more_hash_slot = + PolicyTraits::apply(HashElement{hash_ref()}, element); + assert(hash_of_slot == once_more_hash_slot && + "hash is not idempotent."); + const bool once_more_eq = + PolicyTraits::apply(EqualElement{key, eq_ref()}, element); + assert(is_key_equal == once_more_eq && "equality is not idempotent."); + } + assert((!is_key_equal || is_hash_equal) && + "eq(k1, k2) must imply that hash(k1) == hash(k2). " + "hash/eq functors are inconsistent."); + }; + + if (is_soo()) { + assert_consistent(/*unused*/ nullptr, soo_slot()); + return; + } + // We only do validation for small tables so that it's constant time. + if (capacity() > 16) return; + IterateOverFullSlots(common(), slot_array(), assert_consistent); +#endif + } + // Attempts to find `key` in the table; if it isn't found, returns an iterator // where the value can be inserted into, with the control byte already set to // `key`'s H2. Returns a bool indicating whether an insertion can take place. template std::pair find_or_prepare_insert(const K& key) { + AssertHashEqConsistent(key); if (is_soo()) return find_or_prepare_insert_soo(key); return find_or_prepare_insert_non_soo(key); } diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 10f793ef..195296a1 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -3293,6 +3293,40 @@ TEST(Table, ReserveToNonSoo) { } } +struct InconsistentHashEqType { + InconsistentHashEqType(int v1, int v2) : v1(v1), v2(v2) {} + template + friend H AbslHashValue(H h, InconsistentHashEqType t) { + return H::combine(std::move(h), t.v1); + } + bool operator==(InconsistentHashEqType t) const { return v2 == t.v2; } + int v1, v2; +}; + +TEST(Iterator, InconsistentHashEqFunctorsValidation) { + if (!IsAssertEnabled()) GTEST_SKIP() << "Assertions not enabled."; + + ValueTable t; + for (int i = 0; i < 10; ++i) t.insert({i, i}); + // We need to find/insert multiple times to guarantee that we get the + // assertion because it's possible for the hash to collide with the inserted + // element that has v2==0. In those cases, the new element won't be inserted. + auto find_conflicting_elems = [&] { + for (int i = 100; i < 20000; ++i) { + EXPECT_EQ(t.find({i, 0}), t.end()); + } + }; + EXPECT_DEATH_IF_SUPPORTED(find_conflicting_elems(), + "hash/eq functors are inconsistent."); + auto insert_conflicting_elems = [&] { + for (int i = 100; i < 20000; ++i) { + EXPECT_EQ(t.insert({i, 0}).second, false); + } + }; + EXPECT_DEATH_IF_SUPPORTED(insert_conflicting_elems(), + "hash/eq functors are inconsistent."); +} + REGISTER_TYPED_TEST_SUITE_P( SooTest, Empty, Clear, ClearBug, Contains1, Contains2, ContainsEmpty, CopyConstruct, CopyDifferentCapacities, CopyDifferentSizes, -- cgit v1.2.3 From 1d401d9c5a2d0896ec7abb69efbd199b05f9e55e Mon Sep 17 00:00:00 2001 From: Vitaly Goldshteyn Date: Mon, 10 Jun 2024 14:05:35 -0700 Subject: Use `IterateOverFullSlots` in `absl::erase_if` for hash table. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `EraseIf` itself is not very hot function, but I want to use that as demonstration of the speed of iteration via `IterateOverFullSlots`. Motivation: 1. We are still going to save some resources. 2. It is the first step to implement faster `absl::c_for_each` that may give larger benefits. Will require readability considerations. `BM_EraseIf/num_elements:1000/num_erased:0` is just iteration and it gives 60% speed up. On smaller tables removing all elements shows 25% speed up. Note that on small tables erasing is much faster due to cl/592272653. ``` name old cpu/op new cpu/op delta BM_EraseIf/num_elements:10/num_erased:0 3.41ns ± 5% 3.03ns ± 3% -11.14% (p=0.000 n=37+35) BM_EraseIf/num_elements:1000/num_erased:0 6.06ns ± 3% 2.42ns ± 3% -60.05% (p=0.000 n=34+37) BM_EraseIf/num_elements:10/num_erased:5 5.90ns ± 3% 4.44ns ± 4% -24.88% (p=0.000 n=36+37) BM_EraseIf/num_elements:1000/num_erased:500 11.0ns ± 2% 9.2ns ± 2% -16.60% (p=0.000 n=35+37) BM_EraseIf/num_elements:10/num_erased:10 8.03ns ± 3% 5.77ns ± 2% -28.19% (p=0.000 n=37+37) BM_EraseIf/num_elements:1000/num_erased:1000 9.00ns ± 3% 7.83ns ± 2% -12.98% (p=0.000 n=37+37) name old time/op new time/op delta BM_EraseIf/num_elements:10/num_erased:0 3.42ns ± 5% 3.04ns ± 3% -11.13% (p=0.000 n=37+36) BM_EraseIf/num_elements:1000/num_erased:0 6.07ns ± 3% 2.42ns ± 3% -60.10% (p=0.000 n=34+37) BM_EraseIf/num_elements:10/num_erased:5 5.93ns ± 3% 4.45ns ± 4% -24.89% (p=0.000 n=36+37) BM_EraseIf/num_elements:1000/num_erased:500 11.1ns ± 2% 9.2ns ± 2% -16.61% (p=0.000 n=35+37) BM_EraseIf/num_elements:10/num_erased:10 8.06ns ± 3% 5.79ns ± 2% -28.19% (p=0.000 n=37+37) BM_EraseIf/num_elements:1000/num_erased:1000 9.03ns ± 3% 7.85ns ± 2% -12.98% (p=0.000 n=37+37) name old INSTRUCTIONS/op new INSTRUCTIONS/op delta BM_EraseIf/num_elements:10/num_erased:0 19.5 ± 1% 14.9 ± 0% -23.79% (p=0.000 n=37+37) BM_EraseIf/num_elements:1000/num_erased:0 19.9 ± 0% 12.7 ± 0% -36.20% (p=0.000 n=37+37) BM_EraseIf/num_elements:10/num_erased:5 36.9 ± 1% 30.9 ± 0% -16.47% (p=0.000 n=37+37) BM_EraseIf/num_elements:1000/num_erased:500 44.8 ± 0% 37.6 ± 0% -16.06% (p=0.000 n=37+37) BM_EraseIf/num_elements:10/num_erased:10 53.0 ± 1% 46.9 ± 0% -11.61% (p=0.000 n=37+37) BM_EraseIf/num_elements:1000/num_erased:1000 69.8 ± 0% 62.6 ± 0% -10.32% (p=0.000 n=36+37) name old CYCLES/op new CYCLES/op delta BM_EraseIf/num_elements:10/num_erased:0 6.10 ± 7% 4.91 ± 1% -19.49% (p=0.000 n=37+37) BM_EraseIf/num_elements:1000/num_erased:0 19.4 ± 1% 7.7 ± 2% -60.04% (p=0.000 n=37+37) BM_EraseIf/num_elements:10/num_erased:5 13.9 ± 4% 9.2 ± 3% -33.80% (p=0.000 n=37+37) BM_EraseIf/num_elements:1000/num_erased:500 35.5 ± 0% 29.5 ± 1% -16.74% (p=0.000 n=37+37) BM_EraseIf/num_elements:10/num_erased:10 20.8 ± 5% 13.5 ± 0% -35.07% (p=0.000 n=37+30) BM_EraseIf/num_elements:1000/num_erased:1000 28.9 ± 0% 25.1 ± 0% -13.06% (p=0.000 n=37+37) ``` PiperOrigin-RevId: 642016364 Change-Id: I8be6af5916bd45fd110bb0398c3ffe932a6a083f --- absl/container/BUILD.bazel | 2 + absl/container/CMakeLists.txt | 2 + absl/container/internal/raw_hash_set.h | 70 ++++-- absl/container/internal/raw_hash_set_test.cc | 324 +++++++++++++++++++-------- 4 files changed, 292 insertions(+), 106 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/BUILD.bazel b/absl/container/BUILD.bazel index 2e32b4dd..38fec260 100644 --- a/absl/container/BUILD.bazel +++ b/absl/container/BUILD.bazel @@ -709,8 +709,10 @@ cc_test( ":hash_policy_testing", ":hashtable_debug", ":hashtablez_sampler", + ":node_hash_set", ":raw_hash_set", ":test_allocator", + ":test_instance_tracker", "//absl/base", "//absl/base:config", "//absl/base:core_headers", diff --git a/absl/container/CMakeLists.txt b/absl/container/CMakeLists.txt index a1023def..dd994f0b 100644 --- a/absl/container/CMakeLists.txt +++ b/absl/container/CMakeLists.txt @@ -783,10 +783,12 @@ absl_cc_test( absl::hashtablez_sampler absl::log absl::memory + absl::node_hash_set absl::prefetch absl::raw_hash_set absl::strings absl::test_allocator + absl::test_instance_tracker absl::type_traits GTest::gmock_main ) diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index f494fb1c..d8598725 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -1222,6 +1222,8 @@ class RawHashSetLayout { size_t slot_offset_; }; +struct HashtableFreeFunctionsAccess; + // We only allow a maximum of 1 SOO element, which makes the implementation // much simpler. Complications with multiple SOO elements include: // - Satisfying the guarantee that erasing one element doesn't invalidate @@ -1858,8 +1860,9 @@ inline void* SlotAddress(void* slot_array, size_t slot, size_t slot_size) { } // Iterates over all full slots and calls `cb(const ctrl_t*, SlotType*)`. -// NOTE: no erasure from this table allowed during Callback call. -template +// If kAllowRemoveReentrance is false, no erasure from this table allowed during +// Callback call. This mode is slightly faster. +template ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots( const CommonFields& c, SlotType* slot, Callback cb) { const size_t cap = c.capacity(); @@ -1882,18 +1885,25 @@ ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots( --ctrl; --slot; for (uint32_t i : mask) { - cb(ctrl + i, slot + i); + if (!kAllowRemoveReentrance || ABSL_PREDICT_TRUE(IsFull(ctrl[i]))) { + cb(ctrl + i, slot + i); + } } return; } size_t remaining = c.size(); while (remaining != 0) { for (uint32_t i : GroupFullEmptyOrDeleted(ctrl).MaskFull()) { - cb(ctrl + i, slot + i); + if (!kAllowRemoveReentrance || ABSL_PREDICT_TRUE(IsFull(ctrl[i]))) { + cb(ctrl + i, slot + i); + } --remaining; } - slot += Group::kWidth; ctrl += Group::kWidth; + if (kAllowRemoveReentrance && *(ctrl - 1) == ctrl_t::kSentinel) { + break; + } + slot += Group::kWidth; } } @@ -2430,6 +2440,7 @@ class raw_hash_set { class iterator : private HashSetIteratorGenerationInfo { friend class raw_hash_set; + friend struct HashtableFreeFunctionsAccess; public: using iterator_category = std::forward_iterator_tag; @@ -2736,7 +2747,7 @@ class raw_hash_set { size_t offset = cap; const size_t shift = is_single_group(cap) ? (PerTableSalt(control()) | 1) : 0; - IterateOverFullSlots( + IterateOverFullSlots( that.common(), that.slot_array(), [&](const ctrl_t* that_ctrl, slot_type* that_slot) ABSL_ATTRIBUTE_ALWAYS_INLINE { @@ -3411,6 +3422,8 @@ class raw_hash_set { friend struct absl::container_internal::hashtable_debug_internal:: HashtableDebugAccess; + friend struct absl::container_internal::HashtableFreeFunctionsAccess; + struct FindElement { template const_iterator operator()(const K& key, Args&&...) const { @@ -3521,7 +3534,7 @@ class raw_hash_set { inline void destroy_slots() { assert(!is_soo()); if (PolicyTraits::template destroy_is_trivial()) return; - IterateOverFullSlots( + IterateOverFullSlots( common(), slot_array(), [&](const ctrl_t*, slot_type* slot) ABSL_ATTRIBUTE_ALWAYS_INLINE { this->destroy(slot); }); @@ -3866,7 +3879,8 @@ class raw_hash_set { } // We only do validation for small tables so that it's constant time. if (capacity() > 16) return; - IterateOverFullSlots(common(), slot_array(), assert_consistent); + IterateOverFullSlots( + common(), slot_array(), assert_consistent); #endif } @@ -4030,19 +4044,41 @@ class raw_hash_set { key_equal{}, allocator_type{}}; }; +// Friend access for free functions in raw_hash_set.h. +struct HashtableFreeFunctionsAccess { + template + static typename Set::size_type EraseIf(Predicate& pred, Set* c) { + if (c->empty()) { + return 0; + } + if (c->is_soo()) { + auto it = c->soo_iterator(); + if (!pred(*it)) { + return 0; + } + c->destroy(it.slot()); + c->common().set_empty_soo(); + return 1; + } + size_t num_deleted = 0; + IterateOverFullSlots( + c->common(), c->slot_array(), [&](const ctrl_t* ctrl, auto* slot) { + if (pred(Set::PolicyTraits::element(slot))) { + c->destroy(slot); + EraseMetaOnly(c->common(), static_cast(ctrl - c->control()), + sizeof(*slot)); + ++num_deleted; + } + }); + return num_deleted; + } +}; + // Erases all elements that satisfy the predicate `pred` from the container `c`. template typename raw_hash_set::size_type EraseIf( Predicate& pred, raw_hash_set* c) { - const auto initial_size = c->size(); - for (auto it = c->begin(), last = c->end(); it != last;) { - if (pred(*it)) { - c->erase(it++); - } else { - ++it; - } - } - return initial_size - c->size(); + return HashtableFreeFunctionsAccess::EraseIf(pred, c); } namespace hashtable_debug_internal { diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 195296a1..9b13701f 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -52,12 +52,15 @@ #include "absl/container/internal/hashtable_debug.h" #include "absl/container/internal/hashtablez_sampler.h" #include "absl/container/internal/test_allocator.h" +#include "absl/container/internal/test_instance_tracker.h" +#include "absl/container/node_hash_set.h" #include "absl/functional/function_ref.h" #include "absl/hash/hash.h" #include "absl/log/check.h" #include "absl/log/log.h" #include "absl/memory/memory.h" #include "absl/meta/type_traits.h" +#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" namespace absl { @@ -797,21 +800,22 @@ TEST(Table, EmptyFunctorOptimization) { template class SooTest : public testing::Test {}; -TYPED_TEST_SUITE_P(SooTest); +using SooTableTypes = ::testing::Types; +TYPED_TEST_SUITE(SooTest, SooTableTypes); -TYPED_TEST_P(SooTest, Empty) { +TYPED_TEST(SooTest, Empty) { TypeParam t; EXPECT_EQ(0, t.size()); EXPECT_TRUE(t.empty()); } -TYPED_TEST_P(SooTest, LookupEmpty) { +TYPED_TEST(SooTest, LookupEmpty) { TypeParam t; auto it = t.find(0); EXPECT_TRUE(it == t.end()); } -TYPED_TEST_P(SooTest, Insert1) { +TYPED_TEST(SooTest, Insert1) { TypeParam t; EXPECT_TRUE(t.find(0) == t.end()); auto res = t.emplace(0); @@ -821,7 +825,7 @@ TYPED_TEST_P(SooTest, Insert1) { EXPECT_THAT(*t.find(0), 0); } -TYPED_TEST_P(SooTest, Insert2) { +TYPED_TEST(SooTest, Insert2) { TypeParam t; EXPECT_TRUE(t.find(0) == t.end()); auto res = t.emplace(0); @@ -884,7 +888,7 @@ TEST(Table, InsertCollisionAndFindAfterDelete) { EXPECT_TRUE(t.empty()); } -TYPED_TEST_P(SooTest, EraseInSmallTables) { +TYPED_TEST(SooTest, EraseInSmallTables) { for (int64_t size = 0; size < 64; ++size) { TypeParam t; for (int64_t i = 0; i < size; ++i) { @@ -901,7 +905,7 @@ TYPED_TEST_P(SooTest, EraseInSmallTables) { } } -TYPED_TEST_P(SooTest, InsertWithinCapacity) { +TYPED_TEST(SooTest, InsertWithinCapacity) { TypeParam t; t.reserve(10); const size_t original_capacity = t.capacity(); @@ -935,9 +939,11 @@ TYPED_TEST_P(SooTest, InsertWithinCapacity) { template class SmallTableResizeTest : public testing::Test {}; -TYPED_TEST_SUITE_P(SmallTableResizeTest); +using SmallTableTypes = + ::testing::Types; +TYPED_TEST_SUITE(SmallTableResizeTest, SmallTableTypes); -TYPED_TEST_P(SmallTableResizeTest, InsertIntoSmallTable) { +TYPED_TEST(SmallTableResizeTest, InsertIntoSmallTable) { TypeParam t; for (int i = 0; i < 32; ++i) { t.insert(i); @@ -949,7 +955,7 @@ TYPED_TEST_P(SmallTableResizeTest, InsertIntoSmallTable) { } } -TYPED_TEST_P(SmallTableResizeTest, ResizeGrowSmallTables) { +TYPED_TEST(SmallTableResizeTest, ResizeGrowSmallTables) { for (size_t source_size = 0; source_size < 32; ++source_size) { for (size_t target_size = source_size; target_size < 32; ++target_size) { for (bool rehash : {false, true}) { @@ -971,7 +977,7 @@ TYPED_TEST_P(SmallTableResizeTest, ResizeGrowSmallTables) { } } -TYPED_TEST_P(SmallTableResizeTest, ResizeReduceSmallTables) { +TYPED_TEST(SmallTableResizeTest, ResizeReduceSmallTables) { for (size_t source_size = 0; source_size < 32; ++source_size) { for (size_t target_size = 0; target_size <= source_size; ++target_size) { TypeParam t; @@ -994,13 +1000,6 @@ TYPED_TEST_P(SmallTableResizeTest, ResizeReduceSmallTables) { } } -REGISTER_TYPED_TEST_SUITE_P(SmallTableResizeTest, InsertIntoSmallTable, - ResizeGrowSmallTables, ResizeReduceSmallTables); -using SmallTableTypes = - ::testing::Types; -INSTANTIATE_TYPED_TEST_SUITE_P(InstanceSmallTableResizeTest, - SmallTableResizeTest, SmallTableTypes); - TEST(Table, LazyEmplace) { StringTable t; bool called = false; @@ -1019,13 +1018,13 @@ TEST(Table, LazyEmplace) { EXPECT_THAT(*it, Pair("abc", "ABC")); } -TYPED_TEST_P(SooTest, ContainsEmpty) { +TYPED_TEST(SooTest, ContainsEmpty) { TypeParam t; EXPECT_FALSE(t.contains(0)); } -TYPED_TEST_P(SooTest, Contains1) { +TYPED_TEST(SooTest, Contains1) { TypeParam t; EXPECT_TRUE(t.insert(0).second); @@ -1036,7 +1035,7 @@ TYPED_TEST_P(SooTest, Contains1) { EXPECT_FALSE(t.contains(0)); } -TYPED_TEST_P(SooTest, Contains2) { +TYPED_TEST(SooTest, Contains2) { TypeParam t; EXPECT_TRUE(t.insert(0).second); @@ -1334,7 +1333,7 @@ TEST(Table, RehashWithNoResize) { } } -TYPED_TEST_P(SooTest, InsertEraseStressTest) { +TYPED_TEST(SooTest, InsertEraseStressTest) { TypeParam t; const size_t kMinElementCount = 250; std::deque keys; @@ -1363,7 +1362,7 @@ TEST(Table, InsertOverloads) { Pair("DEF", "!!!"))); } -TYPED_TEST_P(SooTest, LargeTable) { +TYPED_TEST(SooTest, LargeTable) { TypeParam t; for (int64_t i = 0; i != 100000; ++i) t.emplace(i << 40); for (int64_t i = 0; i != 100000; ++i) @@ -1371,7 +1370,7 @@ TYPED_TEST_P(SooTest, LargeTable) { } // Timeout if copy is quadratic as it was in Rust. -TYPED_TEST_P(SooTest, EnsureNonQuadraticAsInRust) { +TYPED_TEST(SooTest, EnsureNonQuadraticAsInRust) { static const size_t kLargeSize = 1 << 15; TypeParam t; @@ -1384,7 +1383,7 @@ TYPED_TEST_P(SooTest, EnsureNonQuadraticAsInRust) { for (const auto& entry : t) t2.insert(entry); } -TYPED_TEST_P(SooTest, ClearBug) { +TYPED_TEST(SooTest, ClearBug) { if (SwisstableGenerationsEnabled()) { GTEST_SKIP() << "Generations being enabled causes extra rehashes."; } @@ -1411,7 +1410,7 @@ TYPED_TEST_P(SooTest, ClearBug) { capacity * sizeof(typename TypeParam::value_type)); } -TYPED_TEST_P(SooTest, Erase) { +TYPED_TEST(SooTest, Erase) { TypeParam t; EXPECT_TRUE(t.find(0) == t.end()); auto res = t.emplace(0); @@ -1422,7 +1421,7 @@ TYPED_TEST_P(SooTest, Erase) { EXPECT_TRUE(t.find(0) == t.end()); } -TYPED_TEST_P(SooTest, EraseMaintainsValidIterator) { +TYPED_TEST(SooTest, EraseMaintainsValidIterator) { TypeParam t; const int kNumElements = 100; for (int i = 0; i < kNumElements; i++) { @@ -1441,7 +1440,7 @@ TYPED_TEST_P(SooTest, EraseMaintainsValidIterator) { EXPECT_EQ(num_erase_calls, kNumElements); } -TYPED_TEST_P(SooTest, EraseBeginEnd) { +TYPED_TEST(SooTest, EraseBeginEnd) { TypeParam t; for (int i = 0; i < 10; ++i) t.insert(i); EXPECT_EQ(t.size(), 10); @@ -1862,7 +1861,7 @@ TEST(Table, GrowthInfoDeletedBit) { RawHashSetTestOnlyAccess::GetCommon(t).growth_info().HasNoDeleted()); } -TYPED_TEST_P(SooTest, Clear) { +TYPED_TEST(SooTest, Clear) { TypeParam t; EXPECT_TRUE(t.find(0) == t.end()); t.clear(); @@ -1875,7 +1874,7 @@ TYPED_TEST_P(SooTest, Clear) { EXPECT_TRUE(t.find(0) == t.end()); } -TYPED_TEST_P(SooTest, Swap) { +TYPED_TEST(SooTest, Swap) { TypeParam t; EXPECT_TRUE(t.find(0) == t.end()); auto res = t.emplace(0); @@ -1889,7 +1888,7 @@ TYPED_TEST_P(SooTest, Swap) { EXPECT_THAT(*u.find(0), 0); } -TYPED_TEST_P(SooTest, Rehash) { +TYPED_TEST(SooTest, Rehash) { TypeParam t; EXPECT_TRUE(t.find(0) == t.end()); t.emplace(0); @@ -1901,7 +1900,7 @@ TYPED_TEST_P(SooTest, Rehash) { EXPECT_THAT(*t.find(1), 1); } -TYPED_TEST_P(SooTest, RehashDoesNotRehashWhenNotNecessary) { +TYPED_TEST(SooTest, RehashDoesNotRehashWhenNotNecessary) { TypeParam t; t.emplace(0); t.emplace(1); @@ -1926,7 +1925,7 @@ TEST(Table, RehashZeroDeallocatesEmptyTable) { EXPECT_EQ(0, t.bucket_count()); } -TYPED_TEST_P(SooTest, RehashZeroForcesRehash) { +TYPED_TEST(SooTest, RehashZeroForcesRehash) { TypeParam t; t.emplace(0); t.emplace(1); @@ -1943,7 +1942,7 @@ TEST(Table, ConstructFromInitList) { StringTable t = {P(), Q(), {}, {{}, {}}}; } -TYPED_TEST_P(SooTest, CopyConstruct) { +TYPED_TEST(SooTest, CopyConstruct) { TypeParam t; t.emplace(0); EXPECT_EQ(1, t.size()); @@ -1964,7 +1963,7 @@ TYPED_TEST_P(SooTest, CopyConstruct) { } } -TYPED_TEST_P(SooTest, CopyDifferentSizes) { +TYPED_TEST(SooTest, CopyDifferentSizes) { TypeParam t; for (int i = 0; i < 100; ++i) { @@ -1978,7 +1977,7 @@ TYPED_TEST_P(SooTest, CopyDifferentSizes) { } } -TYPED_TEST_P(SooTest, CopyDifferentCapacities) { +TYPED_TEST(SooTest, CopyDifferentCapacities) { for (int cap = 1; cap < 100; cap = cap * 2 + 1) { TypeParam t; t.reserve(static_cast(cap)); @@ -2127,7 +2126,7 @@ TEST(Table, Equality3) { EXPECT_NE(u, t); } -TYPED_TEST_P(SooTest, NumDeletedRegression) { +TYPED_TEST(SooTest, NumDeletedRegression) { TypeParam t; t.emplace(0); t.erase(t.find(0)); @@ -2136,7 +2135,7 @@ TYPED_TEST_P(SooTest, NumDeletedRegression) { t.clear(); } -TYPED_TEST_P(SooTest, FindFullDeletedRegression) { +TYPED_TEST(SooTest, FindFullDeletedRegression) { TypeParam t; for (int i = 0; i < 1000; ++i) { t.emplace(i); @@ -2145,7 +2144,7 @@ TYPED_TEST_P(SooTest, FindFullDeletedRegression) { EXPECT_EQ(0, t.size()); } -TYPED_TEST_P(SooTest, ReplacingDeletedSlotDoesNotRehash) { +TYPED_TEST(SooTest, ReplacingDeletedSlotDoesNotRehash) { // We need to disable hashtablez to avoid issues related to SOO and sampling. SetHashtablezEnabled(false); @@ -2409,7 +2408,7 @@ TEST(Nodes, ExtractInsert) { EXPECT_FALSE(node); // NOLINT(bugprone-use-after-move) } -TYPED_TEST_P(SooTest, HintInsert) { +TYPED_TEST(SooTest, HintInsert) { TypeParam t = {1, 2, 3}; auto node = t.extract(1); EXPECT_THAT(t, UnorderedElementsAre(2, 3)); @@ -2449,7 +2448,7 @@ std::vector OrderOfIteration(const T& t) { // we are touching different memory pages to cause the ordering to change. // We also need to keep the old tables around to avoid getting the same memory // blocks over and over. -TYPED_TEST_P(SooTest, IterationOrderChangesByInstance) { +TYPED_TEST(SooTest, IterationOrderChangesByInstance) { for (size_t size : {2, 6, 12, 20}) { const auto reference_table = MakeSimpleTable(size); const auto reference = OrderOfIteration(reference_table); @@ -2468,7 +2467,7 @@ TYPED_TEST_P(SooTest, IterationOrderChangesByInstance) { } } -TYPED_TEST_P(SooTest, IterationOrderChangesOnRehash) { +TYPED_TEST(SooTest, IterationOrderChangesOnRehash) { // We test different sizes with many small numbers, because small table // resize has a different codepath. // Note: iteration order for size() <= 1 is always the same. @@ -2501,7 +2500,7 @@ TYPED_TEST_P(SooTest, IterationOrderChangesOnRehash) { // Verify that pointers are invalidated as soon as a second element is inserted. // This prevents dependency on pointer stability on small tables. -TYPED_TEST_P(SooTest, UnstablePointers) { +TYPED_TEST(SooTest, UnstablePointers) { // We need to disable hashtablez to avoid issues related to SOO and sampling. SetHashtablezEnabled(false); @@ -2571,7 +2570,7 @@ constexpr bool kMsvc = true; constexpr bool kMsvc = false; #endif -TYPED_TEST_P(SooTest, IteratorInvalidAssertsEqualityOperator) { +TYPED_TEST(SooTest, IteratorInvalidAssertsEqualityOperator) { if (!IsAssertEnabled() && !SwisstableGenerationsEnabled()) GTEST_SKIP() << "Assertions not enabled."; @@ -2608,7 +2607,7 @@ TYPED_TEST_P(SooTest, IteratorInvalidAssertsEqualityOperator) { EXPECT_DEATH_IF_SUPPORTED(void(iter2 == iter1), kContainerDiffDeathMessage); } -TYPED_TEST_P(SooTest, IteratorInvalidAssertsEqualityOperatorRehash) { +TYPED_TEST(SooTest, IteratorInvalidAssertsEqualityOperatorRehash) { if (!IsAssertEnabled() && !SwisstableGenerationsEnabled()) GTEST_SKIP() << "Assertions not enabled."; if (kMsvc) GTEST_SKIP() << "MSVC doesn't support | in regex."; @@ -2634,9 +2633,11 @@ TYPED_TEST_P(SooTest, IteratorInvalidAssertsEqualityOperatorRehash) { #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) template class RawHashSamplerTest : public testing::Test {}; -TYPED_TEST_SUITE_P(RawHashSamplerTest); -TYPED_TEST_P(RawHashSamplerTest, Sample) { +using RawHashSamplerTestTypes = ::testing::Types; +TYPED_TEST_SUITE(RawHashSamplerTest, RawHashSamplerTestTypes); + +TYPED_TEST(RawHashSamplerTest, Sample) { constexpr bool soo_enabled = std::is_same::value; // Enable the feature even if the prod default is off. SetHashtablezEnabled(true); @@ -2708,10 +2709,6 @@ TYPED_TEST_P(RawHashSamplerTest, Sample) { } } -REGISTER_TYPED_TEST_SUITE_P(RawHashSamplerTest, Sample); -using RawHashSamplerTestTypes = ::testing::Types; -INSTANTIATE_TYPED_TEST_SUITE_P(My, RawHashSamplerTest, RawHashSamplerTestTypes); - std::vector SampleSooMutation( absl::FunctionRef mutate_table) { // Enable the feature even if the prod default is off. @@ -2867,9 +2864,10 @@ TEST(RawHashSamplerTest, DoNotSampleCustomAllocators) { template class SanitizerTest : public testing::Test {}; -TYPED_TEST_SUITE_P(SanitizerTest); +using SanitizerTableTypes = ::testing::Types; +TYPED_TEST_SUITE(SanitizerTest, SanitizerTableTypes); -TYPED_TEST_P(SanitizerTest, PoisoningUnused) { +TYPED_TEST(SanitizerTest, PoisoningUnused) { TypeParam t; for (size_t reserve_size = 2; reserve_size < 1024; reserve_size = reserve_size * 3 / 2) { @@ -2887,11 +2885,6 @@ TYPED_TEST_P(SanitizerTest, PoisoningUnused) { } } -REGISTER_TYPED_TEST_SUITE_P(SanitizerTest, PoisoningUnused); -using SanitizerTableTypes = ::testing::Types; -INSTANTIATE_TYPED_TEST_SUITE_P(InstanceSanitizerTest, SanitizerTest, - SanitizerTableTypes); - // TODO(b/289225379): poison inline space when empty SOO. TEST(Sanitizer, PoisoningOnErase) { NonSooIntTable t; @@ -3006,7 +2999,7 @@ TEST(Iterator, InvalidUseWithMoveCrashesWithSanitizers) { #endif } -TYPED_TEST_P(SooTest, ReservedGrowthUpdatesWhenTableDoesntGrow) { +TYPED_TEST(SooTest, ReservedGrowthUpdatesWhenTableDoesntGrow) { TypeParam t; for (int i = 0; i < 8; ++i) t.insert(i); // Want to insert twice without invalidating iterators so reserve. @@ -3021,6 +3014,141 @@ TYPED_TEST_P(SooTest, ReservedGrowthUpdatesWhenTableDoesntGrow) { EXPECT_EQ(*it, 0); } +template +class InstanceTrackerTest : public testing::Test {}; + +using ::absl::test_internal::CopyableMovableInstance; +using ::absl::test_internal::InstanceTracker; + +struct InstanceTrackerHash { + size_t operator()(const CopyableMovableInstance& t) const { + return absl::HashOf(t.value()); + } +}; + +using InstanceTrackerTableTypes = ::testing::Types< + absl::node_hash_set, + absl::flat_hash_set>; +TYPED_TEST_SUITE(InstanceTrackerTest, InstanceTrackerTableTypes); + +TYPED_TEST(InstanceTrackerTest, EraseIfAll) { + using Table = TypeParam; + InstanceTracker tracker; + for (int size = 0; size < 100; ++size) { + Table t; + for (int i = 0; i < size; ++i) { + t.emplace(i); + } + absl::erase_if(t, [](const auto&) { return true; }); + ASSERT_EQ(t.size(), 0); + } + EXPECT_EQ(tracker.live_instances(), 0); +} + +TYPED_TEST(InstanceTrackerTest, EraseIfNone) { + using Table = TypeParam; + InstanceTracker tracker; + { + Table t; + for (size_t size = 0; size < 100; ++size) { + absl::erase_if(t, [](const auto&) { return false; }); + ASSERT_EQ(t.size(), size); + t.emplace(size); + } + } + EXPECT_EQ(tracker.live_instances(), 0); +} + +TYPED_TEST(InstanceTrackerTest, EraseIfPartial) { + using Table = TypeParam; + InstanceTracker tracker; + for (int mod : {0, 1}) { + for (int size = 0; size < 100; ++size) { + SCOPED_TRACE(absl::StrCat(mod, " ", size)); + Table t; + std::vector expected; + for (int i = 0; i < size; ++i) { + t.emplace(i); + if (i % 2 != mod) { + expected.emplace_back(i); + } + } + absl::erase_if(t, [mod](const auto& x) { return x.value() % 2 == mod; }); + ASSERT_THAT(t, testing::UnorderedElementsAreArray(expected)); + } + } + EXPECT_EQ(tracker.live_instances(), 0); +} + +TYPED_TEST(SooTest, EraseIfAll) { + auto pred = [](const auto&) { return true; }; + for (int size = 0; size < 100; ++size) { + TypeParam t; + for (int i = 0; i < size; ++i) t.insert(i); + absl::container_internal::EraseIf(pred, &t); + ASSERT_EQ(t.size(), 0); + } +} + +TYPED_TEST(SooTest, EraseIfNone) { + auto pred = [](const auto&) { return false; }; + TypeParam t; + for (size_t size = 0; size < 100; ++size) { + absl::container_internal::EraseIf(pred, &t); + ASSERT_EQ(t.size(), size); + t.insert(size); + } +} + +TYPED_TEST(SooTest, EraseIfPartial) { + for (int mod : {0, 1}) { + auto pred = [&](const auto& x) { + return static_cast(x) % 2 == mod; + }; + for (int size = 0; size < 100; ++size) { + SCOPED_TRACE(absl::StrCat(mod, " ", size)); + TypeParam t; + std::vector expected; + for (int i = 0; i < size; ++i) { + t.insert(i); + if (i % 2 != mod) { + expected.push_back(i); + } + } + absl::container_internal::EraseIf(pred, &t); + ASSERT_THAT(t, testing::UnorderedElementsAreArray(expected)); + } + } +} + +// Test that we are allowed to erase during the callback in erase_if. +// TODO(b/345744331): Consider to change behavior to disallow erasure in the +// callback. +TYPED_TEST(SooTest, EraseIfReentry) { + for (int size = 0; size < 100; ++size) { + SCOPED_TRACE(absl::StrCat(size)); + TypeParam t; + std::vector expected; + for (int i = 0; i < size; ++i) { + t.insert(i); + if (i % 4 == 1 || i % 4 == 2) { + expected.push_back(i); + } + } + auto pred = [&](const auto& x) { + auto value = static_cast(x); + int64_t group = value / 4; + t.erase(group * 4); + if (value % 4 == 3) { + return true; + } + return false; + }; + absl::container_internal::EraseIf(pred, &t); + ASSERT_THAT(t, testing::UnorderedElementsAreArray(expected)); + } +} + TEST(Table, EraseBeginEndResetsReservedGrowth) { bool frozen = false; BadHashFreezableIntTable t{FreezableAlloc(&frozen)}; @@ -3031,7 +3159,8 @@ TEST(Table, EraseBeginEndResetsReservedGrowth) { for (int i = 0; i < 10; ++i) { // Create a long run (hash function returns constant). for (int j = 0; j < 100; ++j) t.insert(j); - // Erase elements from the middle of the long run, which creates tombstones. + // Erase elements from the middle of the long run, which creates + // tombstones. for (int j = 30; j < 60; ++j) t.erase(j); EXPECT_EQ(t.size(), 70); EXPECT_EQ(t.capacity(), cap); @@ -3181,12 +3310,12 @@ TEST(Table, IterateOverFullSlotsEmpty) { auto fail_if_any = [](const ctrl_t*, auto* i) { FAIL() << "expected no slots " << **i; }; - container_internal::IterateOverFullSlots( + container_internal::IterateOverFullSlots( RawHashSetTestOnlyAccess::GetCommon(t), RawHashSetTestOnlyAccess::GetSlots(t), fail_if_any); for (size_t i = 0; i < 256; ++i) { t.reserve(i); - container_internal::IterateOverFullSlots( + container_internal::IterateOverFullSlots( RawHashSetTestOnlyAccess::GetCommon(t), RawHashSetTestOnlyAccess::GetSlots(t), fail_if_any); } @@ -3196,12 +3325,12 @@ TEST(Table, IterateOverFullSlotsFull) { NonSooIntTable t; std::vector expected_slots; - for (int64_t i = 0; i < 128; ++i) { - t.insert(i); - expected_slots.push_back(i); + for (int64_t idx = 0; idx < 128; ++idx) { + t.insert(idx); + expected_slots.push_back(idx); std::vector slots; - container_internal::IterateOverFullSlots( + container_internal::IterateOverFullSlots( RawHashSetTestOnlyAccess::GetCommon(t), RawHashSetTestOnlyAccess::GetSlots(t), [&t, &slots](const ctrl_t* ctrl, auto* i) { @@ -3215,11 +3344,49 @@ TEST(Table, IterateOverFullSlotsFull) { } } +TEST(Table, IterateOverFullSlotsAllowReentrance) { + std::vector expected_values; + for (int64_t idx = 0; idx < 128; ++idx) { + NonSooIntTable t; + for (int val = 0; val <= idx; ++val) { + t.insert(val); + } + + // We are inserting only even values. + // Only one element across 2*k and 2*k+1 should be visited. + if (idx % 2 == 0) { + expected_values.push_back(idx); + } + + std::vector actual_values; + container_internal::IterateOverFullSlots( + RawHashSetTestOnlyAccess::GetCommon(t), + RawHashSetTestOnlyAccess::GetSlots(t), + [&t, &actual_values](const ctrl_t* ctrl, auto* i) { + int64_t value = **i; + // Erase the other element from 2*k and 2*k+1 pair. + t.erase(value ^ 1); + ptrdiff_t ctrl_offset = + ctrl - RawHashSetTestOnlyAccess::GetCommon(t).control(); + ptrdiff_t slot_offset = i - RawHashSetTestOnlyAccess::GetSlots(t); + ASSERT_EQ(ctrl_offset, slot_offset); + // Add an even value from the pair. + // Only one element for each 2*k and 2*k+1 pair should be inserted. + actual_values.push_back(value - value % 2); + }); + EXPECT_THAT(actual_values, + testing::UnorderedElementsAreArray(expected_values)); + } +} + template class SooTable : public testing::Test {}; -TYPED_TEST_SUITE_P(SooTable); +using FreezableSooTableTypes = + ::testing::Types, + FreezableSizedValueSooTable<16>>; +TYPED_TEST_SUITE(SooTable, FreezableSooTableTypes); -TYPED_TEST_P(SooTable, Basic) { +TYPED_TEST(SooTable, Basic) { bool frozen = true; TypeParam t{FreezableAlloc(&frozen)}; if (t.capacity() != SooCapacity()) { @@ -3249,12 +3416,6 @@ TYPED_TEST_P(SooTable, Basic) { EXPECT_EQ(t.size(), 0); } -REGISTER_TYPED_TEST_SUITE_P(SooTable, Basic); -using FreezableSooTableTypes = - ::testing::Types, - FreezableSizedValueSooTable<16>>; -INSTANTIATE_TYPED_TEST_SUITE_P(My, SooTable, FreezableSooTableTypes); - TEST(Table, RehashToSooUnsampled) { SooIntTable t; if (t.capacity() != SooCapacity()) { @@ -3327,21 +3488,6 @@ TEST(Iterator, InconsistentHashEqFunctorsValidation) { "hash/eq functors are inconsistent."); } -REGISTER_TYPED_TEST_SUITE_P( - SooTest, Empty, Clear, ClearBug, Contains1, Contains2, ContainsEmpty, - CopyConstruct, CopyDifferentCapacities, CopyDifferentSizes, - EnsureNonQuadraticAsInRust, Erase, EraseBeginEnd, EraseInSmallTables, - EraseMaintainsValidIterator, FindFullDeletedRegression, HintInsert, Insert1, - Insert2, InsertEraseStressTest, InsertWithinCapacity, - IterationOrderChangesByInstance, IterationOrderChangesOnRehash, - IteratorInvalidAssertsEqualityOperator, - IteratorInvalidAssertsEqualityOperatorRehash, LargeTable, LookupEmpty, - NumDeletedRegression, Rehash, RehashDoesNotRehashWhenNotNecessary, - RehashZeroForcesRehash, ReplacingDeletedSlotDoesNotRehash, - ReservedGrowthUpdatesWhenTableDoesntGrow, Swap, UnstablePointers); -using SooTableTypes = ::testing::Types; -INSTANTIATE_TYPED_TEST_SUITE_P(My, SooTest, SooTableTypes); - } // namespace } // namespace container_internal ABSL_NAMESPACE_END -- cgit v1.2.3 From a0889af0a23030e6bb27c6c7241bade7e59cb763 Mon Sep 17 00:00:00 2001 From: Vitaly Goldshteyn Date: Tue, 11 Jun 2024 10:41:39 -0700 Subject: Disallow reentrance removal in `absl::erase_if`. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Predicates should generally have no side effects since we do not guarantee the order or quantity for the calls. In this change we forbid one specific side effect: modification of the table we are iterating over. As a positive effect we have performance improvements (less computations and less branches). ``` name old cpu/op new cpu/op delta BM_EraseIf/num_elements:10/num_erased:0 3.02ns ± 2% 2.79ns ± 3% -7.44% (p=0.000 n=35+37) BM_EraseIf/num_elements:1000/num_erased:0 2.41ns ± 5% 2.05ns ± 4% -14.88% (p=0.000 n=37+37) BM_EraseIf/num_elements:10/num_erased:5 4.40ns ± 3% 4.22ns ± 3% -4.19% (p=0.000 n=37+37) BM_EraseIf/num_elements:1000/num_erased:500 9.16ns ± 4% 9.13ns ± 3% ~ (p=0.307 n=37+37) BM_EraseIf/num_elements:10/num_erased:10 5.77ns ± 3% 5.50ns ± 4% -4.62% (p=0.000 n=37+37) BM_EraseIf/num_elements:1000/num_erased:1000 7.84ns ± 3% 7.77ns ± 3% -0.94% (p=0.006 n=37+35) name old time/op new time/op delta BM_EraseIf/num_elements:10/num_erased:0 3.02ns ± 2% 2.79ns ± 3% -7.48% (p=0.000 n=35+36) BM_EraseIf/num_elements:1000/num_erased:0 2.41ns ± 5% 2.05ns ± 4% -14.89% (p=0.000 n=37+37) BM_EraseIf/num_elements:10/num_erased:5 4.42ns ± 3% 4.23ns ± 3% -4.22% (p=0.000 n=37+37) BM_EraseIf/num_elements:1000/num_erased:500 9.18ns ± 4% 9.15ns ± 3% ~ (p=0.347 n=37+37) BM_EraseIf/num_elements:10/num_erased:10 5.79ns ± 3% 5.52ns ± 4% -4.61% (p=0.000 n=37+37) BM_EraseIf/num_elements:1000/num_erased:1000 7.87ns ± 3% 7.79ns ± 3% -0.95% (p=0.007 n=37+35) name old INSTRUCTIONS/op new INSTRUCTIONS/op delta BM_EraseIf/num_elements:10/num_erased:0 14.9 ± 0% 12.9 ± 0% -13.46% (p=0.000 n=37+37) BM_EraseIf/num_elements:1000/num_erased:0 12.7 ± 0% 10.3 ± 0% -18.76% (p=0.000 n=37+37) BM_EraseIf/num_elements:10/num_erased:5 30.9 ± 0% 28.9 ± 0% -6.48% (p=0.000 n=37+37) BM_EraseIf/num_elements:1000/num_erased:500 37.6 ± 0% 35.3 ± 0% -6.33% (p=0.000 n=37+37) BM_EraseIf/num_elements:10/num_erased:10 46.9 ± 0% 44.9 ± 0% -4.27% (p=0.000 n=37+37) BM_EraseIf/num_elements:1000/num_erased:1000 62.6 ± 0% 60.2 ± 0% -3.80% (p=0.000 n=37+36) name old CYCLES/op new CYCLES/op delta BM_EraseIf/num_elements:10/num_erased:0 4.91 ± 1% 4.11 ± 1% -16.35% (p=0.000 n=36+35) BM_EraseIf/num_elements:1000/num_erased:0 7.74 ± 2% 6.54 ± 2% -15.54% (p=0.000 n=37+37) BM_EraseIf/num_elements:10/num_erased:5 9.18 ± 3% 8.45 ± 3% -7.88% (p=0.000 n=37+35) BM_EraseIf/num_elements:1000/num_erased:500 29.5 ± 1% 29.3 ± 1% -0.82% (p=0.000 n=36+37) BM_EraseIf/num_elements:10/num_erased:10 13.5 ± 1% 12.6 ± 0% -7.06% (p=0.000 n=33+34) BM_EraseIf/num_elements:1000/num_erased:1000 25.1 ± 0% 24.9 ± 0% -0.90% (p=0.000 n=37+35) ``` PiperOrigin-RevId: 642318040 Change-Id: I78a4a5a9a5881db0818225f9c7c153c562009f66 --- absl/container/internal/raw_hash_set.h | 5 ++++- absl/container/internal/raw_hash_set_test.cc | 28 ---------------------------- 2 files changed, 4 insertions(+), 29 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index d8598725..c63b60e3 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -1902,6 +1902,9 @@ ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots( ctrl += Group::kWidth; if (kAllowRemoveReentrance && *(ctrl - 1) == ctrl_t::kSentinel) { break; + } else { + assert((remaining == 0 || *(ctrl - 1) != ctrl_t::kSentinel) && + "element was erased from hash table unexpectedly"); } slot += Group::kWidth; } @@ -4061,7 +4064,7 @@ struct HashtableFreeFunctionsAccess { return 1; } size_t num_deleted = 0; - IterateOverFullSlots( + IterateOverFullSlots( c->common(), c->slot_array(), [&](const ctrl_t* ctrl, auto* slot) { if (pred(Set::PolicyTraits::element(slot))) { c->destroy(slot); diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 9b13701f..2a6ee656 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -3121,34 +3121,6 @@ TYPED_TEST(SooTest, EraseIfPartial) { } } -// Test that we are allowed to erase during the callback in erase_if. -// TODO(b/345744331): Consider to change behavior to disallow erasure in the -// callback. -TYPED_TEST(SooTest, EraseIfReentry) { - for (int size = 0; size < 100; ++size) { - SCOPED_TRACE(absl::StrCat(size)); - TypeParam t; - std::vector expected; - for (int i = 0; i < size; ++i) { - t.insert(i); - if (i % 4 == 1 || i % 4 == 2) { - expected.push_back(i); - } - } - auto pred = [&](const auto& x) { - auto value = static_cast(x); - int64_t group = value / 4; - t.erase(group * 4); - if (value % 4 == 3) { - return true; - } - return false; - }; - absl::container_internal::EraseIf(pred, &t); - ASSERT_THAT(t, testing::UnorderedElementsAreArray(expected)); - } -} - TEST(Table, EraseBeginEndResetsReservedGrowth) { bool frozen = false; BadHashFreezableIntTable t{FreezableAlloc(&frozen)}; -- cgit v1.2.3 From 10ac811f7c3720761c0fa0cd2b3fe92116b89420 Mon Sep 17 00:00:00 2001 From: Vitaly Goldshteyn Date: Thu, 20 Jun 2024 13:43:52 -0700 Subject: Create `absl::container_internal::c_for_each_fast` for SwissTable. This function is aimed to achieve faster iteration through the entire hash table. It is not ready to be used by the public and stays in `container_internal` namespace. Differences with `absl::c_for_each`: 1. No guarantees on order of iteration. Although for the hash table it is partially not guaranteed already. But we do not even guarantee that it is the same order as in the for loop range. De facto, the order is the same at the moment. 2. No mutating reentrance is allowed. Most notably erasing from the hash_table is not allowed. Based on microbenchmarks, there are following conclusions: 1. c_for_each_fast is clearly faster on big tables with 20-60% speedup. 2. Microbenchmarks show regression on a full small table without any empty slots. We should avoid recommending that for small tables. 3. It seems reasonable to use `c_for_each_fast` in places, where `skip_empty_or_deleted` has significant GCU usage. `skip_empty_or_deleted` usage signals that there are "gaps" between elements, so `c_for_each_fast` should be an improvement. PiperOrigin-RevId: 645142512 Change-Id: I279886b8c8b2545504c2bf7e037d27b2545e044d --- absl/container/BUILD.bazel | 12 ++++- absl/container/CMakeLists.txt | 11 ++++- absl/container/flat_hash_map.h | 33 ++++++++++++++ absl/container/flat_hash_map_test.cc | 53 +++++++++++++++++++++++ absl/container/flat_hash_set.h | 28 ++++++++++++ absl/container/flat_hash_set_test.cc | 33 ++++++++++++++ absl/container/internal/raw_hash_set.h | 27 ++++++++++++ absl/container/internal/raw_hash_set_test.cc | 52 ++++++++++++++++++++++ absl/container/node_hash_map.h | 33 ++++++++++++++ absl/container/node_hash_map_test.cc | 65 ++++++++++++++++++++++++++++ absl/container/node_hash_set.h | 28 ++++++++++++ absl/container/node_hash_set_test.cc | 46 ++++++++++++++++++++ 12 files changed, 417 insertions(+), 4 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/BUILD.bazel b/absl/container/BUILD.bazel index 38fec260..b00c30fd 100644 --- a/absl/container/BUILD.bazel +++ b/absl/container/BUILD.bazel @@ -252,7 +252,7 @@ cc_library( ":raw_hash_map", "//absl/algorithm:container", "//absl/base:core_headers", - "//absl/memory", + "//absl/meta:type_traits", ], ) @@ -292,6 +292,7 @@ cc_library( "//absl/algorithm:container", "//absl/base:core_headers", "//absl/memory", + "//absl/meta:type_traits", ], ) @@ -332,6 +333,7 @@ cc_library( "//absl/algorithm:container", "//absl/base:core_headers", "//absl/memory", + "//absl/meta:type_traits", ], ) @@ -342,13 +344,14 @@ cc_test( linkopts = ABSL_DEFAULT_LINKOPTS, tags = ["no_test_loonix"], deps = [ - ":hash_generator_testing", + ":hash_policy_testing", ":node_hash_map", ":tracked", ":unordered_map_constructor_test", ":unordered_map_lookup_test", ":unordered_map_members_test", ":unordered_map_modifiers_test", + "//absl/base:config", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], @@ -367,6 +370,7 @@ cc_library( "//absl/algorithm:container", "//absl/base:core_headers", "//absl/memory", + "//absl/meta:type_traits", ], ) @@ -377,11 +381,15 @@ cc_test( linkopts = ABSL_DEFAULT_LINKOPTS, tags = ["no_test_loonix"], deps = [ + ":hash_generator_testing", + ":hash_policy_testing", ":node_hash_set", ":unordered_set_constructor_test", ":unordered_set_lookup_test", ":unordered_set_members_test", ":unordered_set_modifiers_test", + "//absl/base:config", + "//absl/memory", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], diff --git a/absl/container/CMakeLists.txt b/absl/container/CMakeLists.txt index dd994f0b..25831d5f 100644 --- a/absl/container/CMakeLists.txt +++ b/absl/container/CMakeLists.txt @@ -292,7 +292,7 @@ absl_cc_library( absl::hash_container_defaults absl::raw_hash_map absl::algorithm_container - absl::memory + absl::type_traits PUBLIC ) @@ -333,6 +333,7 @@ absl_cc_library( absl::algorithm_container absl::core_headers absl::memory + absl::type_traits PUBLIC ) @@ -375,6 +376,7 @@ absl_cc_library( absl::raw_hash_map absl::algorithm_container absl::memory + absl::type_traits PUBLIC ) @@ -386,7 +388,8 @@ absl_cc_test( COPTS ${ABSL_TEST_COPTS} DEPS - absl::hash_generator_testing + absl::config + absl::hash_policy_testing absl::node_hash_map absl::tracked absl::unordered_map_constructor_test @@ -411,6 +414,7 @@ absl_cc_library( absl::raw_hash_set absl::algorithm_container absl::memory + absl::type_traits PUBLIC ) @@ -424,7 +428,10 @@ absl_cc_test( "-DUNORDERED_SET_CXX17" DEPS absl::hash_generator_testing + absl::hash_policy_testing + absl::memory absl::node_hash_set + absl::type_traits absl::unordered_set_constructor_test absl::unordered_set_lookup_test absl::unordered_set_members_test diff --git a/absl/container/flat_hash_map.h b/absl/container/flat_hash_map.h index 3eae404f..ebd9ed67 100644 --- a/absl/container/flat_hash_map.h +++ b/absl/container/flat_hash_map.h @@ -43,6 +43,7 @@ #include "absl/container/hash_container_defaults.h" #include "absl/container/internal/container_memory.h" #include "absl/container/internal/raw_hash_map.h" // IWYU pragma: export +#include "absl/meta/type_traits.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -575,6 +576,38 @@ typename flat_hash_map::size_type erase_if( namespace container_internal { +// c_for_each_fast(flat_hash_map<>, Function) +// +// Container-based version of the `std::for_each()` function to +// apply a function to a container's elements. +// There is no guarantees on the order of the function calls. +// Erasure and/or insertion of elements in the function is not allowed. +template +decay_t c_for_each_fast(const flat_hash_map& c, + Function&& f) { + container_internal::ForEach(f, &c); + return f; +} +template +decay_t c_for_each_fast(flat_hash_map& c, + Function&& f) { + container_internal::ForEach(f, &c); + return f; +} +template +decay_t c_for_each_fast(flat_hash_map&& c, + Function&& f) { + container_internal::ForEach(f, &c); + return f; +} + +} // namespace container_internal + +namespace container_internal { + template struct FlatHashMapPolicy { using slot_policy = container_internal::map_slot_policy; diff --git a/absl/container/flat_hash_map_test.cc b/absl/container/flat_hash_map_test.cc index eaa86925..08915e20 100644 --- a/absl/container/flat_hash_map_test.cc +++ b/absl/container/flat_hash_map_test.cc @@ -45,6 +45,7 @@ using ::testing::_; using ::testing::IsEmpty; using ::testing::Pair; using ::testing::UnorderedElementsAre; +using ::testing::UnorderedElementsAreArray; // Check that absl::flat_hash_map works in a global constructor. struct BeforeMain { @@ -307,6 +308,58 @@ TEST(FlatHashMap, EraseIf) { } } +TEST(FlatHashMap, CForEach) { + flat_hash_map m; + std::vector> expected; + for (int i = 0; i < 100; ++i) { + { + SCOPED_TRACE("mutable object iteration"); + std::vector> v; + absl::container_internal::c_for_each_fast( + m, [&v](std::pair& p) { v.push_back(p); }); + EXPECT_THAT(v, UnorderedElementsAreArray(expected)); + } + { + SCOPED_TRACE("const object iteration"); + std::vector> v; + const flat_hash_map& cm = m; + absl::container_internal::c_for_each_fast( + cm, [&v](const std::pair& p) { v.push_back(p); }); + EXPECT_THAT(v, UnorderedElementsAreArray(expected)); + } + { + SCOPED_TRACE("const object iteration"); + std::vector> v; + absl::container_internal::c_for_each_fast( + flat_hash_map(m), + [&v](std::pair& p) { v.push_back(p); }); + EXPECT_THAT(v, UnorderedElementsAreArray(expected)); + } + m[i] = i; + expected.emplace_back(i, i); + } +} + +TEST(FlatHashMap, CForEachMutate) { + flat_hash_map s; + std::vector> expected; + for (int i = 0; i < 100; ++i) { + std::vector> v; + absl::container_internal::c_for_each_fast( + s, [&v](std::pair& p) { + v.push_back(p); + p.second++; + }); + EXPECT_THAT(v, UnorderedElementsAreArray(expected)); + for (auto& p : expected) { + p.second++; + } + EXPECT_THAT(s, UnorderedElementsAreArray(expected)); + s[i] = i; + expected.emplace_back(i, i); + } +} + // This test requires std::launder for mutable key access in node handles. #if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606 TEST(FlatHashMap, NodeHandleMutableKeyAccess) { diff --git a/absl/container/flat_hash_set.h b/absl/container/flat_hash_set.h index e284a7df..a3e36e05 100644 --- a/absl/container/flat_hash_set.h +++ b/absl/container/flat_hash_set.h @@ -44,6 +44,7 @@ #include "absl/container/internal/container_memory.h" #include "absl/container/internal/raw_hash_set.h" // IWYU pragma: export #include "absl/memory/memory.h" +#include "absl/meta/type_traits.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -479,6 +480,33 @@ typename flat_hash_set::size_type erase_if( namespace container_internal { +// c_for_each_fast(flat_hash_set<>, Function) +// +// Container-based version of the `std::for_each()` function to +// apply a function to a container's elements. +// There is no guarantees on the order of the function calls. +// Erasure and/or insertion of elements in the function is not allowed. +template +decay_t c_for_each_fast(const flat_hash_set& c, + Function&& f) { + container_internal::ForEach(f, &c); + return f; +} +template +decay_t c_for_each_fast(flat_hash_set& c, Function&& f) { + container_internal::ForEach(f, &c); + return f; +} +template +decay_t c_for_each_fast(flat_hash_set&& c, Function&& f) { + container_internal::ForEach(f, &c); + return f; +} + +} // namespace container_internal + +namespace container_internal { + template struct FlatHashSetPolicy { using slot_type = T; diff --git a/absl/container/flat_hash_set_test.cc b/absl/container/flat_hash_set_test.cc index b425bc50..0dd43269 100644 --- a/absl/container/flat_hash_set_test.cc +++ b/absl/container/flat_hash_set_test.cc @@ -181,6 +181,39 @@ TEST(FlatHashSet, EraseIf) { } } +TEST(FlatHashSet, CForEach) { + using ValueType = std::pair; + flat_hash_set s; + std::vector expected; + for (int i = 0; i < 100; ++i) { + { + SCOPED_TRACE("mutable object iteration"); + std::vector v; + absl::container_internal::c_for_each_fast( + s, [&v](const ValueType& p) { v.push_back(p); }); + ASSERT_THAT(v, UnorderedElementsAreArray(expected)); + } + { + SCOPED_TRACE("const object iteration"); + std::vector v; + const flat_hash_set& cs = s; + absl::container_internal::c_for_each_fast( + cs, [&v](const ValueType& p) { v.push_back(p); }); + ASSERT_THAT(v, UnorderedElementsAreArray(expected)); + } + { + SCOPED_TRACE("temporary object iteration"); + std::vector v; + absl::container_internal::c_for_each_fast( + flat_hash_set(s), + [&v](const ValueType& p) { v.push_back(p); }); + ASSERT_THAT(v, UnorderedElementsAreArray(expected)); + } + s.emplace(i, i); + expected.emplace_back(i, i); + } +} + class PoisonSoo { int64_t data_; diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index c63b60e3..a722f522 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -4075,6 +4075,23 @@ struct HashtableFreeFunctionsAccess { }); return num_deleted; } + + template + static void ForEach(Callback& cb, Set* c) { + if (c->empty()) { + return; + } + if (c->is_soo()) { + cb(*c->soo_iterator()); + return; + } + using ElementTypeWithConstness = decltype(*c->begin()); + IterateOverFullSlots( + c->common(), c->slot_array(), [&cb](const ctrl_t*, auto* slot) { + ElementTypeWithConstness& element = Set::PolicyTraits::element(slot); + cb(element); + }); + } }; // Erases all elements that satisfy the predicate `pred` from the container `c`. @@ -4084,6 +4101,16 @@ typename raw_hash_set::size_type EraseIf( return HashtableFreeFunctionsAccess::EraseIf(pred, c); } +// Calls `cb` for all elements in the container `c`. +template +void ForEach(Callback& cb, raw_hash_set* c) { + return HashtableFreeFunctionsAccess::ForEach(cb, c); +} +template +void ForEach(Callback& cb, const raw_hash_set* c) { + return HashtableFreeFunctionsAccess::ForEach(cb, c); +} + namespace hashtable_debug_internal { template struct HashtableDebugAccess> { diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 2a6ee656..673d78a6 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -3121,6 +3121,58 @@ TYPED_TEST(SooTest, EraseIfPartial) { } } +TYPED_TEST(SooTest, ForEach) { + TypeParam t; + std::vector expected; + for (int size = 0; size < 100; ++size) { + SCOPED_TRACE(size); + { + SCOPED_TRACE("mutable iteration"); + std::vector actual; + auto f = [&](auto& x) { actual.push_back(static_cast(x)); }; + absl::container_internal::ForEach(f, &t); + ASSERT_THAT(actual, testing::UnorderedElementsAreArray(expected)); + } + { + SCOPED_TRACE("const iteration"); + std::vector actual; + auto f = [&](auto& x) { + static_assert( + std::is_const>::value, + "no mutable values should be passed to const ForEach"); + actual.push_back(static_cast(x)); + }; + const auto& ct = t; + absl::container_internal::ForEach(f, &ct); + ASSERT_THAT(actual, testing::UnorderedElementsAreArray(expected)); + } + t.insert(size); + expected.push_back(size); + } +} + +TEST(Table, ForEachMutate) { + StringTable t; + using ValueType = StringTable::value_type; + std::vector expected; + for (int size = 0; size < 100; ++size) { + SCOPED_TRACE(size); + std::vector actual; + auto f = [&](ValueType& x) { + actual.push_back(x); + x.second += "a"; + }; + absl::container_internal::ForEach(f, &t); + ASSERT_THAT(actual, testing::UnorderedElementsAreArray(expected)); + for (ValueType& v : expected) { + v.second += "a"; + } + ASSERT_THAT(t, testing::UnorderedElementsAreArray(expected)); + t.emplace(std::to_string(size), std::to_string(size)); + expected.emplace_back(std::to_string(size), std::to_string(size)); + } +} + TEST(Table, EraseBeginEndResetsReservedGrowth) { bool frozen = false; BadHashFreezableIntTable t{FreezableAlloc(&frozen)}; diff --git a/absl/container/node_hash_map.h b/absl/container/node_hash_map.h index 7b74b4b3..5615e496 100644 --- a/absl/container/node_hash_map.h +++ b/absl/container/node_hash_map.h @@ -50,6 +50,7 @@ #include "absl/container/internal/node_slot_policy.h" #include "absl/container/internal/raw_hash_map.h" // IWYU pragma: export #include "absl/memory/memory.h" +#include "absl/meta/type_traits.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -559,6 +560,38 @@ typename node_hash_map::size_type erase_if( namespace container_internal { +// c_for_each_fast(node_hash_map<>, Function) +// +// Container-based version of the `std::for_each()` function to +// apply a function to a container's elements. +// There is no guarantees on the order of the function calls. +// Erasure and/or insertion of elements in the function is not allowed. +template +decay_t c_for_each_fast(const node_hash_map& c, + Function&& f) { + container_internal::ForEach(f, &c); + return f; +} +template +decay_t c_for_each_fast(node_hash_map& c, + Function&& f) { + container_internal::ForEach(f, &c); + return f; +} +template +decay_t c_for_each_fast(node_hash_map&& c, + Function&& f) { + container_internal::ForEach(f, &c); + return f; +} + +} // namespace container_internal + +namespace container_internal { + template class NodeHashMapPolicy : public absl::container_internal::node_slot_policy< diff --git a/absl/container/node_hash_map_test.cc b/absl/container/node_hash_map_test.cc index 9bcf470c..2657828e 100644 --- a/absl/container/node_hash_map_test.cc +++ b/absl/container/node_hash_map_test.cc @@ -14,6 +14,18 @@ #include "absl/container/node_hash_map.h" +#include +#include // NOLINT: used for __cpp_lib_launder +#include +#include +#include +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/container/internal/hash_policy_testing.h" #include "absl/container/internal/tracked.h" #include "absl/container/internal/unordered_map_constructor_test.h" #include "absl/container/internal/unordered_map_lookup_test.h" @@ -29,6 +41,7 @@ using ::testing::Field; using ::testing::IsEmpty; using ::testing::Pair; using ::testing::UnorderedElementsAre; +using ::testing::UnorderedElementsAreArray; using MapTypes = ::testing::Types< absl::node_hash_map m; + std::vector> expected; + for (int i = 0; i < 100; ++i) { + { + SCOPED_TRACE("mutable object iteration"); + std::vector> v; + absl::container_internal::c_for_each_fast( + m, [&v](std::pair& p) { v.push_back(p); }); + EXPECT_THAT(v, UnorderedElementsAreArray(expected)); + } + { + SCOPED_TRACE("const object iteration"); + std::vector> v; + const node_hash_map& cm = m; + absl::container_internal::c_for_each_fast( + cm, [&v](const std::pair& p) { v.push_back(p); }); + EXPECT_THAT(v, UnorderedElementsAreArray(expected)); + } + { + SCOPED_TRACE("const object iteration"); + std::vector> v; + absl::container_internal::c_for_each_fast( + node_hash_map(m), + [&v](std::pair& p) { v.push_back(p); }); + EXPECT_THAT(v, UnorderedElementsAreArray(expected)); + } + m[i] = i; + expected.emplace_back(i, i); + } +} + +TEST(NodeHashMap, CForEachMutate) { + node_hash_map s; + std::vector> expected; + for (int i = 0; i < 100; ++i) { + std::vector> v; + absl::container_internal::c_for_each_fast( + s, [&v](std::pair& p) { + v.push_back(p); + p.second++; + }); + EXPECT_THAT(v, UnorderedElementsAreArray(expected)); + for (auto& p : expected) { + p.second++; + } + EXPECT_THAT(s, UnorderedElementsAreArray(expected)); + s[i] = i; + expected.emplace_back(i, i); + } +} + // This test requires std::launder for mutable key access in node handles. #if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606 TEST(NodeHashMap, NodeHandleMutableKeyAccess) { diff --git a/absl/container/node_hash_set.h b/absl/container/node_hash_set.h index 7228d192..53435ae6 100644 --- a/absl/container/node_hash_set.h +++ b/absl/container/node_hash_set.h @@ -48,6 +48,7 @@ #include "absl/container/internal/node_slot_policy.h" #include "absl/container/internal/raw_hash_set.h" // IWYU pragma: export #include "absl/memory/memory.h" +#include "absl/meta/type_traits.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -468,6 +469,33 @@ typename node_hash_set::size_type erase_if( namespace container_internal { +// c_for_each_fast(node_hash_set<>, Function) +// +// Container-based version of the `std::for_each()` function to +// apply a function to a container's elements. +// There is no guarantees on the order of the function calls. +// Erasure and/or insertion of elements in the function is not allowed. +template +decay_t c_for_each_fast(const node_hash_set& c, + Function&& f) { + container_internal::ForEach(f, &c); + return f; +} +template +decay_t c_for_each_fast(node_hash_set& c, Function&& f) { + container_internal::ForEach(f, &c); + return f; +} +template +decay_t c_for_each_fast(node_hash_set&& c, Function&& f) { + container_internal::ForEach(f, &c); + return f; +} + +} // namespace container_internal + +namespace container_internal { + template struct NodeHashSetPolicy : absl::container_internal::node_slot_policy> { diff --git a/absl/container/node_hash_set_test.cc b/absl/container/node_hash_set_test.cc index 98a8dbdd..e616ac1e 100644 --- a/absl/container/node_hash_set_test.cc +++ b/absl/container/node_hash_set_test.cc @@ -14,10 +14,22 @@ #include "absl/container/node_hash_set.h" +#include +#include +#include +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/container/internal/hash_generator_testing.h" +#include "absl/container/internal/hash_policy_testing.h" #include "absl/container/internal/unordered_set_constructor_test.h" #include "absl/container/internal/unordered_set_lookup_test.h" #include "absl/container/internal/unordered_set_members_test.h" #include "absl/container/internal/unordered_set_modifiers_test.h" +#include "absl/memory/memory.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -28,6 +40,7 @@ using ::absl::container_internal::hash_internal::EnumClass; using ::testing::IsEmpty; using ::testing::Pointee; using ::testing::UnorderedElementsAre; +using ::testing::UnorderedElementsAreArray; using SetTypes = ::testing::Types< node_hash_set>, @@ -137,6 +150,39 @@ TEST(NodeHashSet, EraseIf) { } } +TEST(NodeHashSet, CForEach) { + using ValueType = std::pair; + node_hash_set s; + std::vector expected; + for (int i = 0; i < 100; ++i) { + { + SCOPED_TRACE("mutable object iteration"); + std::vector v; + absl::container_internal::c_for_each_fast( + s, [&v](const ValueType& p) { v.push_back(p); }); + ASSERT_THAT(v, UnorderedElementsAreArray(expected)); + } + { + SCOPED_TRACE("const object iteration"); + std::vector v; + const node_hash_set& cs = s; + absl::container_internal::c_for_each_fast( + cs, [&v](const ValueType& p) { v.push_back(p); }); + ASSERT_THAT(v, UnorderedElementsAreArray(expected)); + } + { + SCOPED_TRACE("temporary object iteration"); + std::vector v; + absl::container_internal::c_for_each_fast( + node_hash_set(s), + [&v](const ValueType& p) { v.push_back(p); }); + ASSERT_THAT(v, UnorderedElementsAreArray(expected)); + } + s.emplace(i, i); + expected.emplace_back(i, i); + } +} + } // namespace } // namespace container_internal ABSL_NAMESPACE_END -- cgit v1.2.3 From 0f29d3e828949fbd7dd5e8e7e551b297ff2cdbee Mon Sep 17 00:00:00 2001 From: Vitaly Goldshteyn Date: Thu, 20 Jun 2024 23:05:54 -0700 Subject: Remove not used after all kAllowRemoveReentrance parameter from IterateOverFullSlots. We decided to not allow reentrance in absl::erase_if and absl::container_internal::c_for_each_fast. PiperOrigin-RevId: 645273965 Change-Id: I75dfc73b93ba10f0e051bf0833723af887e1bb36 --- absl/container/internal/raw_hash_set.h | 28 +++++++------------ absl/container/internal/raw_hash_set_test.cc | 41 ++-------------------------- 2 files changed, 13 insertions(+), 56 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index a722f522..724df193 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -1862,7 +1862,7 @@ inline void* SlotAddress(void* slot_array, size_t slot, size_t slot_size) { // Iterates over all full slots and calls `cb(const ctrl_t*, SlotType*)`. // If kAllowRemoveReentrance is false, no erasure from this table allowed during // Callback call. This mode is slightly faster. -template +template ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots( const CommonFields& c, SlotType* slot, Callback cb) { const size_t cap = c.capacity(); @@ -1885,28 +1885,20 @@ ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots( --ctrl; --slot; for (uint32_t i : mask) { - if (!kAllowRemoveReentrance || ABSL_PREDICT_TRUE(IsFull(ctrl[i]))) { - cb(ctrl + i, slot + i); - } + cb(ctrl + i, slot + i); } return; } size_t remaining = c.size(); while (remaining != 0) { for (uint32_t i : GroupFullEmptyOrDeleted(ctrl).MaskFull()) { - if (!kAllowRemoveReentrance || ABSL_PREDICT_TRUE(IsFull(ctrl[i]))) { - cb(ctrl + i, slot + i); - } + cb(ctrl + i, slot + i); --remaining; } ctrl += Group::kWidth; - if (kAllowRemoveReentrance && *(ctrl - 1) == ctrl_t::kSentinel) { - break; - } else { - assert((remaining == 0 || *(ctrl - 1) != ctrl_t::kSentinel) && - "element was erased from hash table unexpectedly"); - } slot += Group::kWidth; + assert((remaining == 0 || *(ctrl - 1) != ctrl_t::kSentinel) && + "element was erased from hash table unexpectedly"); } } @@ -2750,7 +2742,7 @@ class raw_hash_set { size_t offset = cap; const size_t shift = is_single_group(cap) ? (PerTableSalt(control()) | 1) : 0; - IterateOverFullSlots( + IterateOverFullSlots( that.common(), that.slot_array(), [&](const ctrl_t* that_ctrl, slot_type* that_slot) ABSL_ATTRIBUTE_ALWAYS_INLINE { @@ -3537,7 +3529,7 @@ class raw_hash_set { inline void destroy_slots() { assert(!is_soo()); if (PolicyTraits::template destroy_is_trivial()) return; - IterateOverFullSlots( + IterateOverFullSlots( common(), slot_array(), [&](const ctrl_t*, slot_type* slot) ABSL_ATTRIBUTE_ALWAYS_INLINE { this->destroy(slot); }); @@ -3882,7 +3874,7 @@ class raw_hash_set { } // We only do validation for small tables so that it's constant time. if (capacity() > 16) return; - IterateOverFullSlots( + IterateOverFullSlots( common(), slot_array(), assert_consistent); #endif } @@ -4064,7 +4056,7 @@ struct HashtableFreeFunctionsAccess { return 1; } size_t num_deleted = 0; - IterateOverFullSlots( + IterateOverFullSlots( c->common(), c->slot_array(), [&](const ctrl_t* ctrl, auto* slot) { if (pred(Set::PolicyTraits::element(slot))) { c->destroy(slot); @@ -4086,7 +4078,7 @@ struct HashtableFreeFunctionsAccess { return; } using ElementTypeWithConstness = decltype(*c->begin()); - IterateOverFullSlots( + IterateOverFullSlots( c->common(), c->slot_array(), [&cb](const ctrl_t*, auto* slot) { ElementTypeWithConstness& element = Set::PolicyTraits::element(slot); cb(element); diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 673d78a6..ba603d2b 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -3334,12 +3334,12 @@ TEST(Table, IterateOverFullSlotsEmpty) { auto fail_if_any = [](const ctrl_t*, auto* i) { FAIL() << "expected no slots " << **i; }; - container_internal::IterateOverFullSlots( + container_internal::IterateOverFullSlots( RawHashSetTestOnlyAccess::GetCommon(t), RawHashSetTestOnlyAccess::GetSlots(t), fail_if_any); for (size_t i = 0; i < 256; ++i) { t.reserve(i); - container_internal::IterateOverFullSlots( + container_internal::IterateOverFullSlots( RawHashSetTestOnlyAccess::GetCommon(t), RawHashSetTestOnlyAccess::GetSlots(t), fail_if_any); } @@ -3354,7 +3354,7 @@ TEST(Table, IterateOverFullSlotsFull) { expected_slots.push_back(idx); std::vector slots; - container_internal::IterateOverFullSlots( + container_internal::IterateOverFullSlots( RawHashSetTestOnlyAccess::GetCommon(t), RawHashSetTestOnlyAccess::GetSlots(t), [&t, &slots](const ctrl_t* ctrl, auto* i) { @@ -3368,41 +3368,6 @@ TEST(Table, IterateOverFullSlotsFull) { } } -TEST(Table, IterateOverFullSlotsAllowReentrance) { - std::vector expected_values; - for (int64_t idx = 0; idx < 128; ++idx) { - NonSooIntTable t; - for (int val = 0; val <= idx; ++val) { - t.insert(val); - } - - // We are inserting only even values. - // Only one element across 2*k and 2*k+1 should be visited. - if (idx % 2 == 0) { - expected_values.push_back(idx); - } - - std::vector actual_values; - container_internal::IterateOverFullSlots( - RawHashSetTestOnlyAccess::GetCommon(t), - RawHashSetTestOnlyAccess::GetSlots(t), - [&t, &actual_values](const ctrl_t* ctrl, auto* i) { - int64_t value = **i; - // Erase the other element from 2*k and 2*k+1 pair. - t.erase(value ^ 1); - ptrdiff_t ctrl_offset = - ctrl - RawHashSetTestOnlyAccess::GetCommon(t).control(); - ptrdiff_t slot_offset = i - RawHashSetTestOnlyAccess::GetSlots(t); - ASSERT_EQ(ctrl_offset, slot_offset); - // Add an even value from the pair. - // Only one element for each 2*k and 2*k+1 pair should be inserted. - actual_values.push_back(value - value % 2); - }); - EXPECT_THAT(actual_values, - testing::UnorderedElementsAreArray(expected_values)); - } -} - template class SooTable : public testing::Test {}; using FreezableSooTableTypes = -- cgit v1.2.3 From 0ccc51f9ddbb407d579f8158d5421fbf3eea0524 Mon Sep 17 00:00:00 2001 From: Vitaly Goldshteyn Date: Thu, 27 Jun 2024 02:09:33 -0700 Subject: Add assertions to detect reentrance in `IterateOverFullSlots` and `absl::erase_if`. Since we have potential plans to use this function more widely including `absl::c_for_each`, we need to have good error detection. PiperOrigin-RevId: 647236725 Change-Id: I5035bfb8cef24f80f1bbed83a42380e57d84e428 --- absl/container/internal/raw_hash_set.h | 18 ++++- absl/container/internal/raw_hash_set_test.cc | 117 +++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 3 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 724df193..02e389b9 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -1860,8 +1860,8 @@ inline void* SlotAddress(void* slot_array, size_t slot, size_t slot_size) { } // Iterates over all full slots and calls `cb(const ctrl_t*, SlotType*)`. -// If kAllowRemoveReentrance is false, no erasure from this table allowed during -// Callback call. This mode is slightly faster. +// No insertion to the table allowed during Callback call. +// Erasure is allowed only for the element passed to the callback. template ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots( const CommonFields& c, SlotType* slot, Callback cb) { @@ -1890,16 +1890,22 @@ ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots( return; } size_t remaining = c.size(); + ABSL_ATTRIBUTE_UNUSED const size_t original_size_for_assert = remaining; while (remaining != 0) { for (uint32_t i : GroupFullEmptyOrDeleted(ctrl).MaskFull()) { + assert(IsFull(ctrl[i]) && "hash table was modified unexpectedly"); cb(ctrl + i, slot + i); --remaining; } ctrl += Group::kWidth; slot += Group::kWidth; assert((remaining == 0 || *(ctrl - 1) != ctrl_t::kSentinel) && - "element was erased from hash table unexpectedly"); + "hash table was modified unexpectedly"); } + // NOTE: erasure of the current element is allowed in callback for + // absl::erase_if specialization. So we use `>=`. + assert(original_size_for_assert >= c.size() && + "hash table was modified unexpectedly"); } template @@ -4049,12 +4055,14 @@ struct HashtableFreeFunctionsAccess { if (c->is_soo()) { auto it = c->soo_iterator(); if (!pred(*it)) { + assert(c->size() == 1 && "hash table was modified unexpectedly"); return 0; } c->destroy(it.slot()); c->common().set_empty_soo(); return 1; } + ABSL_ATTRIBUTE_UNUSED const size_t original_size_for_assert = c->size(); size_t num_deleted = 0; IterateOverFullSlots( c->common(), c->slot_array(), [&](const ctrl_t* ctrl, auto* slot) { @@ -4065,6 +4073,10 @@ struct HashtableFreeFunctionsAccess { ++num_deleted; } }); + // NOTE: IterateOverFullSlots allow removal of the current element, so we + // verify the size additionally here. + assert(original_size_for_assert - num_deleted == c->size() && + "hash table was modified unexpectedly"); return num_deleted; } diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index ba603d2b..f1257d4b 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -3173,6 +3173,54 @@ TEST(Table, ForEachMutate) { } } +TYPED_TEST(SooTest, EraseIfReentryDeath) { + if (!IsAssertEnabled()) GTEST_SKIP() << "Assertions not enabled."; + + auto erase_if_with_removal_reentrance = [](size_t reserve_size) { + TypeParam t; + t.reserve(reserve_size); + int64_t first_value = -1; + t.insert(1024); + t.insert(5078); + auto pred = [&](const auto& x) { + if (first_value == -1) { + first_value = static_cast(x); + return false; + } + // We erase on second call to `pred` to reduce the chance that assertion + // will happen in IterateOverFullSlots. + t.erase(first_value); + return true; + }; + absl::container_internal::EraseIf(pred, &t); + }; + // Removal will likely happen in a different group. + EXPECT_DEATH_IF_SUPPORTED(erase_if_with_removal_reentrance(1024 * 16), + "hash table was modified unexpectedly"); + // Removal will happen in the same group. + EXPECT_DEATH_IF_SUPPORTED( + erase_if_with_removal_reentrance(CapacityToGrowth(Group::kWidth - 1)), + "hash table was modified unexpectedly"); +} + +// This test is useful to test soo branch. +TYPED_TEST(SooTest, EraseIfReentrySingleElementDeath) { + if (!IsAssertEnabled()) GTEST_SKIP() << "Assertions not enabled."; + + auto erase_if_with_removal_reentrance = []() { + TypeParam t; + t.insert(1024); + auto pred = [&](const auto& x) { + // We erase ourselves in order to confuse the erase_if. + t.erase(static_cast(x)); + return false; + }; + absl::container_internal::EraseIf(pred, &t); + }; + EXPECT_DEATH_IF_SUPPORTED(erase_if_with_removal_reentrance(), + "hash table was modified unexpectedly"); +} + TEST(Table, EraseBeginEndResetsReservedGrowth) { bool frozen = false; BadHashFreezableIntTable t{FreezableAlloc(&frozen)}; @@ -3368,6 +3416,75 @@ TEST(Table, IterateOverFullSlotsFull) { } } +TEST(Table, IterateOverFullSlotsDeathOnRemoval) { + if (!IsAssertEnabled()) GTEST_SKIP() << "Assertions not enabled."; + + auto iterate_with_reentrant_removal = [](int64_t size, + int64_t reserve_size = -1) { + if (reserve_size == -1) reserve_size = size; + for (int64_t idx = 0; idx < size; ++idx) { + NonSooIntTable t; + t.reserve(static_cast(reserve_size)); + for (int val = 0; val <= idx; ++val) { + t.insert(val); + } + + container_internal::IterateOverFullSlots( + RawHashSetTestOnlyAccess::GetCommon(t), + RawHashSetTestOnlyAccess::GetSlots(t), + [&t](const ctrl_t*, auto* i) { + int64_t value = **i; + // Erase the other element from 2*k and 2*k+1 pair. + t.erase(value ^ 1); + }); + } + }; + + EXPECT_DEATH_IF_SUPPORTED(iterate_with_reentrant_removal(128), + "hash table was modified unexpectedly"); + // Removal will likely happen in a different group. + EXPECT_DEATH_IF_SUPPORTED(iterate_with_reentrant_removal(14, 1024 * 16), + "hash table was modified unexpectedly"); + // Removal will happen in the same group. + EXPECT_DEATH_IF_SUPPORTED(iterate_with_reentrant_removal(static_cast( + CapacityToGrowth(Group::kWidth - 1))), + "hash table was modified unexpectedly"); +} + +TEST(Table, IterateOverFullSlotsDeathOnInsert) { + if (!IsAssertEnabled()) GTEST_SKIP() << "Assertions not enabled."; + + auto iterate_with_reentrant_insert = [](int64_t reserve_size, + int64_t size_divisor = 2) { + int64_t size = reserve_size / size_divisor; + for (int64_t idx = 1; idx <= size; ++idx) { + NonSooIntTable t; + t.reserve(static_cast(reserve_size)); + for (int val = 1; val <= idx; ++val) { + t.insert(val); + } + + container_internal::IterateOverFullSlots( + RawHashSetTestOnlyAccess::GetCommon(t), + RawHashSetTestOnlyAccess::GetSlots(t), + [&t](const ctrl_t*, auto* i) { + int64_t value = **i; + t.insert(-value); + }); + } + }; + + EXPECT_DEATH_IF_SUPPORTED(iterate_with_reentrant_insert(128), + "hash table was modified unexpectedly"); + // Insert will likely happen in a different group. + EXPECT_DEATH_IF_SUPPORTED(iterate_with_reentrant_insert(1024 * 16, 1024 * 2), + "hash table was modified unexpectedly"); + // Insert will happen in the same group. + EXPECT_DEATH_IF_SUPPORTED(iterate_with_reentrant_insert(static_cast( + CapacityToGrowth(Group::kWidth - 1))), + "hash table was modified unexpectedly"); +} + template class SooTable : public testing::Test {}; using FreezableSooTableTypes = -- cgit v1.2.3 From 4eb81046b2df1804220ef5af3fda54b6fa614c27 Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Mon, 1 Jul 2024 06:47:59 -0700 Subject: Static cast instead of reinterpret cast raw hash set slots as casting from void* to T* is well defined PiperOrigin-RevId: 648352837 Change-Id: I082cd0c007706ae8baa8f26cdc85d51b69bffd54 --- absl/container/internal/raw_hash_set.h | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 02e389b9..d4fe8f5c 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -878,7 +878,7 @@ struct GroupPortableImpl { // Note: this includes: kEmpty, kDeleted, kSentinel. // It is useful in contexts when kSentinel is not present. auto MaskNonFull() const { - return BitMask(ctrl & kMsbs8Bytes); + return BitMask(ctrl & kMsbs8Bytes); } NonIterableBitMask MaskEmptyOrDeleted() const { @@ -1134,9 +1134,7 @@ class GrowthInfo { // Returns true if the table satisfies two properties: // 1. Guaranteed to have no kDeleted slots. // 2. There is no growth left. - bool HasNoGrowthLeftAndNoDeleted() const { - return growth_left_info_ == 0; - } + bool HasNoGrowthLeftAndNoDeleted() const { return growth_left_info_ == 0; } // Returns true if table guaranteed to have no k bool HasNoDeleted() const { @@ -1144,9 +1142,7 @@ class GrowthInfo { } // Returns the number of elements left to grow. - size_t GetGrowthLeft() const { - return growth_left_info_ & kGrowthLeftMask; - } + size_t GetGrowthLeft() const { return growth_left_info_ & kGrowthLeftMask; } private: static constexpr size_t kGrowthLeftMask = ((~size_t{}) >> 1); @@ -1421,8 +1417,8 @@ class CommonFields : public CommonFieldsGenerationInfo { should_rehash_for_bug_detection_on_insert(control(), capacity()); } bool should_rehash_for_bug_detection_on_move() const { - return CommonFieldsGenerationInfo:: - should_rehash_for_bug_detection_on_move(control(), capacity()); + return CommonFieldsGenerationInfo::should_rehash_for_bug_detection_on_move( + control(), capacity()); } void reset_reserved_growth(size_t reservation) { CommonFieldsGenerationInfo::reset_reserved_growth(reservation, size()); @@ -1855,8 +1851,8 @@ constexpr size_t BackingArrayAlignment(size_t align_of_slot) { // Returns the address of the ith slot in slots where each slot occupies // slot_size. inline void* SlotAddress(void* slot_array, size_t slot, size_t slot_size) { - return reinterpret_cast(reinterpret_cast(slot_array) + - (slot * slot_size)); + return static_cast(static_cast(slot_array) + + (slot * slot_size)); } // Iterates over all full slots and calls `cb(const ctrl_t*, SlotType*)`. @@ -2100,8 +2096,8 @@ class HashSetResizeHelper { using slot_type = typename PolicyTraits::slot_type; assert(is_single_group(c.capacity())); - auto* new_slots = reinterpret_cast(c.slot_array()); - auto* old_slots_ptr = reinterpret_cast(old_slots()); + auto* new_slots = static_cast(c.slot_array()); + auto* old_slots_ptr = static_cast(old_slots()); size_t shuffle_bit = old_capacity_ / 2 + 1; for (size_t i = 0; i < old_capacity_; ++i) { @@ -3675,8 +3671,7 @@ class raw_hash_set { insert_slot(to_slot(resize_helper.old_soo_data())); return; } else { - auto* old_slots = - reinterpret_cast(resize_helper.old_slots()); + auto* old_slots = static_cast(resize_helper.old_slots()); size_t total_probe_length = 0; for (size_t i = 0; i != resize_helper.old_capacity(); ++i) { if (IsFull(resize_helper.old_ctrl()[i])) { @@ -3692,9 +3687,7 @@ class raw_hash_set { // Casting directly from e.g. char* to slot_type* can cause compilation errors // on objective-C. This function converts to void* first, avoiding the issue. - static slot_type* to_slot(void* buf) { - return reinterpret_cast(buf); - } + static slot_type* to_slot(void* buf) { return static_cast(buf); } // Requires that lhs does not have a full SOO slot. static void move_common(bool that_is_full_soo, allocator_type& rhs_alloc, @@ -3735,7 +3728,7 @@ class raw_hash_set { } } - template + template raw_hash_set& assign_impl(raw_hash_set&& that) { // We don't bother checking for this/that aliasing. We just need to avoid // breaking the invariants in that case. @@ -3880,8 +3873,7 @@ class raw_hash_set { } // We only do validation for small tables so that it's constant time. if (capacity() > 16) return; - IterateOverFullSlots( - common(), slot_array(), assert_consistent); + IterateOverFullSlots(common(), slot_array(), assert_consistent); #endif } -- cgit v1.2.3 From f36d33317ce3ca0a2212ffd264a26fd18e57a509 Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Mon, 1 Jul 2024 23:23:14 -0700 Subject: Make mutable CompressedTuple::get() constexpr. This change makes the mutable overloads of CompressedTuple::get() constexpr. This is consistent with std::get(std::tuple), which is constexpr since C++14. PiperOrigin-RevId: 648603141 Change-Id: Icbd61809f7a06723cf581dbed5488b7bae998cc9 --- absl/container/internal/compressed_tuple.h | 12 +++---- absl/container/internal/compressed_tuple_test.cc | 42 ++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 9 deletions(-) (limited to 'absl/container/internal') diff --git a/absl/container/internal/compressed_tuple.h b/absl/container/internal/compressed_tuple.h index f05a1fdc..6db0468d 100644 --- a/absl/container/internal/compressed_tuple.h +++ b/absl/container/internal/compressed_tuple.h @@ -89,9 +89,9 @@ struct Storage { explicit constexpr Storage(absl::in_place_t, V&& v) : value(std::forward(v)) {} constexpr const T& get() const& { return value; } - T& get() & { return value; } + constexpr T& get() & { return value; } constexpr const T&& get() const&& { return std::move(*this).value; } - T&& get() && { return std::move(*this).value; } + constexpr T&& get() && { return std::move(*this).value; } }; template @@ -102,9 +102,9 @@ struct ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC Storage : T { explicit constexpr Storage(absl::in_place_t, V&& v) : T(std::forward(v)) {} constexpr const T& get() const& { return *this; } - T& get() & { return *this; } + constexpr T& get() & { return *this; } constexpr const T&& get() const&& { return std::move(*this); } - T&& get() && { return std::move(*this); } + constexpr T&& get() && { return std::move(*this); } }; template @@ -237,7 +237,7 @@ class ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple std::forward(base)...) {} template - ElemT& get() & { + constexpr ElemT& get() & { return StorageT::get(); } @@ -247,7 +247,7 @@ class ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple } template - ElemT&& get() && { + constexpr ElemT&& get() && { return std::move(*this).StorageT::get(); } diff --git a/absl/container/internal/compressed_tuple_test.cc b/absl/container/internal/compressed_tuple_test.cc index 3cd9e18b..c3edf542 100644 --- a/absl/container/internal/compressed_tuple_test.cc +++ b/absl/container/internal/compressed_tuple_test.cc @@ -31,14 +31,22 @@ // These are declared at global scope purely so that error messages // are smaller and easier to understand. -enum class CallType { kConstRef, kConstMove }; +enum class CallType { kMutableRef, kConstRef, kMutableMove, kConstMove }; template struct Empty { + constexpr CallType value() & { return CallType::kMutableRef; } constexpr CallType value() const& { return CallType::kConstRef; } + constexpr CallType value() && { return CallType::kMutableMove; } constexpr CallType value() const&& { return CallType::kConstMove; } }; +// Unconditionally return an lvalue reference to `t`. +template +constexpr T& AsLValue(T&& t) { + return t; +} + template struct NotEmpty { T value; @@ -375,8 +383,24 @@ TEST(CompressedTupleTest, Constexpr) { constexpr int value() const { return v; } int v; }; - constexpr CompressedTuple, Empty<0>> x( - 7, 1.25, CompressedTuple(5), {}); + + using Tuple = CompressedTuple, Empty<0>>; + + constexpr int r0 = + AsLValue(Tuple(1, 0.75, CompressedTuple(9), {})).get<0>(); + constexpr double r1 = + AsLValue(Tuple(1, 0.75, CompressedTuple(9), {})).get<1>(); + constexpr int r2 = + AsLValue(Tuple(1, 0.75, CompressedTuple(9), {})).get<2>().get<0>(); + constexpr CallType r3 = + AsLValue(Tuple(1, 0.75, CompressedTuple(9), {})).get<3>().value(); + + EXPECT_EQ(r0, 1); + EXPECT_EQ(r1, 0.75); + EXPECT_EQ(r2, 9); + EXPECT_EQ(r3, CallType::kMutableRef); + + constexpr Tuple x(7, 1.25, CompressedTuple(5), {}); constexpr int x0 = x.get<0>(); constexpr double x1 = x.get<1>(); constexpr int x2 = x.get<2>().get<0>(); @@ -387,6 +411,18 @@ TEST(CompressedTupleTest, Constexpr) { EXPECT_EQ(x2, 5); EXPECT_EQ(x3, CallType::kConstRef); + constexpr int m0 = Tuple(5, 0.25, CompressedTuple(3), {}).get<0>(); + constexpr double m1 = Tuple(5, 0.25, CompressedTuple(3), {}).get<1>(); + constexpr int m2 = + Tuple(5, 0.25, CompressedTuple(3), {}).get<2>().get<0>(); + constexpr CallType m3 = + Tuple(5, 0.25, CompressedTuple(3), {}).get<3>().value(); + + EXPECT_EQ(m0, 5); + EXPECT_EQ(m1, 0.25); + EXPECT_EQ(m2, 3); + EXPECT_EQ(m3, CallType::kMutableMove); + constexpr CompressedTuple, TrivialStruct, int> trivial = {}; constexpr CallType trivial0 = trivial.get<0>().value(); constexpr int trivial1 = trivial.get<1>().value(); -- cgit v1.2.3