Export of internal Abseil changes

--
9e0df3dd23da17cd0ff75c93c1493a858032639c by Martijn Vels <mvels@google.com>:

Prep work for changing Cordz instrumentation

Create CordzInfo::TrackCord() and CordzInfo::MaybeTrackCord() methods on InlineData

This follows a suggestion from kfm@ to move where possible Cordz logic out of Cord and Cord::InlineRep, which we can now that InlineData provides us the abstraction of Cords internal data.

PiperOrigin-RevId: 369844310

--
45f39709033bd3bc09fa1a7880a5b3c9eaa617c7 by Abseil Team <absl-team@google.com>:

Fix dependence on C++20 constexpr default construction of std::atomic.

PiperOrigin-RevId: 369745251

--
195ae230963c95068406ab0e73b4e711b5f3cd62 by Martijn Vels <mvels@google.com>:

Reduce the cost of 'SetCordRep()`

This change inlines SetCordRep(), which is currently two stores and a branch, going forward a naked store. AssertHeld is only enforced for debug builds.

The intention here is not to reduce the 'actual' (or 'self') cost of SetCordRep() as it should only be called for sampled cords, but to reduce the cost of a branch and out of line call at the call site. The compiler can now 'perfectly inline' the single branch / store.

PiperOrigin-RevId: 369696265

--
d722199ed69d413994740624159ac7bd001a9219 by Martijn Vels <mvels@google.com>:

Add kDefaultInit initialization

This avoids double store on init

PiperOrigin-RevId: 369655217

--
3499aed79e6cc12ce36277063ec37991bba0cccd by Martijn Vels <mvels@google.com>:

Introduce CordzUpdateScope

PiperOrigin-RevId: 369491326

--
325c1bc99c7d1aeca7bd1273e51a0900b6faf731 by Abseil Team <absl-team@google.com>:

make unary and logical operators constexpr

PiperOrigin-RevId: 369476773

--
ad3bed3dea5e5d7d281ff36ee786f802630c3728 by Martijn Vels <mvels@google.com>:

Add LOCKABLE attribute to CordzInfo

PiperOrigin-RevId: 369476772
GitOrigin-RevId: 9e0df3dd23da17cd0ff75c93c1493a858032639c
Change-Id: I00e2859328fe8da46d2e04d3f07dfe70ec6cb1f5

1 files changed, 62 insertions, 23 deletions

diff --git a/absl/strings/internal/cordz_info.h b/absl/strings/internal/cordz_info.h
index 5345be75..4cf57664 100644
--- a/absl/strings/internal/cordz_info.h
+++ b/absl/strings/internal/cordz_info.h
@@ -22,6 +22,7 @@
 #include "absl/base/config.h"
 #include "absl/base/thread_annotations.h"
 #include "absl/strings/internal/cord_internal.h"
+#include "absl/strings/internal/cordz_functions.h"
 #include "absl/strings/internal/cordz_handle.h"
 #include "absl/strings/internal/cordz_statistics.h"
 #include "absl/strings/internal/cordz_update_tracker.h"
@@ -41,22 +42,37 @@ namespace cord_internal {
 // and will either be deleted or appended to the global_delete_queue. If it is
 // placed on the global_delete_queue, the CordzInfo object will be cleaned in
 // the destructor of a CordzSampleToken object.
-class CordzInfo : public CordzHandle {
+class ABSL_LOCKABLE CordzInfo : public CordzHandle {
  public:
   using MethodIdentifier = CordzUpdateTracker::MethodIdentifier;
 
-  // All profiled Cords should be accompanied by a call to TrackCord.
   // TrackCord creates a CordzInfo instance which tracks important metrics of
-  // the sampled cord. CordzInfo instances are placed in a global list which is
-  // used to discover and snapshot all actively tracked cords.
-  // Callers are responsible for calling UntrackCord() before the tracked Cord
-  // instance is deleted, or to stop tracking the sampled Cord.
-  // Callers are also responsible for guarding changes to `rep` through the
-  // Lock() and Unlock() calls, and calling SetCordRep() if the root of the
-  // sampled cord changes before the old root has been unreffed and/or deleted.
-  // `method` identifies the Cord method which initiated the cord to be sampled.
-  static CordzInfo* TrackCord(
-      CordRep* rep, MethodIdentifier method = MethodIdentifier::kUnknown);
+  // a sampled cord, and stores the created CordzInfo instance into `cord'. All
+  // CordzInfo instances are placed in a global list which is used to discover
+  // and snapshot all actively tracked cords. Callers are responsible for
+  // calling UntrackCord() before the tracked Cord instance is deleted, or to
+  // stop tracking the sampled Cord. Callers are also responsible for guarding
+  // changes to the 'tree' value of a Cord (InlineData.tree) through the Lock()
+  // and Unlock() calls. Any change resulting in a new tree value for the cord
+  // requires a call to SetCordRep() before the old tree has been unreffed
+  // and/or deleted. `method` identifies the Cord public API method initiating
+  // the cord to be sampled.
+  // Requires `cord` to hold a tree, and `cord.cordz_info()` to be null.
+  static void TrackCord(InlineData& cord, MethodIdentifier method);
+
+  // Identical to TrackCord(), except that this function fills the
+  // `parent_stack` and `parent_method` properties of the returned CordzInfo
+  // instance from the provided `src` instance if `src` is sampled.
+  // This function should be used for sampling 'copy constructed' cords.
+  static void TrackCord(InlineData& cord, const InlineData& src,
+                        MethodIdentifier method);
+
+  // Maybe sample the cord identified by 'cord' for method 'method'.
+  // Uses `cordz_should_profile` to randomly pick cords to be sampled, and if
+  // so, invokes `TrackCord` to start sampling `cord`.
+  static void MaybeTrackCord(InlineData& cord, MethodIdentifier method);
+  static void MaybeTrackCord(InlineData& cord, const InlineData& src,
+                             MethodIdentifier method);
 
   // Stops tracking changes for a sampled cord, and deletes the provided info.
   // This function must be called before the sampled cord instance is deleted,
@@ -65,14 +81,6 @@ class CordzInfo : public CordzHandle {
   // CordInfo instance is being held by a concurrent collection thread.
   static void UntrackCord(CordzInfo* cordz_info);
 
-  // Identical to TrackCord(), except that this function fills the
-  // `parent_stack` and `parent_method` properties of the returned CordzInfo
-  // instance from the provided `src` instance if `src` is not null.
-  // This function should be used for sampling 'copy constructed' cords.
-  static CordzInfo* TrackCord(
-      CordRep* rep, const CordzInfo* src,
-      MethodIdentifier method = MethodIdentifier::kUnknown);
-
   CordzInfo() = delete;
   CordzInfo(const CordzInfo&) = delete;
   CordzInfo& operator=(const CordzInfo&) = delete;
@@ -92,6 +100,9 @@ class CordzInfo : public CordzHandle {
   // then this method will delete this CordzInfo instance.
   void Unlock() ABSL_UNLOCK_FUNCTION(mutex_);
 
+  // Asserts that this CordzInfo instance is locked.
+  void AssertHeld() ABSL_ASSERT_EXCLUSIVE_LOCK(mutex_);
+
   // Updates the `rep' property of this instance. This methods is invoked by
   // Cord logic each time the root node of a sampled Cord changes, and before
   // the old root reference count is deleted. This guarantees that collection
@@ -136,6 +147,9 @@ class CordzInfo : public CordzHandle {
                      MethodIdentifier method);
   ~CordzInfo() override;
 
+  void Track();
+  void Untrack();
+
   // Returns the parent method from `src`, which is either `parent_method_` or
   // `method_` depending on `parent_method_` being kUnknown.
   // Returns kUnknown if `src` is null.
@@ -147,9 +161,6 @@ class CordzInfo : public CordzHandle {
   // Returns 0 if `src` is null.
   static int FillParentStack(const CordzInfo* src, void** stack);
 
-  void Track();
-  void Untrack();
-
   // 'Unsafe' head/next/prev accessors not requiring the lock being held.
   // These are used exclusively for iterations (Head / Next) where we enforce
   // a token being held, so reading an 'old' / deleted pointer is fine.
@@ -184,6 +195,34 @@ class CordzInfo : public CordzHandle {
   std::atomic<int64_t> size_{0};
 };
 
+inline ABSL_ATTRIBUTE_ALWAYS_INLINE void CordzInfo::MaybeTrackCord(
+    InlineData& cord, MethodIdentifier method) {
+  if (ABSL_PREDICT_FALSE(cordz_should_profile())) {
+    TrackCord(cord, method);
+  }
+}
+
+inline ABSL_ATTRIBUTE_ALWAYS_INLINE void CordzInfo::MaybeTrackCord(
+    InlineData& cord, const InlineData& src, MethodIdentifier method) {
+  if (ABSL_PREDICT_FALSE(cordz_should_profile())) {
+    TrackCord(cord, src, method);
+  }
+}
+
+inline void CordzInfo::AssertHeld() ABSL_ASSERT_EXCLUSIVE_LOCK(mutex_) {
+#ifndef NDEBUG
+  mutex_.AssertHeld();
+#endif
+}
+
+inline void CordzInfo::SetCordRep(CordRep* rep) {
+  AssertHeld();
+  rep_ = rep;
+  if (rep) {
+    size_.store(rep->length);
+  }
+}
+
 }  // namespace cord_internal
 ABSL_NAMESPACE_END
 }  // namespace absl