Bug 1800263 - Part 3: Add a root marking mode with its own tracer r=sfink

This adds a new marking mode for root marking and templates MarkingTracer on a
options bitmask. The GCMarker's tracer becomes a variant and we change the
constructed variant when the mode changes.

The templating allows us to make parts of the marking path conditional using if
constexpr on the marking options.

Differential Revision: https://phabricator.services.mozilla.com/D162395

js/src/gc/GC.cpp

@@ -2729,7 +2729,9 @@ void GCRuntime::beginMarkPhase(AutoGCSession& session) {
     checkNoRuntimeRoots(session);
   } else {
     AutoUpdateLiveCompartments updateLive(this);
+    marker.setRootMarkingMode(true);
     traceRuntimeForMajorGC(marker.tracer(), session);
+    marker.setRootMarkingMode(false);
   }
 
   updateSchedulingStateOnGCStart();

js/src/gc/GCMarker.h

@@ -8,6 +8,7 @@
 #define gc_GCMarker_h
 
 #include "mozilla/Maybe.h"
+#include "mozilla/Variant.h"
 
 #include "ds/OrderedHashTable.h"
 #include "gc/Barrier.h"
@@ -201,17 +202,33 @@ class MarkStack {
 #endif
 };
 
-class MarkingTracer final : public GenericTracerImpl<MarkingTracer> {
+// Bitmask of options to parameterize MarkingTracerT.
+namespace MarkingOptions {
+enum : uint32_t {
+  None = 0,
+
+  // Set the compartment's hasMarkedCells flag for roots.
+  MarkRootCompartments = 1
+};
+}  // namespace MarkingOptions
+
+template <uint32_t markingOptions>
+class MarkingTracerT
+    : public GenericTracerImpl<MarkingTracerT<markingOptions>> {
  public:
-  explicit MarkingTracer(JSRuntime* runtime);
+  explicit MarkingTracerT(JSRuntime* runtime);
+  virtual ~MarkingTracerT() = default;
 
   template <typename T>
   void onEdge(T** thingp, const char* name);
-  friend class js::GenericTracerImpl<MarkingTracer>;
+  friend class GenericTracerImpl<MarkingTracerT<markingOptions>>;
 
   GCMarker* getMarker();
 };
 
+using MarkingTracer = MarkingTracerT<MarkingOptions::None>;
+using RootMarkingTracer = MarkingTracerT<MarkingOptions::MarkRootCompartments>;
+
 } /* namespace gc */
 
 class GCMarker {
@@ -219,6 +236,11 @@ class GCMarker {
     // Have not yet started marking.
     NotActive,
 
+    // Root marking mode. This sets the hasMarkedCells flag on compartments
+    // containing objects and scripts, which is used to make sure we clean up
+    // dead compartments.
+    RootMarking,
+
     // Main marking mode. Weakmap marking will be populating the
     // gcEphemeronEdges
     // tables but not consulting them. The state will transition to WeakMarking
@@ -242,7 +264,9 @@ class GCMarker {
   [[nodiscard]] bool init();
 
   JSRuntime* runtime() { return runtime_; }
-  JSTracer* tracer() { return &tracer_; }
+  JSTracer* tracer() {
+    return tracer_.match([](auto& t) -> JSTracer* { return &t; });
+  }
 
 #ifdef JS_GC_ZEAL
   void setMaxCapacity(size_t maxCap) { stack.setMaxCapacity(maxCap); }
@@ -300,6 +324,8 @@ class GCMarker {
   void setMarkColor(gc::MarkColor newColor);
   gc::MarkColor markColor() const { return markColor_; }
 
+  void setRootMarkingMode(bool newState);
+
   bool enterWeakMarkingMode();
   void leaveWeakMarkingMode();
 
@@ -424,8 +450,11 @@ class GCMarker {
   template <typename F>
   void forEachDelayedMarkingArena(F&& f);
 
-  /* The JSTracer used for marking, which calls back into this class. */
-  gc::MarkingTracer tracer_;
+  /*
+   * The JSTracer used for marking. This can change depending on the current
+   * state.
+   */
+  mozilla::Variant<gc::MarkingTracer, gc::RootMarkingTracer> tracer_;
 
   JSRuntime* const runtime_;
 

js/src/gc/Marking.cpp

@@ -903,20 +903,24 @@ static inline bool ShouldMark(GCMarker* gcmarker, T* thing) {
   return zone->shouldMarkInZone(gcmarker->markColor());
 }
 
-MarkingTracer::MarkingTracer(JSRuntime* runtime)
-    : GenericTracerImpl(runtime, JS::TracerKind::Marking,
-                        JS::TraceOptions(JS::WeakMapTraceAction::Expand,
-                                         JS::WeakEdgeTraceAction::Skip)) {
+template <uint32_t opts>
+MarkingTracerT<opts>::MarkingTracerT(JSRuntime* runtime)
+    : GenericTracerImpl<MarkingTracerT<opts>>(
+          runtime, JS::TracerKind::Marking,
+          JS::TraceOptions(JS::WeakMapTraceAction::Expand,
+                           JS::WeakEdgeTraceAction::Skip)) {
   // The MarkingTracer is owned by the the GCMarker.
   MOZ_ASSERT(this == runtime->gc.marker.tracer());
 }
 
-MOZ_ALWAYS_INLINE GCMarker* MarkingTracer::getMarker() {
+template <uint32_t opts>
+MOZ_ALWAYS_INLINE GCMarker* MarkingTracerT<opts>::getMarker() {
   return GCMarker::fromTracer(this);
 }
 
+template <uint32_t opts>
 template <typename T>
-void MarkingTracer::onEdge(T** thingp, const char* name) {
+void MarkingTracerT<opts>::onEdge(T** thingp, const char* name) {
   T* thing = *thingp;
 
   // Do per-type marking precondition checks.
@@ -927,7 +931,7 @@ void MarkingTracer::onEdge(T** thingp, const char* name) {
     return;
   }
 
-  MOZ_ASSERT(!IsOwnedByOtherRuntime(runtime(), thing));
+  MOZ_ASSERT(!IsOwnedByOtherRuntime(this->runtime(), thing));
 
 #ifdef DEBUG
   CheckMarkedThing(marker, thing);
@@ -936,12 +940,18 @@ void MarkingTracer::onEdge(T** thingp, const char* name) {
   AutoClearTracingSource acts(this);
   marker->markAndTraverse(thing);
 
-  // Mark the compartment as live.
-  SetCompartmentHasMarkedCells(thing);
+  if constexpr (opts == MarkingOptions::MarkRootCompartments) {
+    // Mark the compartment as live.
+    SetCompartmentHasMarkedCells(thing);
+  }
 }
 
-#define INSTANTIATE_ONEDGE_METHOD(name, type, _1, _2) \
-  template void MarkingTracer::onEdge<type>(type * *thingp, const char* name);
+#define INSTANTIATE_ONEDGE_METHOD(name, type, _1, _2)                 \
+  template void MarkingTracerT<MarkingOptions::None>::onEdge<type>(   \
+      type * *thingp, const char* name);                              \
+  template void                                                       \
+  MarkingTracerT<MarkingOptions::MarkRootCompartments>::onEdge<type>( \
+      type * *thingp, const char* name);
 JS_FOR_EACH_TRACEKIND(INSTANTIATE_ONEDGE_METHOD)
 #undef INSTANTIATE_ONEDGE_METHOD
 
@@ -1767,7 +1777,7 @@ size_t MarkStack::sizeOfExcludingThis(
  * potential key.
  */
 GCMarker::GCMarker(JSRuntime* rt)
-    : tracer_(rt),
+    : tracer_(mozilla::VariantType<MarkingTracer>(), rt),
       runtime_(rt),
       stack(),
       grayPosition(0),
@@ -1905,6 +1915,18 @@ void GCMarker::repush(JSObject* obj) {
   pushTaggedPtr(obj);
 }
 
+void GCMarker::setRootMarkingMode(bool newState) {
+  if (newState) {
+    MOZ_ASSERT(state == RegularMarking);
+    state = RootMarking;
+    tracer_.emplace<RootMarkingTracer>(runtime());
+  } else {
+    MOZ_ASSERT(state == RootMarking);
+    state = RegularMarking;
+    tracer_.emplace<MarkingTracer>(runtime());
+  }
+}
+
 bool GCMarker::enterWeakMarkingMode() {
   MOZ_ASSERT(tracer()->weakMapAction() == JS::WeakMapTraceAction::Expand);
   MOZ_ASSERT(state != WeakMarking);

js/src/gc/Sweeping.cpp

@@ -561,8 +561,12 @@ IncrementalProgress GCRuntime::markGrayRoots(SliceBudget& budget,
   gcstats::AutoPhase ap(stats(), phase);
 
   AutoUpdateLiveCompartments updateLive(this);
+  marker.setRootMarkingMode(true);
+  auto guard =
+      mozilla::MakeScopeExit([this]() { marker.setRootMarkingMode(false); });
 
-  if (traceEmbeddingGrayRoots(marker.tracer(), budget) == NotFinished) {
+  IncrementalProgress result = traceEmbeddingGrayRoots(marker.tracer(), budget);
+  if (result == NotFinished) {
     return NotFinished;
   }