marker.cc

Go to the documentation of this file.

// Copyright 2020 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
 
#include "src/heap/cppgc/marker.h"
 
#include <cstddef>
#include <cstdint>
#include <memory>
 
#include "include/cppgc/heap-consistency.h"
#include "include/cppgc/platform.h"
#include "src/base/platform/time.h"
#include "src/heap/base/incremental-marking-schedule.h"
#include "src/heap/cppgc/globals.h"
#include "src/heap/cppgc/heap-config.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/heap-page.h"
#include "src/heap/cppgc/heap-visitor.h"
#include "src/heap/cppgc/heap.h"
#include "src/heap/cppgc/liveness-broker.h"
#include "src/heap/cppgc/marking-state.h"
#include "src/heap/cppgc/marking-visitor.h"
#include "src/heap/cppgc/marking-worklists.h"
#include "src/heap/cppgc/process-heap.h"
#include "src/heap/cppgc/stats-collector.h"
#include "src/heap/cppgc/write-barrier.h"
 
#if defined(CPPGC_CAGED_HEAP)
#include "include/cppgc/internal/caged-heap-local-data.h"
#endif
 
namespace cppgc {
namespace internal {
 
namespace {
 
bool EnterIncrementalMarkingIfNeeded(MarkingConfig config, HeapBase& heap) {
  if (config.marking_type == MarkingConfig::MarkingType::kIncremental ||
      config.marking_type ==
          MarkingConfig::MarkingType::kIncrementalAndConcurrent) {
    WriteBarrier::FlagUpdater::Enter();
    heap.set_incremental_marking_in_progress(true);
    return true;
  }
  return false;
}
 
bool ExitIncrementalMarkingIfNeeded(MarkingConfig config, HeapBase& heap) {
  if (config.marking_type == MarkingConfig::MarkingType::kIncremental ||
      config.marking_type ==
          MarkingConfig::MarkingType::kIncrementalAndConcurrent) {
    WriteBarrier::FlagUpdater::Exit();
    heap.set_incremental_marking_in_progress(false);
    return true;
  }
  return false;
}
 
static constexpr size_t kDefaultDeadlineCheckInterval = 150u;
 
template <StatsCollector::ScopeId scope_id,
          size_t kDeadlineCheckInterval = kDefaultDeadlineCheckInterval,
          typename WorklistLocal, typename Callback>
bool DrainWorklistWithBytesAndTimeDeadline(StatsCollector* stats_collector,
                                           BasicMarkingState& marking_state,
                                           size_t marked_bytes_deadline,
                                           v8::base::TimeTicks time_deadline,
                                           WorklistLocal& worklist_local,
                                           Callback callback) {
  return DrainWorklistWithPredicate<kDeadlineCheckInterval>(
      [&marking_state, marked_bytes_deadline, time_deadline]() {
        return (marked_bytes_deadline <= marking_state.marked_bytes()) ||
               (time_deadline <= v8::base::TimeTicks::Now());
      },
      [stats_collector]() {
        return StatsCollector::DisabledScope(stats_collector, scope_id);
      },
      worklist_local, callback);
}
 
size_t GetNextIncrementalStepDuration(
    heap::base::IncrementalMarkingSchedule& schedule, HeapBase& heap) {
  return schedule.GetNextIncrementalStepDuration(
      heap.stats_collector()->allocated_object_size());
}
 
}  // namespace
 
constexpr v8::base::TimeDelta MarkerBase::kMaximumIncrementalStepDuration;
 
class MarkerBase::IncrementalMarkingTask final : public cppgc::Task {
 public:
  using Handle = SingleThreadedHandle;
 
  IncrementalMarkingTask(MarkerBase*, StackState);
 
  static Handle Post(cppgc::TaskRunner*, MarkerBase*);
 
 private:
  void Run() final;
 
  MarkerBase* const marker_;
  StackState stack_state_;
  // TODO(chromium:1056170): Change to CancelableTask.
  Handle handle_;
};
 
MarkerBase::IncrementalMarkingTask::IncrementalMarkingTask(
    MarkerBase* marker, StackState stack_state)
    : marker_(marker),
      stack_state_(stack_state),
      handle_(Handle::NonEmptyTag{}) {}
 
// static
MarkerBase::IncrementalMarkingTask::Handle
MarkerBase::IncrementalMarkingTask::Post(cppgc::TaskRunner* runner,
                                         MarkerBase* marker) {
  // Incremental GC is possible only via the GCInvoker, so getting here
  // guarantees that either non-nestable tasks or conservative stack
  // scanning are supported. This is required so that the incremental
  // task can safely finalize GC if needed.
  DCHECK_IMPLIES(marker->heap().stack_support() !=
                     HeapBase::StackSupport::kSupportsConservativeStackScan,
                 runner->NonNestableTasksEnabled());
 
  const bool non_nestable_tasks_enabled = runner->NonNestableTasksEnabled();
 
  auto task = std::make_unique<IncrementalMarkingTask>(
      marker, non_nestable_tasks_enabled ? StackState::kNoHeapPointers
                                         : StackState::kMayContainHeapPointers);
  auto handle = task->handle_;
  if (non_nestable_tasks_enabled) {
    runner->PostNonNestableTask(std::move(task));
  } else {
    runner->PostTask(std::move(task));
  }
  return handle;
}
 
void MarkerBase::IncrementalMarkingTask::Run() {
  if (handle_.IsCanceled()) return;
 
  StatsCollector::EnabledScope stats_scope(marker_->heap().stats_collector(),
                                           StatsCollector::kIncrementalMark);
 
  if (marker_->IncrementalMarkingStep(stack_state_)) {
    // Incremental marking is done so should finalize GC.
    marker_->heap().FinalizeIncrementalGarbageCollectionIfNeeded(stack_state_);
  }
}
 
MarkerBase::MarkerBase(HeapBase& heap, cppgc::Platform* platform,
                       MarkingConfig config)
    : heap_(heap),
      config_(config),
      platform_(platform),
      foreground_task_runner_(platform_->GetForegroundTaskRunner()),
      incremental_marking_allocation_observer_(*this),
      mutator_marking_state_(heap, marking_worklists_,
                             heap.compactor().compaction_worklists()) {
  DCHECK_IMPLIES(config_.collection_type == CollectionType::kMinor,
                 heap_.generational_gc_supported());
}
 
MarkerBase::~MarkerBase() {
  // The fixed point iteration may have found not-fully-constructed objects.
  // Such objects should have already been found through the stack scan though
  // and should thus already be marked.
  if (!marking_worklists_.not_fully_constructed_worklist()->IsEmpty()) {
#if DEBUG
    DCHECK_NE(StackState::kNoHeapPointers, config_.stack_state);
    std::unordered_set<HeapObjectHeader*> objects =
        mutator_marking_state_.not_fully_constructed_worklist().Extract();
    for (HeapObjectHeader* object : objects) DCHECK(object->IsMarked());
#else
    marking_worklists_.not_fully_constructed_worklist()->Clear();
#endif
  }
 
  // |discovered_ephemeron_pairs_worklist_| may still hold ephemeron pairs with
  // dead keys.
  if (!marking_worklists_.discovered_ephemeron_pairs_worklist()->IsEmpty()) {
#if DEBUG
    MarkingWorklists::EphemeronPairItem item;
    while (mutator_marking_state_.discovered_ephemeron_pairs_worklist().Pop(
        &item)) {
      DCHECK(!HeapObjectHeader::FromObject(item.key).IsMarked());
    }
#else
    marking_worklists_.discovered_ephemeron_pairs_worklist()->Clear();
#endif
  }
 
  marking_worklists_.weak_containers_worklist()->Clear();
}
 
MarkerBase::IncrementalMarkingAllocationObserver::
    IncrementalMarkingAllocationObserver(MarkerBase& marker)
    : marker_(marker) {}
 
void MarkerBase::IncrementalMarkingAllocationObserver::
    AllocatedObjectSizeIncreased(size_t delta) {
  current_allocated_size_ += delta;
  if (current_allocated_size_ > kMinAllocatedBytesPerStep) {
    marker_.AdvanceMarkingOnAllocation();
    current_allocated_size_ = 0;
  }
}
 
void MarkerBase::StartMarking() {
  DCHECK(!is_marking_);
  StatsCollector::EnabledScope stats_scope(
      heap().stats_collector(),
      config_.marking_type == MarkingConfig::MarkingType::kAtomic
          ? StatsCollector::kAtomicMark
          : StatsCollector::kIncrementalMark);
 
  heap().stats_collector()->NotifyMarkingStarted(
      config_.collection_type, config_.marking_type, config_.is_forced_gc);
 
  is_marking_ = true;
  if (EnterIncrementalMarkingIfNeeded(config_, heap())) {
    StatsCollector::EnabledScope inner_stats_scope(
        heap().stats_collector(), StatsCollector::kMarkIncrementalStart);
 
    // Performing incremental or concurrent marking.
    schedule().NotifyIncrementalMarkingStart();
    // Scanning the stack is expensive so we only do it at the atomic pause.
    VisitLocalRoots(StackState::kNoHeapPointers);
    ScheduleIncrementalMarkingTask();
    if (config_.marking_type ==
        MarkingConfig::MarkingType::kIncrementalAndConcurrent) {
      mutator_marking_state_.Publish();
      concurrent_marker().Start();
    }
    MarkStrongCrossThreadRoots();
    heap().stats_collector()->RegisterObserver(
        &incremental_marking_allocation_observer_);
  }
}
 
void MarkerBase::HandleNotFullyConstructedObjects() {
  if (config_.stack_state == StackState::kNoHeapPointers) {
    mutator_marking_state_.FlushNotFullyConstructedObjects();
  } else {
    MarkNotFullyConstructedObjects();
  }
}
 
void MarkerBase::EnterAtomicPause(StackState stack_state) {
  StatsCollector::EnabledScope top_stats_scope(heap().stats_collector(),
                                               StatsCollector::kAtomicMark);
  StatsCollector::EnabledScope stats_scope(heap().stats_collector(),
                                           StatsCollector::kMarkAtomicPrologue);
 
  const MarkingConfig::MarkingType old_marking_type = config_.marking_type;
 
  if (ExitIncrementalMarkingIfNeeded(config_, heap())) {
    // Cancel remaining incremental tasks. Concurrent marking jobs are left to
    // run in parallel with the atomic pause until the mutator thread runs out
    // of work.
    incremental_marking_handle_.CancelIfNonEmpty();
    heap().stats_collector()->UnregisterObserver(
        &incremental_marking_allocation_observer_);
  }
  config_.stack_state = stack_state;
  config_.marking_type = MarkingConfig::MarkingType::kAtomic;
  mutator_marking_state_.set_in_atomic_pause();
 
  {
    // VisitLocalRoots() also resets the LABs.
    VisitLocalRoots(config_.stack_state);
    // Early marking of strong cross-thread roots before parallel marking. Helps
    // avoiding long single-threaded marking phases.
    MarkStrongCrossThreadRoots();
    HandleNotFullyConstructedObjects();
  }
  if (old_marking_type ==
      MarkingConfig::MarkingType::kIncrementalAndConcurrent) {
    // Start parallel marking.
    mutator_marking_state_.Publish();
    ConcurrentMarkerBase& marker = concurrent_marker();
    if (marker.IsActive()) {
      marker.NotifyIncrementalMutatorStepCompleted();
    } else {
      marker.Start();
    }
  }
}
 
void MarkerBase::ReEnableConcurrentMarking() {
  CHECK(is_marking_);
 
  if (config_.marking_type == MarkingConfig::MarkingType::kAtomic) {
    return;
  }
 
  CHECK_EQ(config_.marking_type, MarkingConfig::MarkingType::kIncremental);
  config_.marking_type = MarkingConfig::MarkingType::kIncrementalAndConcurrent;
  mutator_marking_state_.Publish();
  ConcurrentMarkerBase& marker = concurrent_marker();
  CHECK(!marker.IsActive());
  marker.Start();
  CHECK(marker.IsActive());
}
 
void MarkerBase::LeaveAtomicPause() {
  {
    StatsCollector::EnabledScope top_stats_scope(heap().stats_collector(),
                                                 StatsCollector::kAtomicMark);
    StatsCollector::EnabledScope stats_scope(
        heap().stats_collector(), StatsCollector::kMarkAtomicEpilogue);
    DCHECK(!incremental_marking_handle_);
    const size_t overall_marked_bytes =
        mutator_marking_state_.marked_bytes() +
        concurrent_marker().concurrently_marked_bytes();
    heap().stats_collector()->NotifyMarkingCompleted(overall_marked_bytes);
    is_marking_ = false;
  }
  ProcessWeakness();
  heap().SetStackStateOfPrevGC(config_.stack_state);
}
 
void MarkerBase::EnterProcessGlobalAtomicPause() { VisitCrossThreadRoots(); }
 
void MarkerBase::FinishMarking(StackState stack_state) {
  DCHECK(is_marking_);
  EnterAtomicPause(stack_state);
  EnterProcessGlobalAtomicPause();
  {
    StatsCollector::EnabledScope stats_scope(heap().stats_collector(),
                                             StatsCollector::kAtomicMark);
    CHECK(AdvanceMarkingWithLimits(v8::base::TimeDelta::Max(), SIZE_MAX));
    if (JoinConcurrentMarkingIfNeeded()) {
      CHECK(AdvanceMarkingWithLimits(v8::base::TimeDelta::Max(), SIZE_MAX));
    }
    mutator_marking_state_.Publish();
  }
  LeaveAtomicPause();
}
 
class WeakCallbackJobTask final : public cppgc::JobTask {
 public:
  WeakCallbackJobTask(MarkerBase* marker,
                      MarkingWorklists::WeakCallbackWorklist* callback_worklist,
                      LivenessBroker& broker)
      : marker_(marker),
        callback_worklist_(callback_worklist),
        broker_(broker) {}
 
  void Run(JobDelegate* delegate) override {
    StatsCollector::EnabledConcurrentScope stats_scope(
        marker_->heap().stats_collector(),
        StatsCollector::kConcurrentWeakCallback);
    MarkingWorklists::WeakCallbackWorklist::Local local(*callback_worklist_);
    MarkingWorklists::WeakCallbackItem item;
    while (local.Pop(&item)) {
      item.callback(broker_, item.parameter);
    }
  }
 
  size_t GetMaxConcurrency(size_t worker_count) const override {
    return std::min(static_cast<size_t>(1),
                    callback_worklist_->Size() + worker_count);
  }
 
 private:
  MarkerBase* marker_;
  MarkingWorklists::WeakCallbackWorklist* callback_worklist_;
  LivenessBroker& broker_;
};
 
void MarkerBase::ProcessCrossThreadWeaknessIfNeeded() {
  DCHECK_EQ(MarkingConfig::MarkingType::kAtomic, config_.marking_type);
 
  if (processed_cross_thread_weakness_) {
    return;
  }
 
  StatsCollector::EnabledScope stats_scope(heap().stats_collector(),
                                           StatsCollector::kAtomicWeak);
  // Weakness callbacks are forbidden from allocating objects.
  cppgc::subtle::DisallowGarbageCollectionScope disallow_gc_scope(heap_);
 
  RootMarkingVisitor root_marking_visitor(mutator_marking_state_);
 
  // Processing cross-thread roots requires taking the global process lock.
  // Process these weak roots first to minimize the time the lock is held.
  ProcessGlobalLock::AssertHeld();
  CHECK(visited_cross_thread_persistents_in_atomic_pause_);
  heap().GetWeakCrossThreadPersistentRegion().Iterate(root_marking_visitor);
  ProcessGlobalLock::Unlock<ProcessGlobalLock::Reason::kForGC>();
  processed_cross_thread_weakness_ = true;
}
 
void MarkerBase::ProcessWeakness() {
  DCHECK_EQ(MarkingConfig::MarkingType::kAtomic, config_.marking_type);
 
  ProcessCrossThreadWeaknessIfNeeded();
 
  // Weakness callbacks are forbidden from allocating objects.
  cppgc::subtle::DisallowGarbageCollectionScope disallow_gc_scope(heap_);
 
  StatsCollector::EnabledScope stats_scope(heap().stats_collector(),
                                           StatsCollector::kAtomicWeak);
 
  RootMarkingVisitor root_marking_visitor(mutator_marking_state_);
 
  // Launch the parallel job before anything else to provide the maximum time
  // slice for processing.
  LivenessBroker broker = LivenessBrokerFactory::Create();
  std::unique_ptr<cppgc::JobHandle> job_handle{nullptr};
  if (heap().marking_support() ==
      cppgc::Heap::MarkingType::kIncrementalAndConcurrent) {
    job_handle = platform_->PostJob(
        cppgc::TaskPriority::kUserBlocking,
        std::make_unique<WeakCallbackJobTask>(
            this, marking_worklists_.parallel_weak_callback_worklist(),
            broker));
  }
 
  // Process same-thread roots.
  heap().GetWeakPersistentRegion().Iterate(root_marking_visitor);
 
  // Call weak callbacks on objects that may now be pointing to dead objects.
#if defined(CPPGC_YOUNG_GENERATION)
  if (heap().generational_gc_supported()) {
    auto& remembered_set = heap().remembered_set();
    if (config_.collection_type == CollectionType::kMinor) {
      // Custom callbacks assume that untraced pointers point to not yet freed
      // objects. They must make sure that upon callback completion no
      // UntracedMember points to a freed object. This may not hold true if a
      // custom callback for an old object operates with a reference to a young
      // object that was freed on a minor collection cycle. To maintain the
      // invariant that UntracedMembers always point to valid objects, execute
      // custom callbacks for old objects on each minor collection cycle.
      remembered_set.ExecuteCustomCallbacks(broker);
    } else {
      // For major GCs, just release all the remembered weak callbacks.
      remembered_set.ReleaseCustomCallbacks();
    }
  }
#endif  // defined(CPPGC_YOUNG_GENERATION)
 
  {
    // First, process weak container callbacks.
    StatsCollector::EnabledScope inner_stats_scope(
        heap().stats_collector(),
        StatsCollector::kWeakContainerCallbacksProcessing);
    MarkingWorklists::WeakCallbackItem item;
    MarkingWorklists::WeakCallbackWorklist::Local& collections_local =
        mutator_marking_state_.weak_container_callback_worklist();
    while (collections_local.Pop(&item)) {
      item.callback(broker, item.parameter);
    }
  }
  {
    // Then, process custom weak callbacks.
    StatsCollector::EnabledScope inner_stats_scope(
        heap().stats_collector(), StatsCollector::kCustomCallbacksProcessing);
    MarkingWorklists::WeakCallbackItem item;
    MarkingWorklists::WeakCustomCallbackWorklist::Local& custom_callbacks =
        mutator_marking_state_.weak_custom_callback_worklist();
    while (custom_callbacks.Pop(&item)) {
      item.callback(broker, item.parameter);
#if defined(CPPGC_YOUNG_GENERATION)
      if (heap().generational_gc_supported())
        heap().remembered_set().AddWeakCallback(item);
#endif  // defined(CPPGC_YOUNG_GENERATION)
    }
  }
 
  if (job_handle) {
    job_handle->Join();
  } else {
    MarkingWorklists::WeakCallbackItem item;
    MarkingWorklists::WeakCallbackWorklist::Local& local =
        mutator_marking_state_.parallel_weak_callback_worklist();
    while (local.Pop(&item)) {
      item.callback(broker, item.parameter);
    }
  }
 
  // Weak callbacks should not add any new objects for marking.
  DCHECK(marking_worklists_.marking_worklist()->IsEmpty());
}
 
void MarkerBase::VisitLocalRoots(StackState stack_state) {
  StatsCollector::EnabledScope stats_scope(heap().stats_collector(),
                                           StatsCollector::kMarkVisitRoots);
 
  // Reset LABs before scanning roots. LABs are cleared to allow
  // ObjectStartBitmap handling without considering LABs.
  heap().object_allocator().ResetLinearAllocationBuffers();
 
  {
    StatsCollector::DisabledScope inner_stats_scope(
        heap().stats_collector(), StatsCollector::kMarkVisitPersistents);
    RootMarkingVisitor root_marking_visitor(mutator_marking_state_);
    heap().GetStrongPersistentRegion().Iterate(root_marking_visitor);
  }
 
  if (stack_state != StackState::kNoHeapPointers) {
    StatsCollector::DisabledScope stack_stats_scope(
        heap().stats_collector(), StatsCollector::kMarkVisitStack);
    heap().stack()->SetMarkerIfNeededAndCallback([this]() {
      heap().stack()->IteratePointersUntilMarker(&stack_visitor());
    });
  }
 
#if defined(CPPGC_YOUNG_GENERATION)
  if (config_.collection_type == CollectionType::kMinor) {
    StatsCollector::EnabledScope inner_stats_scope(
        heap().stats_collector(), StatsCollector::kMarkVisitRememberedSets);
    heap().remembered_set().Visit(visitor(), conservative_visitor(),
                                  mutator_marking_state_);
  }
#endif  // defined(CPPGC_YOUNG_GENERATION)
}
 
void MarkerBase::VisitCrossThreadRoots() {
  StatsCollector::DisabledScope inner_stats_scope(
      heap().stats_collector(),
      StatsCollector::kMarkVisitCrossThreadPersistents);
  CHECK_EQ(config_.marking_type, MarkingConfig::MarkingType::kAtomic);
  CHECK(!visited_cross_thread_persistents_in_atomic_pause_);
  // Lock guards against changes to {Weak}CrossThreadPersistent handles, that
  // may conflict with marking. E.g., a WeakCrossThreadPersistent may be
  // converted into a CrossThreadPersistent which requires that the handle
  // is either cleared or the object is retained.
  ProcessGlobalLock::Lock<ProcessGlobalLock::Reason::kForGC>();
  RootMarkingVisitor root_marking_visitor(mutator_marking_state_);
  heap().GetStrongCrossThreadPersistentRegion().Iterate(root_marking_visitor);
  visited_cross_thread_persistents_in_atomic_pause_ = true;
}
 
void MarkerBase::MarkStrongCrossThreadRoots() {
  ProcessGlobalLock::Lock<ProcessGlobalLock::Reason::kForGC>();
  RootMarkingVisitor root_marking_visitor(mutator_marking_state_);
  heap().GetStrongCrossThreadPersistentRegion().Iterate(root_marking_visitor);
  ProcessGlobalLock::Unlock<ProcessGlobalLock::Reason::kForGC>();
}
 
void MarkerBase::ScheduleIncrementalMarkingTask() {
  DCHECK(platform_);
  if (!foreground_task_runner_ || incremental_marking_handle_) return;
  incremental_marking_handle_ =
      IncrementalMarkingTask::Post(foreground_task_runner_.get(), this);
}
 
bool MarkerBase::IncrementalMarkingStepForTesting(StackState stack_state) {
  return IncrementalMarkingStep(stack_state);
}
 
bool MarkerBase::IncrementalMarkingStep(StackState stack_state) {
  if (stack_state == StackState::kNoHeapPointers) {
    mutator_marking_state_.FlushNotFullyConstructedObjects();
  }
  config_.stack_state = stack_state;
 
  return AdvanceMarkingWithLimits();
}
 
void MarkerBase::AdvanceMarkingOnAllocation() {
  StatsCollector::EnabledScope stats_scope(heap().stats_collector(),
                                           StatsCollector::kIncrementalMark);
  StatsCollector::EnabledScope nested_scope(heap().stats_collector(),
                                            StatsCollector::kMarkOnAllocation);
  AdvanceMarkingOnAllocationImpl();
}
 
void MarkerBase::AdvanceMarkingOnAllocationImpl() {
  if (AdvanceMarkingWithLimits()) {
    // Schedule another incremental task for finalizing without a stack.
    ScheduleIncrementalMarkingTask();
  }
}
 
bool MarkerBase::JoinConcurrentMarkingIfNeeded() {
  if (config_.marking_type != MarkingConfig::MarkingType::kAtomic ||
      !concurrent_marker().Join())
    return false;
 
  // Concurrent markers may have pushed some "leftover" in-construction objects
  // after flushing in EnterAtomicPause.
  HandleNotFullyConstructedObjects();
  DCHECK(marking_worklists_.not_fully_constructed_worklist()->IsEmpty());
  return true;
}
 
void MarkerBase::NotifyConcurrentMarkingOfWorkIfNeeded(
    cppgc::TaskPriority priority) {
  ConcurrentMarkerBase& marker = concurrent_marker();
  if (marker.IsActive()) {
    marker.NotifyOfWorkIfNeeded(priority);
  }
}
 
void MarkerBase::AdvanceMarkingWithLimitsEpilogue() {
  // Epilogue is only needed when marking is not finished.
  DCHECK_NE(MarkingConfig::MarkingType::kAtomic, config_.marking_type);
  ScheduleIncrementalMarkingTask();
  if (config_.marking_type ==
      MarkingConfig::MarkingType::kIncrementalAndConcurrent) {
    concurrent_marker().NotifyIncrementalMutatorStepCompleted();
  }
}
 
bool MarkerBase::AdvanceMarkingWithLimits(v8::base::TimeDelta max_duration,
                                          size_t marked_bytes_limit) {
  if (V8_UNLIKELY(main_marking_disabled_for_testing_)) {
    AdvanceMarkingWithLimitsEpilogue();
    return false;
  }
 
  StatsCollector::EnabledScope deadline_scope(
      heap().stats_collector(),
      StatsCollector::kMarkTransitiveClosureWithDeadline, "max_duration_ms",
      max_duration.InMillisecondsF(), "max_bytes", marked_bytes_limit);
  last_bytes_marked_ = 0;
  const auto deadline = v8::base::TimeTicks::Now() + max_duration;
  if (marked_bytes_limit == 0) {
    marked_bytes_limit = GetNextIncrementalStepDuration(schedule(), heap_);
  }
  // `ProcessWorklistsWithDeadline()` below checks against `marked_bytes()`
  // which are never reset.
  size_t marked_bytes_deadline =
      marked_bytes_limit + mutator_marking_state_.marked_bytes();
  if (marked_bytes_deadline < marked_bytes_limit) {
    marked_bytes_deadline = SIZE_MAX;
  }
  const bool is_done =
      ProcessWorklistsWithDeadline(marked_bytes_deadline, deadline);
  last_bytes_marked_ = mutator_marking_state_.RecentlyMarkedBytes();
  schedule().AddMutatorThreadMarkedBytes(last_bytes_marked_);
  mutator_marking_state_.Publish();
  if (!is_done) {
    AdvanceMarkingWithLimitsEpilogue();
  }
  return is_done;
}
 
bool MarkerBase::ProcessWorklistsWithDeadline(
    size_t marked_bytes_deadline, v8::base::TimeTicks time_deadline) {
  StatsCollector* stats_collector = heap().stats_collector();
  StatsCollector::EnabledScope stats_scope(
      stats_collector, StatsCollector::kMarkTransitiveClosure);
  bool saved_did_discover_new_ephemeron_pairs;
  do {
    mutator_marking_state_.ResetDidDiscoverNewEphemeronPairs();
    if ((config_.marking_type == MarkingConfig::MarkingType::kAtomic) ||
        schedule().ShouldFlushEphemeronPairs()) {
      mutator_marking_state_.FlushDiscoveredEphemeronPairs();
    }
 
    // Bailout objects may be complicated to trace and thus might take longer
    // than other objects. Therefore we reduce the interval between deadline
    // checks to guarantee the deadline is not exceeded.
    if (!DrainWorklistWithBytesAndTimeDeadline<
            StatsCollector::kMarkProcessBailOutObjects,
            kDefaultDeadlineCheckInterval / 5>(
            stats_collector, mutator_marking_state_, SIZE_MAX, time_deadline,
            mutator_marking_state_.concurrent_marking_bailout_worklist(),
            [this](const MarkingWorklists::ConcurrentMarkingBailoutItem& item) {
              mutator_marking_state_.AccountMarkedBytes(
                  BasePage::FromPayload(const_cast<void*>(item.parameter)),
                  item.bailedout_size);
              item.callback(&visitor(), item.parameter);
            })) {
      return false;
    }
 
    if (!DrainWorklistWithBytesAndTimeDeadline<
            StatsCollector::kMarkProcessNotFullyconstructedWorklist>(
            stats_collector, mutator_marking_state_, marked_bytes_deadline,
            time_deadline,
            mutator_marking_state_.previously_not_fully_constructed_worklist(),
            [this](HeapObjectHeader* header) {
              mutator_marking_state_.AccountMarkedBytes(*header);
              DynamicallyTraceMarkedObject<AccessMode::kNonAtomic>(visitor(),
                                                                   *header);
            })) {
      return false;
    }
 
    if (!DrainWorklistWithBytesAndTimeDeadline<
            StatsCollector::kMarkProcessMarkingWorklist>(
            stats_collector, mutator_marking_state_, marked_bytes_deadline,
            time_deadline, mutator_marking_state_.marking_worklist(),
            [this](const MarkingWorklists::MarkingItem& item) {
              const HeapObjectHeader& header =
                  HeapObjectHeader::FromObject(item.base_object_payload);
              DCHECK(!header.IsInConstruction<AccessMode::kNonAtomic>());
              DCHECK(header.IsMarked<AccessMode::kAtomic>());
              mutator_marking_state_.AccountMarkedBytes(header);
              item.callback(&visitor(), item.base_object_payload);
            })) {
      return false;
    }
 
    if (!DrainWorklistWithBytesAndTimeDeadline<
            StatsCollector::kMarkProcessWriteBarrierWorklist>(
            stats_collector, mutator_marking_state_, marked_bytes_deadline,
            time_deadline, mutator_marking_state_.write_barrier_worklist(),
            [this](HeapObjectHeader* header) {
              mutator_marking_state_.AccountMarkedBytes(*header);
              DynamicallyTraceMarkedObject<AccessMode::kNonAtomic>(visitor(),
                                                                   *header);
            })) {
      return false;
    }
 
    if (!DrainWorklistWithBytesAndTimeDeadline<
            StatsCollector::kMarkProcessRetraceWorklist>(
            stats_collector, mutator_marking_state_, marked_bytes_deadline,
            time_deadline,
            mutator_marking_state_.retrace_marked_objects_worklist(),
            [this](HeapObjectHeader* header) {
              // Retracing does not increment marked bytes as the object has
              // already been processed before.
              DynamicallyTraceMarkedObject<AccessMode::kNonAtomic>(visitor(),
                                                                   *header);
            })) {
      return false;
    }
 
    saved_did_discover_new_ephemeron_pairs =
        mutator_marking_state_.DidDiscoverNewEphemeronPairs();
    if (!DrainWorklistWithBytesAndTimeDeadline<
            StatsCollector::kMarkProcessEphemerons>(
            stats_collector, mutator_marking_state_, marked_bytes_deadline,
            time_deadline,
            mutator_marking_state_.ephemeron_pairs_for_processing_worklist(),
            [this](const MarkingWorklists::EphemeronPairItem& item) {
              mutator_marking_state_.ProcessEphemeron(
                  item.key, item.value, item.value_desc, visitor());
            })) {
      return false;
    }
  } while (!mutator_marking_state_.marking_worklist().IsLocalAndGlobalEmpty() ||
           saved_did_discover_new_ephemeron_pairs);
  return true;
}
 
void MarkerBase::MarkNotFullyConstructedObjects() {
  // Parallel marking may still be running which is why atomic extraction is
  // required.
  std::unordered_set<HeapObjectHeader*> objects =
      mutator_marking_state_.not_fully_constructed_worklist()
          .Extract<AccessMode::kAtomic>();
  if (objects.empty()) {
    return;
  }
  StatsCollector::DisabledScope stats_scope(
      heap().stats_collector(),
      StatsCollector::kMarkVisitNotFullyConstructedObjects);
  for (HeapObjectHeader* object : objects) {
    DCHECK(object);
    // TraceConservativelyIfNeeded delegates to either in-construction or
    // fully constructed handling. Both handlers have their own marked bytes
    // accounting and markbit handling (bailout).
    conservative_visitor().TraceConservativelyIfNeeded(*object);
  }
}
 
void MarkerBase::ClearAllWorklistsForTesting() {
  marking_worklists_.ClearForTesting();
  auto* compaction_worklists = heap_.compactor().compaction_worklists();
  if (compaction_worklists) compaction_worklists->ClearForTesting();
}
 
void MarkerBase::SetMainThreadMarkingDisabledForTesting(bool value) {
  main_marking_disabled_for_testing_ = value;
}
 
void MarkerBase::WaitForConcurrentMarkingForTesting() {
  concurrent_marker().Join();
}
 
MarkerBase::PauseConcurrentMarkingScope::PauseConcurrentMarkingScope(
    MarkerBase& marker)
    : marker_(marker), resume_on_exit_(marker_.concurrent_marker().Cancel()) {}
 
MarkerBase::PauseConcurrentMarkingScope::~PauseConcurrentMarkingScope() {
  if (resume_on_exit_) {
    marker_.concurrent_marker().Start();
  }
}
 
Marker::Marker(HeapBase& heap, cppgc::Platform* platform, MarkingConfig config)
    : MarkerBase(heap, platform, config),
      marking_visitor_(heap, mutator_marking_state_),
      conservative_marking_visitor_(heap, mutator_marking_state_,
                                    marking_visitor_),
      schedule_(::heap::base::IncrementalMarkingSchedule::Create()),
      concurrent_marker_(heap_, marking_worklists_, *schedule_.get(),
                         platform_) {}
 
}  // namespace internal
}  // namespace cppgc