code-range.cc

Go to the documentation of this file.

// Copyright 2021 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
 
#include "src/heap/code-range.h"
 
#include <algorithm>
#include <atomic>
#include <limits>
#include <utility>
 
#include "src/base/bits.h"
#include "src/base/lazy-instance.h"
#include "src/base/once.h"
#include "src/codegen/constants-arch.h"
#include "src/common/globals.h"
#include "src/flags/flags.h"
#include "src/heap/heap-inl.h"
#include "src/utils/allocation.h"
#if defined(V8_OS_WIN64)
#include "src/diagnostics/unwinding-info-win64.h"
#endif  // V8_OS_WIN64
 
namespace v8 {
namespace internal {
 
namespace {
 
DEFINE_LAZY_LEAKY_OBJECT_GETTER(CodeRangeAddressHint, GetCodeRangeAddressHint)
 
void FunctionInStaticBinaryForAddressHint() {}
 
}  // anonymous namespace
 
Address CodeRangeAddressHint::GetAddressHint(size_t code_range_size,
                                             size_t allocate_page_size) {
  base::MutexGuard guard(&mutex_);
 
  Address result = 0;
  auto it = recently_freed_.find(code_range_size);
  // No recently freed region has been found, try to provide a hint for placing
  // a code region.
  if (it == recently_freed_.end() || it->second.empty()) {
    return RoundUp(FUNCTION_ADDR(&FunctionInStaticBinaryForAddressHint),
                   allocate_page_size);
  }
 
  result = it->second.back();
  CHECK(IsAligned(result, allocate_page_size));
  it->second.pop_back();
  return result;
}
 
void CodeRangeAddressHint::NotifyFreedCodeRange(Address code_range_start,
                                                size_t code_range_size) {
  base::MutexGuard guard(&mutex_);
  recently_freed_[code_range_size].push_back(code_range_start);
}
 
CodeRange::~CodeRange() { Free(); }
 
// static
size_t CodeRange::GetWritableReservedAreaSize() {
  return kReservedCodeRangePages * MemoryAllocator::GetCommitPageSize();
}
 
#define TRACE(...) \
  if (v8_flags.trace_code_range_allocation) PrintF(__VA_ARGS__)
 
bool CodeRange::InitReservation(v8::PageAllocator* page_allocator,
                                size_t requested, bool immutable) {
  DCHECK_NE(requested, 0);
  if (V8_EXTERNAL_CODE_SPACE_BOOL) {
    page_allocator = GetPlatformPageAllocator();
  }
 
  if (requested <= kMinimumCodeRangeSize) {
    requested = kMinimumCodeRangeSize;
  }
 
  const size_t kPageSize = MutablePageMetadata::kPageSize;
  const size_t allocate_page_size = page_allocator->AllocatePageSize();
  CHECK(IsAligned(kPageSize, allocate_page_size));
 
  DCHECK_IMPLIES(kPlatformRequiresCodeRange,
                 requested <= kMaximalCodeRangeSize);
 
  VirtualMemoryCage::ReservationParams params;
  params.page_allocator = page_allocator;
  params.reservation_size = requested;
  params.base_alignment =
      VirtualMemoryCage::ReservationParams::kAnyBaseAlignment;
  params.page_size = kPageSize;
  if (v8_flags.jitless) {
    params.permissions = PageAllocator::Permission::kNoAccess;
    params.page_initialization_mode =
        base::PageInitializationMode::kAllocatedPagesCanBeUninitialized;
    params.page_freeing_mode = base::PageFreeingMode::kMakeInaccessible;
  } else {
    params.permissions = PageAllocator::Permission::kNoAccessWillJitLater;
    params.page_initialization_mode =
        base::PageInitializationMode::kRecommitOnly;
    params.page_freeing_mode = base::PageFreeingMode::kDiscard;
  }
 
#if defined(V8_TARGET_OS_IOS) || defined(V8_TARGET_OS_CHROMEOS)
  // iOS:
  // We only get one shot at doing MAP_JIT on iOS. So we need to make it
  // the least restrictive so it succeeds otherwise we will terminate the
  // process on the failed allocation.
  // ChromeOS:
  // Chrome on ChromeOS uses libgcc unwinding library which seems to work an
  // order of magnitude slower if we allocate CodeRange closer to the binary.
  // In non-official builds Chrome collects a lot of stack traces just in case,
  // so the slowdown of a single backtrace() call results in a noticeable
  // increase of test times. As a workaround, do a one shot allocation without
  // providing a hint.
  // TODO(https://crbug.com/40096218): investigate this ChromeOS issue.
  params.requested_start_hint = kNullAddress;
  if (!VirtualMemoryCage::InitReservation(params)) return false;
#else
  constexpr size_t kRadiusInMB =
      kMaxPCRelativeCodeRangeInMB > 1024 ? kMaxPCRelativeCodeRangeInMB : 4096;
  auto preferred_region = GetPreferredRegion(kRadiusInMB, kPageSize);
 
  TRACE("=== Preferred region: [%p, %p)\n",
        reinterpret_cast<void*>(preferred_region.begin()),
        reinterpret_cast<void*>(preferred_region.end()));
 
  // For configurations with enabled pointer compression and shared external
  // code range we can afford trying harder to allocate code range near .text
  // section.
  const bool kShouldTryHarder = V8_EXTERNAL_CODE_SPACE_BOOL &&
                                COMPRESS_POINTERS_IN_SHARED_CAGE_BOOL &&
                                v8_flags.better_code_range_allocation;
 
  if (kShouldTryHarder) {
    // TODO(v8:11880): consider using base::OS::GetFirstFreeMemoryRangeWithin()
    // to avoid attempts that's going to fail anyway.
 
    VirtualMemoryCage candidate_cage;
 
    // Try to allocate code range at the end of preferred region, by going
    // towards the start in steps.
    const int kAllocationTries = 16;
    params.requested_start_hint =
        RoundDown(preferred_region.end() - requested, kPageSize);
    Address step =
        RoundDown(preferred_region.size() / kAllocationTries, kPageSize);
    for (int i = 0; i < kAllocationTries; i++) {
      TRACE("=== Attempt #%d, hint=%p\n", i,
            reinterpret_cast<void*>(params.requested_start_hint));
      if (candidate_cage.InitReservation(params)) {
        TRACE("=== Attempt #%d (%p): [%p, %p)\n", i,
              reinterpret_cast<void*>(params.requested_start_hint),
              reinterpret_cast<void*>(candidate_cage.region().begin()),
              reinterpret_cast<void*>(candidate_cage.region().end()));
        // Allocation succeeded, check if it's in the preferred range.
        if (preferred_region.contains(candidate_cage.region())) break;
        // This allocation is not the one we are searhing for.
        candidate_cage.Free();
      }
      if (step == 0) break;
      params.requested_start_hint -= step;
    }
    if (candidate_cage.IsReserved()) {
      *static_cast<VirtualMemoryCage*>(this) = std::move(candidate_cage);
    }
  }
  if (!IsReserved()) {
    Address the_hint = GetCodeRangeAddressHint()->GetAddressHint(
        requested, allocate_page_size);
    // Last resort, use whatever region we could get with minimum constraints.
    params.requested_start_hint = the_hint;
    if (!VirtualMemoryCage::InitReservation(params)) {
      params.requested_start_hint = kNullAddress;
      if (!VirtualMemoryCage::InitReservation(params)) return false;
    }
    TRACE("=== Fallback attempt, hint=%p: [%p, %p)\n",
          reinterpret_cast<void*>(params.requested_start_hint),
          reinterpret_cast<void*>(region().begin()),
          reinterpret_cast<void*>(region().end()));
  }
 
  if (v8_flags.abort_on_far_code_range &&
      !preferred_region.contains(region())) {
    // We didn't manage to allocate the code range close enough.
    FATAL("Failed to allocate code range close to the .text section");
  }
#endif  // defined(V8_TARGET_OS_IOS) || defined(V8_TARGET_OS_CHROMEOS)
 
  // On some platforms, specifically Win64, we need to reserve some pages at
  // the beginning of an executable space. See
  //   https://cs.chromium.org/chromium/src/components/crash/content/
  //     app/crashpad_win.cc?rcl=fd680447881449fba2edcf0589320e7253719212&l=204
  // for details.
  const size_t required_writable_area_size = GetWritableReservedAreaSize();
  // The size of the area that might have been excluded from the area
  // allocatable by the BoundedPageAllocator.
  size_t excluded_allocatable_area_size = 0;
  if (required_writable_area_size > 0) {
    CHECK_LE(required_writable_area_size, kPageSize);
 
    // If the start of the reservation is not kPageSize-aligned then
    // there's a non-allocatable region before the area controlled by
    // the BoundedPageAllocator. Use it if it's big enough.
    const Address non_allocatable_size = page_allocator_->begin() - base();
 
    TRACE("=== non-allocatable region: [%p, %p)\n",
          reinterpret_cast<void*>(base()),
          reinterpret_cast<void*>(base() + non_allocatable_size));
 
    // Exclude the first page from allocatable pages if the required writable
    // area doesn't fit into the non-allocatable area.
    if (non_allocatable_size < required_writable_area_size) {
      TRACE("=== Exclude the first page from allocatable area\n");
      excluded_allocatable_area_size = kPageSize;
      CHECK(page_allocator_->AllocatePagesAt(page_allocator_->begin(),
                                             excluded_allocatable_area_size,
                                             PageAllocator::kNoAccess));
    }
    // Commit required amount of writable memory.
    if (!reservation()->SetPermissions(base(), required_writable_area_size,
                                       PageAllocator::kReadWrite)) {
      return false;
    }
#if defined(V8_OS_WIN64)
    if (win64_unwindinfo::CanRegisterUnwindInfoForNonABICompliantCodeRange()) {
      win64_unwindinfo::RegisterNonABICompliantCodeRange(
          reinterpret_cast<void*>(base()), size());
    }
#endif  // V8_OS_WIN64
  }
 
// Don't pre-commit the code cage on Windows since it uses memory and it's not
// required for recommit.
// iOS cannot adjust page permissions for MAP_JIT'd pages, they are set as RWX
// at the start.
#if !defined(V8_OS_WIN) && !defined(V8_OS_IOS)
  if (params.page_initialization_mode ==
      base::PageInitializationMode::kRecommitOnly) {
    void* base = reinterpret_cast<void*>(page_allocator_->begin() +
                                         excluded_allocatable_area_size);
    size_t size = page_allocator_->size() - excluded_allocatable_area_size;
    if (ThreadIsolation::Enabled()) {
      if (!ThreadIsolation::MakeExecutable(reinterpret_cast<Address>(base),
                                           size)) {
        return false;
      }
    } else if (!params.page_allocator->SetPermissions(
                   base, size, PageAllocator::kReadWriteExecute)) {
      return false;
    }
    if (immutable) {
#ifdef DEBUG
      immutable_ = true;
#endif
#ifdef V8_ENABLE_MEMORY_SEALING
      params.page_allocator->SealPages(base, size);
#endif
    }
    DiscardSealedMemoryScope discard_scope("Discard global code range.");
    if (!params.page_allocator->DiscardSystemPages(base, size)) return false;
  }
#endif  // !defined(V8_OS_WIN)
 
  return true;
}
 
// Preferred region for the code range is an intersection of the following
// regions:
// a) [builtins - kMaxPCRelativeDistance, builtins + kMaxPCRelativeDistance)
// b) [RoundDown(builtins, 4GB), RoundUp(builtins, 4GB)) in order to ensure
// Requirement (a) is there to avoid remaping of embedded builtins into
// the code for architectures where PC-relative jump/call distance is big
// enough.
// Requirement (b) is aiming at helping CPU branch predictors in general and
// in case V8_EXTERNAL_CODE_SPACE is enabled it ensures that
// ExternalCodeCompressionScheme works for all pointers in the code range.
// static
base::AddressRegion CodeRange::GetPreferredRegion(size_t radius_in_megabytes,
                                                  size_t allocate_page_size) {
#ifdef V8_TARGET_ARCH_64_BIT
  // Compute builtins location.
  Address embedded_blob_code_start =
      reinterpret_cast<Address>(Isolate::CurrentEmbeddedBlobCode());
  Address embedded_blob_code_end;
  if (embedded_blob_code_start == kNullAddress) {
    // When there's no embedded blob use address of a function from the binary
    // as an approximation.
    embedded_blob_code_start =
        FUNCTION_ADDR(&FunctionInStaticBinaryForAddressHint);
    embedded_blob_code_end = embedded_blob_code_start + 1;
  } else {
    embedded_blob_code_end =
        embedded_blob_code_start + Isolate::CurrentEmbeddedBlobCodeSize();
  }
 
  // Fulfil requirement (a).
  constexpr size_t max_size = std::numeric_limits<size_t>::max();
  size_t radius = radius_in_megabytes * MB;
 
  Address region_start =
      RoundUp(embedded_blob_code_end - radius, allocate_page_size);
  if (region_start > embedded_blob_code_end) {
    // |region_start| underflowed.
    region_start = 0;
  }
  Address region_end =
      RoundDown(embedded_blob_code_start + radius, allocate_page_size);
  if (region_end < embedded_blob_code_start) {
    // |region_end| overflowed.
    region_end = RoundDown(max_size, allocate_page_size);
  }
 
  // Fulfil requirement (b).
  constexpr size_t k4GB = size_t{4} * GB;
  Address four_gb_cage_start = RoundDown(embedded_blob_code_start, k4GB);
  Address four_gb_cage_end = four_gb_cage_start + k4GB;
 
  region_start = std::max(region_start, four_gb_cage_start);
  region_end = std::min(region_end, four_gb_cage_end);
 
  return base::AddressRegion(region_start, region_end - region_start);
#else
  return {};
#endif  // V8_TARGET_ARCH_64_BIT
}
 
void CodeRange::Free() {
  // TODO(361480580): this DCHECK is temporarily disabled since we free the
  // global CodeRange in the PoolTest.
  // DCHECK(!immutable_);
 
  if (IsReserved()) {
#if defined(V8_OS_WIN64)
    if (win64_unwindinfo::CanRegisterUnwindInfoForNonABICompliantCodeRange()) {
      win64_unwindinfo::UnregisterNonABICompliantCodeRange(
          reinterpret_cast<void*>(base()));
    }
#endif  // V8_OS_WIN64
    GetCodeRangeAddressHint()->NotifyFreedCodeRange(
        reservation()->region().begin(), reservation()->region().size());
    VirtualMemoryCage::Free();
  }
}
 
uint8_t* CodeRange::RemapEmbeddedBuiltins(Isolate* isolate,
                                          const uint8_t* embedded_blob_code,
                                          size_t embedded_blob_code_size) {
  base::MutexGuard guard(&remap_embedded_builtins_mutex_);
 
  // Remap embedded builtins into the end of the address range controlled by
  // the BoundedPageAllocator.
  const base::AddressRegion code_region(page_allocator()->begin(),
                                        page_allocator()->size());
  CHECK_NE(code_region.begin(), kNullAddress);
  CHECK(!code_region.is_empty());
 
  uint8_t* embedded_blob_code_copy =
      embedded_blob_code_copy_.load(std::memory_order_acquire);
  if (embedded_blob_code_copy) {
    DCHECK(
        code_region.contains(reinterpret_cast<Address>(embedded_blob_code_copy),
                             embedded_blob_code_size));
    SLOW_DCHECK(memcmp(embedded_blob_code, embedded_blob_code_copy,
                       embedded_blob_code_size) == 0);
    return embedded_blob_code_copy;
  }
 
  const size_t kAllocatePageSize = page_allocator()->AllocatePageSize();
  const size_t kCommitPageSize = page_allocator()->CommitPageSize();
  size_t allocate_code_size =
      RoundUp(embedded_blob_code_size, kAllocatePageSize);
 
  // Allocate the re-embedded code blob in such a way that it will be reachable
  // by PC-relative addressing from biggest possible region.
  const size_t max_pc_relative_code_range = kMaxPCRelativeCodeRangeInMB * MB;
  size_t hint_offset =
      std::min(max_pc_relative_code_range, code_region.size()) -
      allocate_code_size;
  void* hint = reinterpret_cast<void*>(code_region.begin() + hint_offset);
 
  embedded_blob_code_copy =
      reinterpret_cast<uint8_t*>(page_allocator()->AllocatePages(
          hint, allocate_code_size, kAllocatePageSize,
          PageAllocator::kNoAccessWillJitLater));
 
  if (!embedded_blob_code_copy) {
    V8::FatalProcessOutOfMemory(
        isolate, "Can't allocate space for re-embedded builtins");
  }
  CHECK_EQ(embedded_blob_code_copy, hint);
 
  if (code_region.size() > max_pc_relative_code_range) {
    // The re-embedded code blob might not be reachable from the end part of
    // the code range, so ensure that code pages will never be allocated in
    // the "unreachable" area.
    Address unreachable_start =
        reinterpret_cast<Address>(embedded_blob_code_copy) +
        max_pc_relative_code_range;
 
    if (code_region.contains(unreachable_start)) {
      size_t unreachable_size = code_region.end() - unreachable_start;
 
      void* result = page_allocator()->AllocatePages(
          reinterpret_cast<void*>(unreachable_start), unreachable_size,
          kAllocatePageSize, PageAllocator::kNoAccess);
      CHECK_EQ(reinterpret_cast<Address>(result), unreachable_start);
    }
  }
 
  size_t code_size = RoundUp(embedded_blob_code_size, kCommitPageSize);
  if constexpr (base::OS::IsRemapPageSupported()) {
    // By default, the embedded builtins are not remapped, but copied. This
    // costs memory, since builtins become private dirty anonymous memory,
    // rather than shared, clean, file-backed memory for the embedded version.
    // If the OS supports it, we can remap the builtins *on top* of the space
    // allocated in the code range, making the "copy" shared, clean, file-backed
    // memory, and thus saving sizeof(builtins).
    //
    // Builtins should start at a page boundary, see
    // platform-embedded-file-writer-mac.cc. If it's not the case (e.g. if the
    // embedded builtins are not coming from the binary), fall back to copying.
    if (IsAligned(reinterpret_cast<uintptr_t>(embedded_blob_code),
                  kCommitPageSize)) {
      bool ok = base::OS::RemapPages(embedded_blob_code, code_size,
                                     embedded_blob_code_copy,
                                     base::OS::MemoryPermission::kReadExecute);
 
      if (ok) {
        embedded_blob_code_copy_.store(embedded_blob_code_copy,
                                       std::memory_order_release);
        return embedded_blob_code_copy;
      }
    }
  }
 
  if (V8_HEAP_USE_PTHREAD_JIT_WRITE_PROTECT ||
      V8_HEAP_USE_BECORE_JIT_WRITE_PROTECT || ThreadIsolation::Enabled()) {
    // iOS code pages are already RWX and don't need to be modified.
#if !defined(V8_TARGET_OS_IOS)
    if (!page_allocator()->RecommitPages(embedded_blob_code_copy, code_size,
                                         PageAllocator::kReadWriteExecute)) {
      V8::FatalProcessOutOfMemory(isolate,
                                  "Re-embedded builtins: recommit pages");
    }
#endif  // defined(V8_TARGET_OS_IOS)
    RwxMemoryWriteScope rwx_write_scope(
        "Enable write access to copy the blob code into the code range");
    memcpy(embedded_blob_code_copy, embedded_blob_code,
           embedded_blob_code_size);
  } else {
    if (!page_allocator()->SetPermissions(embedded_blob_code_copy, code_size,
                                          PageAllocator::kReadWrite)) {
      V8::FatalProcessOutOfMemory(isolate,
                                  "Re-embedded builtins: set permissions");
    }
    memcpy(embedded_blob_code_copy, embedded_blob_code,
           embedded_blob_code_size);
 
    if (!page_allocator()->SetPermissions(embedded_blob_code_copy, code_size,
                                          PageAllocator::kReadExecute)) {
      V8::FatalProcessOutOfMemory(isolate,
                                  "Re-embedded builtins: set permissions");
    }
  }
  embedded_blob_code_copy_.store(embedded_blob_code_copy,
                                 std::memory_order_release);
  return embedded_blob_code_copy;
}
 
}  // namespace internal
}  // namespace v8