pgo.cc

Go to the documentation of this file.

// Copyright 2022 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
 
#include "src/wasm/pgo.h"
 
#include "src/wasm/decoder.h"
#include "src/wasm/wasm-module-builder.h"  // For {ZoneBuffer}.
 
namespace v8::internal::wasm {
 
constexpr uint8_t kFunctionExecutedBit = 1 << 0;
constexpr uint8_t kFunctionTieredUpBit = 1 << 1;
 
class ProfileGenerator {
 public:
  ProfileGenerator(const WasmModule* module,
                   const std::atomic<uint32_t>* tiering_budget_array)
      : module_(module),
        type_feedback_mutex_guard_(&module->type_feedback.mutex),
        tiering_budget_array_(tiering_budget_array) {}
 
  base::OwnedVector<uint8_t> GetProfileData() {
    ZoneBuffer buffer{&zone_};
 
    SerializeTypeFeedback(buffer);
    SerializeTieringInfo(buffer);
 
    return base::OwnedCopyOf(buffer);
  }
 
 private:
  void SerializeTypeFeedback(ZoneBuffer& buffer) {
    const std::unordered_map<uint32_t, FunctionTypeFeedback>&
        feedback_for_function = module_->type_feedback.feedback_for_function;
 
    // Get an ordered list of function indexes, so we generate deterministic
    // data.
    std::vector<uint32_t> ordered_function_indexes;
    ordered_function_indexes.reserve(feedback_for_function.size());
    for (const auto& entry : feedback_for_function) {
      // Skip functions for which we have no feedback.
      if (entry.second.feedback_vector.empty()) continue;
      ordered_function_indexes.push_back(entry.first);
    }
    std::sort(ordered_function_indexes.begin(), ordered_function_indexes.end());
 
    buffer.write_u32v(static_cast<uint32_t>(ordered_function_indexes.size()));
    for (const uint32_t func_index : ordered_function_indexes) {
      buffer.write_u32v(func_index);
      // Serialize {feedback_vector}.
      const FunctionTypeFeedback& feedback =
          feedback_for_function.at(func_index);
      buffer.write_u32v(static_cast<uint32_t>(feedback.feedback_vector.size()));
      for (const CallSiteFeedback& call_site_feedback :
           feedback.feedback_vector) {
        int cases = call_site_feedback.num_cases();
        buffer.write_i32v(cases);
        for (int i = 0; i < cases; ++i) {
          buffer.write_i32v(call_site_feedback.function_index(i));
          buffer.write_i32v(call_site_feedback.call_count(i));
        }
      }
      // Serialize {call_targets}.
      buffer.write_u32v(static_cast<uint32_t>(feedback.call_targets.size()));
      for (uint32_t call_target : feedback.call_targets) {
        buffer.write_u32v(call_target);
      }
    }
  }
 
  void SerializeTieringInfo(ZoneBuffer& buffer) {
    const std::unordered_map<uint32_t, FunctionTypeFeedback>&
        feedback_for_function = module_->type_feedback.feedback_for_function;
    const uint32_t initial_budget = v8_flags.wasm_tiering_budget;
    for (uint32_t declared_index = 0;
         declared_index < module_->num_declared_functions; ++declared_index) {
      uint32_t func_index = declared_index + module_->num_imported_functions;
      auto feedback_it = feedback_for_function.find(func_index);
      int prio = feedback_it == feedback_for_function.end()
                     ? 0
                     : feedback_it->second.tierup_priority;
      DCHECK_LE(0, prio);
      uint32_t remaining_budget =
          tiering_budget_array_[declared_index].load(std::memory_order_relaxed);
      DCHECK_GE(initial_budget, remaining_budget);
 
      bool was_tiered_up = prio > 0;
      bool was_executed = was_tiered_up || remaining_budget != initial_budget;
 
      // TODO(13209): Make this less V8-specific for productionization.
      buffer.write_u8((was_executed ? kFunctionExecutedBit : 0) |
                      (was_tiered_up ? kFunctionTieredUpBit : 0));
    }
  }
 
 private:
  const WasmModule* module_;
  AccountingAllocator allocator_;
  Zone zone_{&allocator_, "wasm::ProfileGenerator"};
  base::MutexGuard type_feedback_mutex_guard_;
  const std::atomic<uint32_t>* const tiering_budget_array_;
};
 
void DeserializeTypeFeedback(Decoder& decoder, const WasmModule* module) {
  base::MutexGuard mutex_guard{&module->type_feedback.mutex};
  std::unordered_map<uint32_t, FunctionTypeFeedback>& feedback_for_function =
      module->type_feedback.feedback_for_function;
  uint32_t num_entries = decoder.consume_u32v("num function entries");
  CHECK_LE(num_entries, module->num_declared_functions);
  for (uint32_t missing_entries = num_entries; missing_entries > 0;
       --missing_entries) {
    FunctionTypeFeedback function_feedback;
    uint32_t function_index = decoder.consume_u32v("function index");
    // Deserialize {feedback_vector}.
    uint32_t feedback_vector_size =
        decoder.consume_u32v("feedback vector size");
    function_feedback.feedback_vector =
        base::OwnedVector<CallSiteFeedback>::NewForOverwrite(
            feedback_vector_size);
    for (CallSiteFeedback& feedback : function_feedback.feedback_vector) {
      int num_cases = decoder.consume_i32v("num cases");
      if (num_cases == 0) continue;  // no feedback
      if (num_cases == 1) {          // monomorphic
        int called_function_index = decoder.consume_i32v("function index");
        int call_count = decoder.consume_i32v("call count");
        feedback = CallSiteFeedback{called_function_index, call_count};
      } else {  // polymorphic
        auto* polymorphic = new CallSiteFeedback::PolymorphicCase[num_cases];
        for (int i = 0; i < num_cases; ++i) {
          polymorphic[i].function_index =
              decoder.consume_i32v("function index");
          polymorphic[i].absolute_call_frequency =
              decoder.consume_i32v("call count");
        }
        feedback = CallSiteFeedback{polymorphic, num_cases};
      }
    }
    // Deserialize {call_targets}.
    uint32_t num_call_targets = decoder.consume_u32v("num call targets");
    function_feedback.call_targets =
        base::OwnedVector<uint32_t>::NewForOverwrite(num_call_targets);
    for (uint32_t& call_target : function_feedback.call_targets) {
      call_target = decoder.consume_u32v("call target");
    }
 
    // Finally, insert the new feedback into the map. Overwrite existing
    // feedback, but check for consistency.
    auto [feedback_it, is_new] = feedback_for_function.emplace(
        function_index, std::move(function_feedback));
    if (!is_new) {
      FunctionTypeFeedback& old_feedback = feedback_it->second;
      CHECK(old_feedback.feedback_vector.empty() ||
            old_feedback.feedback_vector.size() == feedback_vector_size);
      CHECK_EQ(old_feedback.call_targets.as_vector(),
               function_feedback.call_targets.as_vector());
      std::swap(old_feedback.feedback_vector,
                function_feedback.feedback_vector);
    }
  }
}
 
std::unique_ptr<ProfileInformation> DeserializeTieringInformation(
    Decoder& decoder, const WasmModule* module) {
  std::vector<uint32_t> executed_functions;
  std::vector<uint32_t> tiered_up_functions;
  uint32_t start = module->num_imported_functions;
  uint32_t end = start + module->num_declared_functions;
  for (uint32_t func_index = start; func_index < end; ++func_index) {
    uint8_t tiering_info = decoder.consume_u8("tiering info");
    CHECK_EQ(0, tiering_info & ~3);
    bool was_executed = tiering_info & kFunctionExecutedBit;
    bool was_tiered_up = tiering_info & kFunctionTieredUpBit;
    if (was_tiered_up) tiered_up_functions.push_back(func_index);
    if (was_executed) executed_functions.push_back(func_index);
  }
 
  return std::make_unique<ProfileInformation>(std::move(executed_functions),
                                              std::move(tiered_up_functions));
}
 
std::unique_ptr<ProfileInformation> RestoreProfileData(
    const WasmModule* module, base::Vector<uint8_t> profile_data) {
  Decoder decoder{profile_data.begin(), profile_data.end()};
 
  DeserializeTypeFeedback(decoder, module);
  std::unique_ptr<ProfileInformation> pgo_info =
      DeserializeTieringInformation(decoder, module);
 
  CHECK(decoder.ok());
  CHECK_EQ(decoder.pc(), decoder.end());
 
  return pgo_info;
}
 
void DumpProfileToFile(const WasmModule* module,
                       base::Vector<const uint8_t> wire_bytes,
                       std::atomic<uint32_t>* tiering_budget_array) {
  CHECK(!wire_bytes.empty());
  // File are named `profile-wasm-<hash>`.
  // We use the same hash as for reported scripts, to make it easier to
  // correlate files to wasm modules (see {CreateWasmScript}).
  uint32_t hash = static_cast<uint32_t>(GetWireBytesHash(wire_bytes));
  base::EmbeddedVector<char, 32> filename;
  SNPrintF(filename, "profile-wasm-%08x", hash);
 
  ProfileGenerator profile_generator{module, tiering_budget_array};
  base::OwnedVector<uint8_t> profile_data = profile_generator.GetProfileData();
 
  PrintF(
      "Dumping Wasm PGO data to file '%s' (module size %zu, %u declared "
      "functions, %zu bytes PGO data)\n",
      filename.begin(), wire_bytes.size(), module->num_declared_functions,
      profile_data.size());
  if (FILE* file = base::OS::FOpen(filename.begin(), "wb")) {
    size_t written = fwrite(profile_data.begin(), 1, profile_data.size(), file);
    CHECK_EQ(profile_data.size(), written);
    base::Fclose(file);
  }
}
 
std::unique_ptr<ProfileInformation> LoadProfileFromFile(
    const WasmModule* module, base::Vector<const uint8_t> wire_bytes) {
  CHECK(!wire_bytes.empty());
  // File are named `profile-wasm-<hash>`.
  // We use the same hash as for reported scripts, to make it easier to
  // correlate files to wasm modules (see {CreateWasmScript}).
  uint32_t hash = static_cast<uint32_t>(GetWireBytesHash(wire_bytes));
  base::EmbeddedVector<char, 32> filename;
  SNPrintF(filename, "profile-wasm-%08x", hash);
 
  FILE* file = base::OS::FOpen(filename.begin(), "rb");
  if (!file) {
    PrintF("No Wasm PGO data found: Cannot open file '%s'\n", filename.begin());
    return {};
  }
 
  fseek(file, 0, SEEK_END);
  size_t size = ftell(file);
  rewind(file);
 
  PrintF("Loading Wasm PGO data from file '%s' (%zu bytes)\n", filename.begin(),
         size);
  base::OwnedVector<uint8_t> profile_data =
      base::OwnedVector<uint8_t>::NewForOverwrite(size);
  for (size_t read = 0; read < size;) {
    read += fread(profile_data.begin() + read, 1, size - read, file);
    CHECK(!ferror(file));
  }
 
  base::Fclose(file);
 
  return RestoreProfileData(module, profile_data.as_vector());
}
 
}  // namespace v8::internal::wasm