turboshaft-graph-interface.cc

Go to the documentation of this file.

// Copyright 2023 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
 
#include "src/wasm/turboshaft-graph-interface.h"
 
#include <optional>
 
#include "absl/container/btree_map.h"
#include "include/v8-fast-api-calls.h"
#include "src/base/logging.h"
#include "src/builtins/builtins.h"
#include "src/builtins/data-view-ops.h"
#include "src/common/globals.h"
#include "src/compiler/turboshaft/assembler.h"
#include "src/compiler/turboshaft/builtin-call-descriptors.h"
#include "src/compiler/turboshaft/graph.h"
#include "src/compiler/turboshaft/wasm-assembler-helpers.h"
#include "src/compiler/wasm-compiler-definitions.h"
#include "src/objects/object-list-macros.h"
#include "src/objects/torque-defined-classes.h"
#include "src/trap-handler/trap-handler.h"
#include "src/wasm/compilation-environment.h"
#include "src/wasm/function-body-decoder-impl.h"
#include "src/wasm/function-compiler.h"
#include "src/wasm/inlining-tree.h"
#include "src/wasm/jump-table-assembler.h"
#include "src/wasm/memory-tracing.h"
#include "src/wasm/wasm-engine.h"
#include "src/wasm/wasm-linkage.h"
#include "src/wasm/wasm-objects-inl.h"
#include "src/wasm/wasm-objects.h"
#include "src/wasm/wasm-opcodes-inl.h"
 
namespace v8::internal::wasm {
 
#include "src/compiler/turboshaft/define-assembler-macros.inc"
 
using compiler::AccessBuilder;
using compiler::CallDescriptor;
using compiler::MemoryAccessKind;
using compiler::Operator;
using compiler::TrapId;
using TSBlock = compiler::turboshaft::Block;
using compiler::turboshaft::BuiltinCallDescriptor;
using compiler::turboshaft::CallOp;
using compiler::turboshaft::ConditionWithHint;
using compiler::turboshaft::ConstantOp;
using compiler::turboshaft::ConstOrV;
using compiler::turboshaft::DidntThrowOp;
using compiler::turboshaft::Float32;
using compiler::turboshaft::Float64;
using compiler::turboshaft::FrameState;
using compiler::turboshaft::Graph;
using compiler::turboshaft::Label;
using compiler::turboshaft::LoadOp;
using compiler::turboshaft::LoopLabel;
using compiler::turboshaft::MemoryRepresentation;
using compiler::turboshaft::OpEffects;
using compiler::turboshaft::Operation;
using compiler::turboshaft::OperationMatcher;
using compiler::turboshaft::OpIndex;
using compiler::turboshaft::OptionalOpIndex;
using compiler::turboshaft::OptionalV;
using compiler::turboshaft::PendingLoopPhiOp;
using compiler::turboshaft::RegisterRepresentation;
using compiler::turboshaft::Simd128ConstantOp;
using compiler::turboshaft::StoreOp;
using compiler::turboshaft::StringOrNull;
using compiler::turboshaft::SupportedOperations;
using compiler::turboshaft::Tuple;
using compiler::turboshaft::V;
using compiler::turboshaft::Variable;
using compiler::turboshaft::WasmArrayNullable;
using compiler::turboshaft::WasmStackCheckOp;
using compiler::turboshaft::WasmStringRefNullable;
using compiler::turboshaft::WasmStructNullable;
using compiler::turboshaft::WasmTypeAnnotationOp;
using compiler::turboshaft::WasmTypeCastOp;
using compiler::turboshaft::Word32;
using compiler::turboshaft::WordPtr;
using compiler::turboshaft::WordRepresentation;
 
namespace {
 
ExternalArrayType GetExternalArrayType(DataViewOp op_type) {
  switch (op_type) {
#define V(Name)                \
  case DataViewOp::kGet##Name: \
  case DataViewOp::kSet##Name: \
    return kExternal##Name##Array;
    DATAVIEW_OP_LIST(V)
#undef V
    case DataViewOp::kByteLength:
      UNREACHABLE();
  }
}
 
size_t GetTypeSize(DataViewOp op_type) {
  ExternalArrayType array_type = GetExternalArrayType(op_type);
  switch (array_type) {
#define ELEMENTS_KIND_TO_ELEMENT_SIZE(Type, type, TYPE, ctype) \
  case kExternal##Type##Array:                                 \
    return sizeof(ctype);
 
    TYPED_ARRAYS(ELEMENTS_KIND_TO_ELEMENT_SIZE)
#undef ELEMENTS_KIND_TO_ELEMENT_SIZE
  }
}
 
bool ReverseBytesSupported(size_t size_in_bytes) {
  switch (size_in_bytes) {
    case 4:
    case 16:
      return true;
    case 8:
      return Is64();
    default:
      return false;
  }
}
 
enum class BranchHintingMode {
  kNone,
  kModuleProvided,
  kStress,
};
 
class BranchHintingStresser {
 public:
  BranchHint GetNextHint() {
    // To strike a balance between randomness and simplicity, we simply
    // iterate over the bits of the random seed.
    int bit = v8_flags.random_seed & (1 << cursor_);
    static_assert(sizeof(v8_flags.random_seed) * kBitsPerByte == 32);
    cursor_ = (cursor_ + 1) & 31;
    return bit == 0 ? BranchHint::kFalse : BranchHint::kTrue;
  }
 
 private:
  int cursor_{0};
};
 
}  // namespace
 
// TODO(14108): Annotate runtime functions as not having side effects
// where appropriate.
OpIndex WasmGraphBuilderBase::CallRuntime(
    Zone* zone, Runtime::FunctionId f,
    std::initializer_list<const OpIndex> args, V<Context> context) {
  const Runtime::Function* fun = Runtime::FunctionForId(f);
  OpIndex isolate_root = __ LoadRootRegister();
  DCHECK_EQ(1, fun->result_size);
  int builtin_slot_offset =
      IsolateData::BuiltinSlotOffset(Builtin::kWasmCEntry);
  OpIndex centry_stub =
      __ Load(isolate_root, LoadOp::Kind::RawAligned(),
              MemoryRepresentation::UintPtr(), builtin_slot_offset);
  // CallRuntime is always called with 0 or 1 argument, so a vector of size 4
  // always suffices.
  SmallZoneVector<OpIndex, 4> centry_args(zone);
  for (OpIndex arg : args) centry_args.emplace_back(arg);
  centry_args.emplace_back(__ ExternalConstant(ExternalReference::Create(f)));
  centry_args.emplace_back(__ Word32Constant(fun->nargs));
  centry_args.emplace_back(context);
  const CallDescriptor* call_descriptor =
      compiler::Linkage::GetRuntimeCallDescriptor(
          __ graph_zone(), f, fun->nargs, Operator::kNoProperties,
          CallDescriptor::kNoFlags);
  const TSCallDescriptor* ts_call_descriptor = TSCallDescriptor::Create(
      call_descriptor, compiler::CanThrow::kYes,
      compiler::LazyDeoptOnThrow::kNo, __ graph_zone());
  return __ Call(centry_stub, OpIndex::Invalid(), base::VectorOf(centry_args),
                 ts_call_descriptor);
}
 
OpIndex WasmGraphBuilderBase::GetBuiltinPointerTarget(Builtin builtin) {
  static_assert(std::is_same<Smi, BuiltinPtr>(), "BuiltinPtr must be Smi");
  return __ SmiConstant(Smi::FromInt(static_cast<int>(builtin)));
}
 
V<WordPtr> WasmGraphBuilderBase::GetTargetForBuiltinCall(
    Builtin builtin, StubCallMode stub_mode) {
  return stub_mode == StubCallMode::kCallWasmRuntimeStub
             ? __ RelocatableWasmBuiltinCallTarget(builtin)
             : GetBuiltinPointerTarget(builtin);
}
 
V<BigInt> WasmGraphBuilderBase::BuildChangeInt64ToBigInt(
    V<Word64> input, StubCallMode stub_mode) {
  Builtin builtin = Is64() ? Builtin::kI64ToBigInt : Builtin::kI32PairToBigInt;
  V<WordPtr> target = GetTargetForBuiltinCall(builtin, stub_mode);
  CallInterfaceDescriptor interface_descriptor =
      Builtins::CallInterfaceDescriptorFor(builtin);
  const CallDescriptor* call_descriptor =
      compiler::Linkage::GetStubCallDescriptor(
          __ graph_zone(),  // zone
          interface_descriptor,
          0,                         // stack parameter count
          CallDescriptor::kNoFlags,  // flags
          Operator::kNoProperties,   // properties
          stub_mode);
  const TSCallDescriptor* ts_call_descriptor = TSCallDescriptor::Create(
      call_descriptor, compiler::CanThrow::kNo, compiler::LazyDeoptOnThrow::kNo,
      __ graph_zone());
  if constexpr (Is64()) {
    return V<BigInt>::Cast(__ Call(target, {input}, ts_call_descriptor));
  }
  V<Word32> low_word = __ TruncateWord64ToWord32(input);
  V<Word32> high_word = __ TruncateWord64ToWord32(__ ShiftRightLogical(
      input, __ Word32Constant(32), WordRepresentation::Word64()));
  return V<BigInt>::Cast(
      __ Call(target, {low_word, high_word}, ts_call_descriptor));
}
 
std::pair<V<Word32>, V<HeapObject>>
WasmGraphBuilderBase::BuildImportedFunctionTargetAndImplicitArg(
    ConstOrV<Word32> func_index,
    V<WasmTrustedInstanceData> trusted_instance_data) {
  V<WasmDispatchTable> dispatch_table = LOAD_IMMUTABLE_PROTECTED_INSTANCE_FIELD(
      trusted_instance_data, DispatchTableForImports, WasmDispatchTable);
  // Handle constant indexes specially to reduce graph size, even though later
  // optimization would optimize this to the same result.
  if (func_index.is_constant()) {
    int offset = WasmDispatchTable::OffsetOf(func_index.constant_value());
    V<Word32> target = __ Load(dispatch_table, LoadOp::Kind::TaggedBase(),
                               MemoryRepresentation::Uint32(),
                               offset + WasmDispatchTable::kTargetBias);
    V<ExposedTrustedObject> implicit_arg =
        V<ExposedTrustedObject>::Cast(__ LoadProtectedPointerField(
            dispatch_table, LoadOp::Kind::TaggedBase(),
            offset + WasmDispatchTable::kImplicitArgBias));
    return {target, implicit_arg};
  }
 
  V<WordPtr> dispatch_table_entry_offset =
      __ WordPtrMul(__ ChangeUint32ToUintPtr(func_index.value()),
                    WasmDispatchTable::kEntrySize);
  V<Word32> target = __ Load(
      dispatch_table, dispatch_table_entry_offset, LoadOp::Kind::TaggedBase(),
      MemoryRepresentation::Uint32(),
      WasmDispatchTable::kEntriesOffset + WasmDispatchTable::kTargetBias);
  V<ExposedTrustedObject> implicit_arg = V<ExposedTrustedObject>::Cast(
      __ LoadProtectedPointerField(dispatch_table, dispatch_table_entry_offset,
                                   LoadOp::Kind::TaggedBase(),
                                   WasmDispatchTable::kEntriesOffset +
                                       WasmDispatchTable::kImplicitArgBias,
                                   0));
  return {target, implicit_arg};
}
 
std::pair<V<Word32>, V<ExposedTrustedObject>>
WasmGraphBuilderBase::BuildFunctionTargetAndImplicitArg(
    V<WasmInternalFunction> internal_function) {
  V<ExposedTrustedObject> implicit_arg =
      V<ExposedTrustedObject>::Cast(__ LoadProtectedPointerField(
          internal_function, LoadOp::Kind::TaggedBase().Immutable(),
          WasmInternalFunction::kProtectedImplicitArgOffset));
 
  V<Word32> target = __ Load(internal_function, LoadOp::Kind::TaggedBase(),
                             MemoryRepresentation::Uint32(),
                             WasmInternalFunction::kRawCallTargetOffset);
 
  return {target, implicit_arg};
}
 
RegisterRepresentation WasmGraphBuilderBase::RepresentationFor(
    ValueTypeBase type) {
  switch (type.kind()) {
    case kI8:
    case kI16:
    case kI32:
      return RegisterRepresentation::Word32();
    case kI64:
      return RegisterRepresentation::Word64();
    case kF16:
    case kF32:
      return RegisterRepresentation::Float32();
    case kF64:
      return RegisterRepresentation::Float64();
    case kRefNull:
    case kRef:
      return RegisterRepresentation::Tagged();
    case kS128:
      return RegisterRepresentation::Simd128();
    case kVoid:
    case kTop:
    case kBottom:
      UNREACHABLE();
  }
}
 
// Load the trusted data from a WasmInstanceObject.
V<WasmTrustedInstanceData>
WasmGraphBuilderBase::LoadTrustedDataFromInstanceObject(
    V<HeapObject> instance_object) {
  return V<WasmTrustedInstanceData>::Cast(__ LoadTrustedPointerField(
      instance_object, LoadOp::Kind::TaggedBase().Immutable(),
      kWasmTrustedInstanceDataIndirectPointerTag,
      WasmInstanceObject::kTrustedDataOffset));
}
 
void WasmGraphBuilderBase::BuildModifyThreadInWasmFlagHelper(
    Zone* zone, OpIndex thread_in_wasm_flag_address, bool new_value) {
  if (v8_flags.debug_code) {
    V<Word32> flag_value =
        __ Load(thread_in_wasm_flag_address, LoadOp::Kind::RawAligned(),
                MemoryRepresentation::Int32(), 0);
 
    IF (UNLIKELY(__ Word32Equal(flag_value, new_value))) {
      OpIndex message_id = __ TaggedIndexConstant(static_cast<int32_t>(
          new_value ? AbortReason::kUnexpectedThreadInWasmSet
                    : AbortReason::kUnexpectedThreadInWasmUnset));
      CallRuntime(zone, Runtime::kAbort, {message_id}, __ NoContextConstant());
      __ Unreachable();
    }
  }
 
  __ Store(thread_in_wasm_flag_address, __ Word32Constant(new_value),
           LoadOp::Kind::RawAligned(), MemoryRepresentation::Int32(),
           compiler::kNoWriteBarrier);
}
 
void WasmGraphBuilderBase::BuildModifyThreadInWasmFlag(Zone* zone,
                                                       bool new_value) {
  if (!trap_handler::IsTrapHandlerEnabled()) return;
 
  OpIndex isolate_root = __ LoadRootRegister();
  OpIndex thread_in_wasm_flag_address =
      __ Load(isolate_root, LoadOp::Kind::RawAligned().Immutable(),
              MemoryRepresentation::UintPtr(),
              Isolate::thread_in_wasm_flag_address_offset());
  BuildModifyThreadInWasmFlagHelper(zone, thread_in_wasm_flag_address,
                                    new_value);
}
 
// TODO(14108): Annotate C functions as not having side effects where
// appropriate.
OpIndex WasmGraphBuilderBase::CallC(const MachineSignature* sig,
                                    ExternalReference ref,
                                    std::initializer_list<OpIndex> args) {
  return WasmGraphBuilderBase::CallC(sig, __ ExternalConstant(ref), args);
}
 
OpIndex WasmGraphBuilderBase::CallC(const MachineSignature* sig,
                                    OpIndex function,
                                    std::initializer_list<OpIndex> args) {
  DCHECK_LE(sig->return_count(), 1);
  DCHECK_EQ(sig->parameter_count(), args.size());
  const CallDescriptor* call_descriptor =
      compiler::Linkage::GetSimplifiedCDescriptor(__ graph_zone(), sig);
  const TSCallDescriptor* ts_call_descriptor = TSCallDescriptor::Create(
      call_descriptor, compiler::CanThrow::kNo, compiler::LazyDeoptOnThrow::kNo,
      __ graph_zone());
  return __ Call(function, OpIndex::Invalid(), base::VectorOf(args),
                 ts_call_descriptor);
}
 
void WasmGraphBuilderBase::BuildSetNewStackLimit(V<WordPtr> old_limit,
                                                 V<WordPtr> new_limit) {
  // Set the new interrupt limit and real limit. Use a compare-and-swap for
  // the interrupt limit to avoid overwriting a pending interrupt.
  __ AtomicCompareExchange(
      __ IsolateField(IsolateFieldId::kJsLimitAddress), __ UintPtrConstant(0),
      old_limit, new_limit, RegisterRepresentation::WordPtr(),
      MemoryRepresentation::UintPtr(), compiler::MemoryAccessKind::kNormal);
  __ Store(__ LoadRootRegister(), new_limit, StoreOp::Kind::RawAligned(),
           MemoryRepresentation::UintPtr(), compiler::kNoWriteBarrier,
           IsolateData::real_jslimit_offset());
}
 
V<WordPtr> WasmGraphBuilderBase::BuildSwitchToTheCentralStack(
    V<WordPtr> old_limit) {
  // Set the is_on_central_stack flag.
  OpIndex isolate_root = __ LoadRootRegister();
  __ Store(isolate_root, __ Word32Constant(1), LoadOp::Kind::RawAligned(),
           MemoryRepresentation::Uint8(), compiler::kNoWriteBarrier,
           IsolateData::is_on_central_stack_flag_offset());
 
  // Save the old fp and the target sp in the StackMemory's stack switch info.
  // We are not on the main stack, so the active stack must be set.
  V<WordPtr> active_stack = __ Load(isolate_root, LoadOp::Kind::RawAligned(),
                                    MemoryRepresentation::UintPtr(),
                                    IsolateData::active_stack_offset());
  __ Store(active_stack, __ FramePointer(), StoreOp::Kind::RawAligned(),
           MemoryRepresentation::UintPtr(), compiler::kNoWriteBarrier,
           StackMemory::stack_switch_source_fp_offset());
  V<WordPtr> central_stack_sp = __ Load(
      isolate_root, LoadOp::Kind::RawAligned(), MemoryRepresentation::UintPtr(),
      Isolate::central_stack_sp_offset());
  __ Store(active_stack, central_stack_sp, StoreOp::Kind::RawAligned(),
           MemoryRepresentation::UintPtr(), compiler::kNoWriteBarrier,
           StackMemory::stack_switch_target_sp_offset());
 
  // Switch the stack limit and the stack pointer.
  V<WordPtr> central_stack_limit = __ Load(
      isolate_root, LoadOp::Kind::RawAligned(), MemoryRepresentation::UintPtr(),
      Isolate::central_stack_limit_offset());
  BuildSetNewStackLimit(old_limit, central_stack_limit);
  OpIndex old_sp = __ LoadStackPointer();
  __ SetStackPointer(central_stack_sp);
  return old_sp;
}
 
// Returns the old (secondary stack's) sp and stack limit.
std::pair<V<WordPtr>, V<WordPtr>>
WasmGraphBuilderBase::BuildSwitchToTheCentralStackIfNeeded() {
  V<WordPtr> isolate_root = __ LoadRootRegister();
  V<Word32> is_on_central_stack_flag = __ Load(
      isolate_root, LoadOp::Kind::RawAligned(), MemoryRepresentation::Uint8(),
      IsolateData::is_on_central_stack_flag_offset());
  ScopedVar<WordPtr> old_sp_var(this, __ IntPtrConstant(0));
  ScopedVar<WordPtr> old_limit_var(this, __ IntPtrConstant(0));
  IF_NOT (LIKELY(is_on_central_stack_flag)) {
    V<WordPtr> old_limit = __ Load(isolate_root, LoadOp::Kind::RawAligned(),
                                   MemoryRepresentation::UintPtr(),
                                   IsolateData::real_jslimit_offset());
    V<WordPtr> old_sp = BuildSwitchToTheCentralStack(old_limit);
    old_sp_var = old_sp;
    old_limit_var = old_limit;
  }
  return {old_sp_var, old_limit_var};
}
 
void WasmGraphBuilderBase::BuildSwitchBackFromCentralStack(
    V<WordPtr> old_sp, V<WordPtr> old_limit) {
  IF_NOT (LIKELY(__ WordPtrEqual(old_sp, __ IntPtrConstant(0)))) {
    // Reset is_on_central_stack flag.
    V<WordPtr> isolate_root = __ LoadRootRegister();
    __ Store(isolate_root, __ Word32Constant(0), StoreOp::Kind::RawAligned(),
             MemoryRepresentation::Uint8(), compiler::kNoWriteBarrier,
             IsolateData::is_on_central_stack_flag_offset());
 
    // Clear stack switch info.
    // We are not on the main stack, so the active stack must be set.
    V<WordPtr> active_stack = __ Load(isolate_root, LoadOp::Kind::RawAligned(),
                                      MemoryRepresentation::UintPtr(),
                                      IsolateData::active_stack_offset());
    __ Store(active_stack, __ UintPtrConstant(0), StoreOp::Kind::RawAligned(),
             MemoryRepresentation::UintPtr(), compiler::kNoWriteBarrier,
             StackMemory::stack_switch_source_fp_offset());
 
    // Restore the old stack limit and stack pointer.
    V<WordPtr> real_jslimit = __ Load(isolate_root, LoadOp::Kind::RawAligned(),
                                      MemoryRepresentation::UintPtr(),
                                      IsolateData::real_jslimit_offset());
    BuildSetNewStackLimit(real_jslimit, old_limit);
    __ SetStackPointer(old_sp);
  }
}
 
class TurboshaftGraphBuildingInterface : public WasmGraphBuilderBase {
 private:
  class BlockPhis;
  class InstanceCache;
 
 public:
  enum Mode {
    kRegular,
    kInlinedUnhandled,
    kInlinedWithCatch,
    kInlinedTailCall
  };
  using ValidationTag = Decoder::NoValidationTag;
  using FullDecoder =
      WasmFullDecoder<ValidationTag, TurboshaftGraphBuildingInterface>;
  static constexpr bool kUsesPoppedArgs = true;
 
  struct Value : public ValueBase<ValidationTag> {
    OpIndex op = OpIndex::Invalid();
    template <typename... Args>
    explicit Value(Args&&... args) V8_NOEXCEPT
        : ValueBase(std::forward<Args>(args)...) {}
  };
 
  struct Control : public ControlBase<Value, ValidationTag> {
    TSBlock* merge_block = nullptr;
    // for 'if', loops, and 'try'/'try-table' respectively.
    TSBlock* false_or_loop_or_catch_block = nullptr;
    BitVector* assigned = nullptr;             // Only for loops.
    V<Object> exception = OpIndex::Invalid();  // Only for 'try-catch'.
 
    template <typename... Args>
    explicit Control(Args&&... args) V8_NOEXCEPT
        : ControlBase(std::forward<Args>(args)...) {}
  };
 
 public:
  // For non-inlined functions.
  TurboshaftGraphBuildingInterface(
      Zone* zone, CompilationEnv* env, Assembler& assembler,
      std::unique_ptr<AssumptionsJournal>* assumptions,
      ZoneVector<WasmInliningPosition>* inlining_positions, int func_index,
      bool shared, const WireBytesStorage* wire_bytes)
      : WasmGraphBuilderBase(zone, assembler),
        mode_(kRegular),
        block_phis_(zone),
        env_(env),
        owned_instance_cache_(std::make_unique<InstanceCache>(assembler)),
        instance_cache_(*owned_instance_cache_.get()),
        assumptions_(assumptions),
        inlining_positions_(inlining_positions),
        ssa_env_(zone),
        func_index_(func_index),
        shared_(shared),
        wire_bytes_(wire_bytes),
        return_phis_(nullptr),
        is_inlined_tail_call_(false) {
    DCHECK_NOT_NULL(env_);
    DCHECK_NOT_NULL(env_->module);
  }
 
  // For inlined functions.
  TurboshaftGraphBuildingInterface(
      Zone* zone, CompilationEnv* env, Assembler& assembler, Mode mode,
      InstanceCache& instance_cache,
      std::unique_ptr<AssumptionsJournal>* assumptions,
      ZoneVector<WasmInliningPosition>* inlining_positions, int func_index,
      bool shared, const WireBytesStorage* wire_bytes,
      base::Vector<OpIndex> real_parameters, TSBlock* return_block,
      BlockPhis* return_phis, TSBlock* catch_block, bool is_inlined_tail_call,
      OptionalV<FrameState> parent_frame_state)
      : WasmGraphBuilderBase(zone, assembler),
        mode_(mode),
        block_phis_(zone),
        env_(env),
        instance_cache_(instance_cache),
        assumptions_(assumptions),
        inlining_positions_(inlining_positions),
        ssa_env_(zone),
        func_index_(func_index),
        shared_(shared),
        wire_bytes_(wire_bytes),
        real_parameters_(real_parameters),
        return_block_(return_block),
        return_phis_(return_phis),
        return_catch_block_(catch_block),
        is_inlined_tail_call_(is_inlined_tail_call),
        parent_frame_state_(parent_frame_state) {
    DCHECK_NE(mode_, kRegular);
    DCHECK_EQ(return_block == nullptr, mode == kInlinedTailCall);
    DCHECK_EQ(catch_block != nullptr, mode == kInlinedWithCatch);
  }
 
  void StartFunction(FullDecoder* decoder) {
    if (mode_ == kRegular) __ Bind(__ NewBlock());
    // Set 0 as the current source position (before locals declarations).
    __ SetCurrentOrigin(WasmPositionToOpIndex(0, inlining_id_));
    ssa_env_.resize(decoder->num_locals());
    uint32_t index = 0;
    V<WasmTrustedInstanceData> trusted_instance_data;
    if (mode_ == kRegular) {
      static_assert(kWasmInstanceDataParameterIndex == 0);
      trusted_instance_data = __ WasmInstanceDataParameter();
      for (; index < decoder->sig_->parameter_count(); index++) {
        // Parameter indices are shifted by 1 because parameter 0 is the
        // instance.
        ssa_env_[index] = __ Parameter(
            index + 1, RepresentationFor(decoder->sig_->GetParam(index)));
      }
      instance_cache_.Initialize(trusted_instance_data, decoder->module_);
    } else {
      trusted_instance_data = real_parameters_[0];
      for (; index < decoder->sig_->parameter_count(); index++) {
        // Parameter indices are shifted by 1 because parameter 0 is the
        // instance.
        ssa_env_[index] = real_parameters_[index + 1];
      }
      if (!is_inlined_tail_call_) {
        return_phis_->InitReturnPhis(decoder->sig_->returns());
      }
    }
    while (index < decoder->num_locals()) {
      ValueType type = decoder->local_type(index);
      OpIndex op;
      if (!type.is_defaultable()) {
        DCHECK(type.is_reference());
        op = __ RootConstant(RootIndex::kOptimizedOut);
      } else {
        op = DefaultValue(type);
      }
      while (index < decoder->num_locals() &&
             decoder->local_type(index) == type) {
        ssa_env_[index++] = op;
      }
    }
 
    if (v8_flags.wasm_inlining) {
      if (mode_ == kRegular) {
        if (v8_flags.liftoff) {
          inlining_decisions_ = InliningTree::CreateRoot(
              decoder->zone_, decoder->module_, func_index_);
        } else {
          set_no_liftoff_inlining_budget(
              InliningTree::NoLiftoffBudget(decoder->module_, func_index_));
        }
      } else {
#if DEBUG
        // We don't have support for inlining asm.js functions, those should
        // never be selected in `InliningTree`.
        DCHECK(!wasm::is_asmjs_module(decoder->module_));
 
        if (v8_flags.liftoff && inlining_decisions_) {
          // DCHECK that `inlining_decisions_` is consistent.
          DCHECK(inlining_decisions_->is_inlined());
          DCHECK_EQ(inlining_decisions_->function_index(), func_index_);
          base::MutexGuard mutex_guard(&decoder->module_->type_feedback.mutex);
          if (inlining_decisions_->feedback_found()) {
            DCHECK_NE(
                decoder->module_->type_feedback.feedback_for_function.find(
                    func_index_),
                decoder->module_->type_feedback.feedback_for_function.end());
            DCHECK_EQ(inlining_decisions_->function_calls().size(),
                      decoder->module_->type_feedback.feedback_for_function
                          .find(func_index_)
                          ->second.feedback_vector.size());
            DCHECK_EQ(inlining_decisions_->function_calls().size(),
                      decoder->module_->type_feedback.feedback_for_function
                          .find(func_index_)
                          ->second.call_targets.size());
          }
        }
#endif
      }
    }
 
    if (v8_flags.debug_code) {
      IF_NOT (LIKELY(__ HasInstanceType(trusted_instance_data,
                                        WASM_TRUSTED_INSTANCE_DATA_TYPE))) {
        OpIndex message_id = __ TaggedIndexConstant(
            static_cast<int32_t>(AbortReason::kUnexpectedInstanceType));
        CallRuntime(decoder->zone(), Runtime::kAbort, {message_id},
                    __ NoContextConstant());
        __ Unreachable();
      }
    }
 
    if (mode_ == kRegular) {
      StackCheck(WasmStackCheckOp::Kind::kFunctionEntry, decoder);
    }
 
    if (v8_flags.trace_wasm) {
      __ SetCurrentOrigin(
          WasmPositionToOpIndex(decoder->position(), inlining_id_));
      CallRuntime(decoder->zone(), Runtime::kWasmTraceEnter, {},
                  __ NoContextConstant());
    }
 
    auto branch_hints_it = decoder->module_->branch_hints.find(func_index_);
    if (v8_flags.stress_branch_hinting) {
      // Stress mode takes precedence over a branch hints section.
      branch_hinting_mode_ = BranchHintingMode::kStress;
    } else if (branch_hints_it != decoder->module_->branch_hints.end()) {
      branch_hints_ = &branch_hints_it->second;
      branch_hinting_mode_ = BranchHintingMode::kModuleProvided;
    } else {
      branch_hinting_mode_ = BranchHintingMode::kNone;
    }
  }
 
  void StartFunctionBody(FullDecoder* decoder, Control* block) {}
 
  void FinishFunction(FullDecoder* decoder) {
    if (v8_flags.liftoff && inlining_decisions_ &&
        inlining_decisions_->feedback_found()) {
      DCHECK_EQ(
          feedback_slot_,
          static_cast<int>(inlining_decisions_->function_calls().size()) - 1);
    }
    if (mode_ == kRegular) {
      // Just accessing `source_positions` at the maximum `OpIndex` already
      // pre-allocates the underlying storage such that we avoid repeatedly
      // resizing/copying in the following loop.
      __ output_graph().source_positions()[__ output_graph().EndIndex()];
 
      for (OpIndex index : __ output_graph().AllOperationIndices()) {
        SourcePosition position = OpIndexToSourcePosition(
            __ output_graph().operation_origins()[index]);
        __ output_graph().source_positions()[index] = position;
      }
      if (v8_flags.trace_wasm_inlining) {
        uint32_t node_count =
            __ output_graph().NumberOfOperationsForDebugging();
        PrintF("[function %d: emitted %d nodes]\n", func_index_, node_count);
      }
    }
  }
 
  void OnFirstError(FullDecoder*) {}
 
  void NextInstruction(FullDecoder* decoder, WasmOpcode) {
    __ SetCurrentOrigin(
        WasmPositionToOpIndex(decoder->position(), inlining_id_));
  }
 
  // ******** Control Flow ********
  // The basic structure of control flow is {block_phis_}. It contains a mapping
  // from blocks to phi inputs corresponding to the SSA values plus the stack
  // merge values at the beginning of the block.
  // - When we create a new block (to be bound in the future), we register it to
  //   {block_phis_} with {NewBlockWithPhis}.
  // - When we encounter an jump to a block, we invoke {SetupControlFlowEdge}.
  // - Finally, when we bind a block, we setup its phis, the SSA environment,
  //   and its merge values, with {BindBlockAndGeneratePhis}.
  // - When we create a loop, we generate PendingLoopPhis for the SSA state and
  //   the incoming stack values. We also create a block which will act as a
  //   merge block for all loop backedges (since a loop in Turboshaft can only
  //   have one backedge). When we PopControl a loop, we enter the merge block
  //   to create its Phis for all backedges as necessary, and use those values
  //   to patch the backedge of the PendingLoopPhis of the loop.
 
  void Block(FullDecoder* decoder, Control* block) {
    block->merge_block = NewBlockWithPhis(decoder, block->br_merge());
  }
 
  void Loop(FullDecoder* decoder, Control* block) {
    TSBlock* loop = __ NewLoopHeader();
    __ Goto(loop);
    __ Bind(loop);
 
    bool can_be_innermost = false;  // unused
    BitVector* assigned = WasmDecoder<ValidationTag>::AnalyzeLoopAssignment(
        decoder, decoder->pc(), decoder->num_locals(), decoder->zone(),
        &can_be_innermost);
    block->assigned = assigned;
 
    for (uint32_t i = 0; i < decoder->num_locals(); i++) {
      if (!assigned->Contains(i)) continue;
      OpIndex phi = __ PendingLoopPhi(
          ssa_env_[i], RepresentationFor(decoder->local_type(i)));
      ssa_env_[i] = phi;
    }
    uint32_t arity = block->start_merge.arity;
    Value* stack_base = arity > 0 ? decoder->stack_value(arity) : nullptr;
    for (uint32_t i = 0; i < arity; i++) {
      OpIndex phi = __ PendingLoopPhi(stack_base[i].op,
                                      RepresentationFor(stack_base[i].type));
      block->start_merge[i].op = phi;
    }
 
    StackCheck(WasmStackCheckOp::Kind::kLoop, decoder);
 
    TSBlock* loop_merge = NewBlockWithPhis(decoder, &block->start_merge);
    block->merge_block = loop_merge;
    block->false_or_loop_or_catch_block = loop;
  }
 
  void If(FullDecoder* decoder, const Value& cond, Control* if_block) {
    TSBlock* true_block = __ NewBlock();
    TSBlock* false_block = NewBlockWithPhis(decoder, nullptr);
    TSBlock* merge_block = NewBlockWithPhis(decoder, &if_block->end_merge);
    if_block->false_or_loop_or_catch_block = false_block;
    if_block->merge_block = merge_block;
    SetupControlFlowEdge(decoder, false_block);
    __ Branch({cond.op, GetBranchHint(decoder)}, true_block, false_block);
    __ Bind(true_block);
  }
 
  void Else(FullDecoder* decoder, Control* if_block) {
    if (if_block->reachable()) {
      SetupControlFlowEdge(decoder, if_block->merge_block);
      __ Goto(if_block->merge_block);
    }
    BindBlockAndGeneratePhis(decoder, if_block->false_or_loop_or_catch_block,
                             nullptr);
  }
 
  void BrOrRet(FullDecoder* decoder, uint32_t depth, uint32_t drop_values = 0) {
    if (depth == decoder->control_depth() - 1) {
      DoReturn(decoder, drop_values);
    } else {
      Control* target = decoder->control_at(depth);
      SetupControlFlowEdge(decoder, target->merge_block, drop_values);
      __ Goto(target->merge_block);
    }
  }
 
  void BrIf(FullDecoder* decoder, const Value& cond, uint32_t depth) {
    BranchHint hint = GetBranchHint(decoder);
    if (depth == decoder->control_depth() - 1) {
      IF ({cond.op, hint}) {
        DoReturn(decoder, 0);
      }
    } else {
      Control* target = decoder->control_at(depth);
      SetupControlFlowEdge(decoder, target->merge_block);
      TSBlock* non_branching = __ NewBlock();
      __ Branch({cond.op, hint}, target->merge_block, non_branching);
      __ Bind(non_branching);
    }
  }
 
  // An analysis to determine whether a br_table should be lowered to a switch
  // or a series of compare and branch. This can be for small tables or larger
  // 'sparse' ones, which include many cases but few targets. A sparse table may
  // look like this: br_table [ 1, 0, 0, 0, 0, 0, 2, 0 ] which can be lowered to
  // two conditional branches followed by an unconditional one. The advantages
  // of this are reducing the space required for the table and reducing the
  // latency.
  template <typename ValidationTag>
  class BrTableAnalysis {
   public:
    static constexpr int32_t kMaxComparesPerTarget = 2;
    static constexpr uint32_t kMaxTargets = 3;
    static constexpr int32_t kMaxTableCount = 20;
    using CaseVector = base::SmallVector<uint8_t, 8>;
    using TargetMap = absl::btree_map<uint32_t, CaseVector>;
 
    bool LowerToBranches(Decoder* decoder, const BranchTableImmediate& imm) {
      BranchTableIterator<ValidationTag> iterator(decoder, imm);
      while (iterator.has_next()) {
        uint32_t i = iterator.cur_index();
        uint32_t target = iterator.next();
 
        if (i == imm.table_count) {
          AddDefault(target);
        } else if (!TryAddTarget(target, i)) {
          return false;
        }
      }
      primary_indices_ = other_targets_[primary_target()];
      other_targets_.erase(primary_target());
      size_t total_targets = other_targets_.size() + 1;
      if (default_target() != primary_target() &&
          !other_targets_.count(default_target())) {
        total_targets++;
      }
      return total_targets <= kMaxTargets;
    }
    // The most often occurring target, or the default if there is no other
    // target with multiple cases.
    uint32_t primary_target() const { return primary_target_.value(); }
    // The default target, for when the br_table index is out-of-range.
    uint32_t default_target() const { return default_target_.value(); }
    // other_targets doesn't include the primary target, nor the default if it
    // isn't an in-range target.
    const TargetMap& other_targets() const { return other_targets_; }
    // All the indices which target the primary target.
    const CaseVector& primary_indices() const { return primary_indices_; }
 
   private:
    bool TryAddTarget(uint32_t target, uint32_t index) {
      DCHECK_LT(index, kMaxTableCount);
      CaseVector& cases = other_targets_[target];
      if (other_targets_.size() > kMaxTargets) {
        return false;
      }
      if (cases.size() == kMaxComparesPerTarget) {
        if (primary_target_.has_value() && target != primary_target()) {
          return false;
        }
        primary_target_ = target;
      }
      cases.push_back(index);
      return true;
    }
    void AddDefault(uint32_t target) {
      default_target_ = target;
      if (!primary_target_.has_value()) {
        primary_target_ = default_target();
      }
    }
 
    std::optional<uint32_t> default_target_;
    std::optional<uint32_t> primary_target_;
    CaseVector primary_indices_;
    TargetMap other_targets_;
  };
 
  void BrTable(FullDecoder* decoder, const BranchTableImmediate& imm,
               const Value& key) {
    if (imm.table_count < BrTableAnalysis<ValidationTag>::kMaxTableCount) {
      BrTableAnalysis<ValidationTag> table_analysis;
      if (table_analysis.LowerToBranches(decoder, imm)) {
        auto generate_cond =
            [this](const Value& key,
                   const BrTableAnalysis<ValidationTag>::CaseVector& cases)
            -> OpIndex {
          switch (cases.size()) {
            default:
              static_assert(
                  BrTableAnalysis<ValidationTag>::kMaxComparesPerTarget <= 2);
              UNREACHABLE();
            case 1:
              return __ Word32Equal(key.op, __ Word32Constant(cases[0]));
            case 2: {
              return __ Word32BitwiseOr(__ Word32Equal(key.op, cases[0]),
                                        __ Word32Equal(key.op, cases[1]));
            }
          }
        };
        auto insert_cond_branch = [this, &decoder](OpIndex cond,
                                                   uint32_t depth) {
          BranchHint hint = GetBranchHint(decoder);
          if (depth == decoder->control_depth() - 1) {
            IF ({cond, hint}) {
              DoReturn(decoder, 0);
            }
          } else {
            Control* target = decoder->control_at(depth);
            SetupControlFlowEdge(decoder, target->merge_block);
            TSBlock* non_branching = __ NewBlock();
            __ Branch({cond, hint}, target->merge_block, non_branching);
            __ Bind(non_branching);
          }
        };
        // Insert conditional branches to the other targets.
        for (auto const& [target, cases] : table_analysis.other_targets()) {
          DCHECK_LE(cases.size(),
                    BrTableAnalysis<ValidationTag>::kMaxComparesPerTarget);
          insert_cond_branch(generate_cond(key, cases), target);
        }
        // If needed, insert the range check for the primary target.
        if (table_analysis.primary_target() !=
            table_analysis.default_target()) {
          OpIndex lower = __ Word32Equal(__ Int32LessThan(key.op, 0), 0);
          OpIndex upper =
              __ Int32LessThan(key.op, __ Word32Constant(imm.table_count));
          OpIndex cond = __ Word32BitwiseAnd(lower, upper);
          insert_cond_branch(cond, table_analysis.primary_target());
        }
        // Always fallthrough and branch to the default case.
        BrOrRet(decoder, table_analysis.default_target());
        return;
      }
    }
    compiler::turboshaft::SwitchOp::Case* cases =
        __ output_graph().graph_zone()
            -> AllocateArray<compiler::turboshaft::SwitchOp::Case>(
                             imm.table_count);
    BranchTableIterator<ValidationTag> new_block_iterator(decoder, imm);
    SmallZoneVector<TSBlock*, 16> intermediate_blocks(decoder->zone_);
    TSBlock* default_case = nullptr;
    while (new_block_iterator.has_next()) {
      TSBlock* intermediate = __ NewBlock();
      intermediate_blocks.emplace_back(intermediate);
      uint32_t i = new_block_iterator.cur_index();
      if (i == imm.table_count) {
        default_case = intermediate;
      } else {
        cases[i] = {static_cast<int>(i), intermediate, BranchHint::kNone};
      }
      new_block_iterator.next();
    }
    DCHECK_NOT_NULL(default_case);
    __ Switch(key.op, base::VectorOf(cases, imm.table_count), default_case);
 
    int i = 0;
    BranchTableIterator<ValidationTag> branch_iterator(decoder, imm);
    while (branch_iterator.has_next()) {
      TSBlock* intermediate = intermediate_blocks[i];
      i++;
      __ Bind(intermediate);
      BrOrRet(decoder, branch_iterator.next());
    }
  }
 
  void FallThruTo(FullDecoder* decoder, Control* block) {
    // TODO(14108): Why is {block->reachable()} not reliable here? Maybe it is
    // not in other spots as well.
    if (__ current_block() != nullptr) {
      SetupControlFlowEdge(decoder, block->merge_block);
      __ Goto(block->merge_block);
    }
  }
 
  void PopControl(FullDecoder* decoder, Control* block) {
    switch (block->kind) {
      case kControlIf:
        if (block->reachable()) {
          SetupControlFlowEdge(decoder, block->merge_block);
          __ Goto(block->merge_block);
        }
        BindBlockAndGeneratePhis(decoder, block->false_or_loop_or_catch_block,
                                 nullptr);
        // Exceptionally for one-armed if, we cannot take the values from the
        // stack; we have to pass the stack values at the beginning of the
        // if-block.
        SetupControlFlowEdge(decoder, block->merge_block, 0, OpIndex::Invalid(),
                             &block->start_merge);
        __ Goto(block->merge_block);
        BindBlockAndGeneratePhis(decoder, block->merge_block,
                                 block->br_merge());
        break;
      case kControlIfElse:
      case kControlBlock:
      case kControlTry:
      case kControlTryCatch:
      case kControlTryCatchAll:
        // {block->reachable()} is not reliable here for exceptions, because
        // the decoder sets the reachability to the upper block's reachability
        // before calling this interface function.
        if (__ current_block() != nullptr) {
          SetupControlFlowEdge(decoder, block->merge_block);
          __ Goto(block->merge_block);
        }
        BindBlockAndGeneratePhis(decoder, block->merge_block,
                                 block->br_merge());
        break;
      case kControlTryTable:
        DCHECK_EQ(__ current_block(), nullptr);
        BindBlockAndGeneratePhis(decoder, block->merge_block,
                                 block->br_merge());
        break;
      case kControlLoop: {
        TSBlock* post_loop = NewBlockWithPhis(decoder, nullptr);
        if (block->reachable()) {
          SetupControlFlowEdge(decoder, post_loop);
          __ Goto(post_loop);
        }
        if (!block->false_or_loop_or_catch_block->IsBound()) {
          // The loop is unreachable. In this case, no operations have been
          // emitted for it. Do nothing.
        } else if (block->merge_block->PredecessorCount() == 0) {
          // Turns out, the loop has no backedges, i.e. it is not quite a loop
          // at all. Replace it with a merge, and its PendingPhis with one-input
          // phis.
          block->false_or_loop_or_catch_block->SetKind(
              compiler::turboshaft::Block::Kind::kMerge);
          for (auto& op : __ output_graph().operations(
                   *block->false_or_loop_or_catch_block)) {
            PendingLoopPhiOp* pending_phi = op.TryCast<PendingLoopPhiOp>();
            if (!pending_phi) break;
            OpIndex replaced = __ output_graph().Index(op);
            __ output_graph().Replace<compiler::turboshaft::PhiOp>(
                replaced, base::VectorOf({pending_phi -> first()}),
                pending_phi->rep);
          }
        } else {
          // We abuse the start merge of the loop, which is not used otherwise
          // anymore, to store backedge inputs for the pending phi stack values
          // of the loop.
          BindBlockAndGeneratePhis(decoder, block->merge_block,
                                   block->br_merge());
          __ Goto(block->false_or_loop_or_catch_block);
          auto operations = __ output_graph().operations(
              *block -> false_or_loop_or_catch_block);
          auto to = operations.begin();
          // The VariableReducer can introduce loop phis as well which are at
          // the beginning of the block. We need to skip them.
          while (to != operations.end() &&
                 to->Is<compiler::turboshaft::PhiOp>()) {
            ++to;
          }
          for (auto it = block->assigned->begin(); it != block->assigned->end();
               ++it, ++to) {
            // The last bit represents the instance cache.
            if (*it == static_cast<int>(ssa_env_.size())) break;
            PendingLoopPhiOp& pending_phi = to->Cast<PendingLoopPhiOp>();
            OpIndex replaced = __ output_graph().Index(*to);
            __ output_graph().Replace<compiler::turboshaft::PhiOp>(
                replaced, base::VectorOf({pending_phi.first(), ssa_env_[*it]}),
                pending_phi.rep);
          }
          for (uint32_t i = 0; i < block->br_merge()->arity; ++i, ++to) {
            PendingLoopPhiOp& pending_phi = to->Cast<PendingLoopPhiOp>();
            OpIndex replaced = __ output_graph().Index(*to);
            __ output_graph().Replace<compiler::turboshaft::PhiOp>(
                replaced,
                base::VectorOf(
                    {pending_phi.first(), (*block->br_merge())[i].op}),
                pending_phi.rep);
          }
        }
        BindBlockAndGeneratePhis(decoder, post_loop, nullptr);
        break;
      }
    }
  }
 
  void DoReturn(FullDecoder* decoder, uint32_t drop_values) {
    size_t return_count = decoder->sig_->return_count();
    SmallZoneVector<OpIndex, 16> return_values(return_count, decoder->zone_);
    Value* stack_base = return_count == 0
                            ? nullptr
                            : decoder->stack_value(static_cast<uint32_t>(
                                  return_count + drop_values));
    for (size_t i = 0; i < return_count; i++) {
      return_values[i] = stack_base[i].op;
    }
    if (v8_flags.trace_wasm) {
      V<WordPtr> info = __ IntPtrConstant(0);
      if (return_count == 1) {
        wasm::ValueType return_type = decoder->sig_->GetReturn(0);
        int size = return_type.value_kind_size();
        // TODO(14108): This won't fit everything.
        info = __ StackSlot(size, size);
        // TODO(14108): Write barrier might be needed.
        __ Store(
            info, return_values[0], StoreOp::Kind::RawAligned(),
            MemoryRepresentation::FromMachineType(return_type.machine_type()),
            compiler::kNoWriteBarrier);
      }
      CallRuntime(decoder->zone(), Runtime::kWasmTraceExit, {info},
                  __ NoContextConstant());
    }
    if (mode_ == kRegular || mode_ == kInlinedTailCall) {
      __ Return(__ Word32Constant(0), base::VectorOf(return_values),
                v8_flags.experimental_wasm_growable_stacks);
    } else {
      // Do not add return values if we are in unreachable code.
      if (__ generating_unreachable_operations()) return;
      for (size_t i = 0; i < return_count; i++) {
        return_phis_->AddInputForPhi(i, return_values[i]);
      }
      __ Goto(return_block_);
    }
  }
 
  void UnOp(FullDecoder* decoder, WasmOpcode opcode, const Value& value,
            Value* result) {
    result->op = UnOpImpl(opcode, value.op, value.type);
  }
 
  void BinOp(FullDecoder* decoder, WasmOpcode opcode, const Value& lhs,
             const Value& rhs, Value* result) {
    result->op = BinOpImpl(opcode, lhs.op, rhs.op);
  }
 
  void TraceInstruction(FullDecoder* decoder, uint32_t markid) {
    // TODO(14108): Implement.
  }
 
  void I32Const(FullDecoder* decoder, Value* result, int32_t value) {
    result->op = __ Word32Constant(value);
  }
 
  void I64Const(FullDecoder* decoder, Value* result, int64_t value) {
    result->op = __ Word64Constant(value);
  }
 
  void F32Const(FullDecoder* decoder, Value* result, float value) {
    result->op = __ Float32Constant(value);
  }
 
  void F64Const(FullDecoder* decoder, Value* result, double value) {
    result->op = __ Float64Constant(value);
  }
 
  void S128Const(FullDecoder* decoder, const Simd128Immediate& imm,
                 Value* result) {
    result->op = __ Simd128Constant(imm.value);
  }
 
  void RefNull(FullDecoder* decoder, ValueType type, Value* result) {
    result->op = __ Null(type);
  }
 
  void RefFunc(FullDecoder* decoder, uint32_t function_index, Value* result) {
    ModuleTypeIndex sig_index =
        decoder->module_->functions[function_index].sig_index;
    bool shared = decoder->module_->type(sig_index).is_shared;
    result->op = __ WasmRefFunc(trusted_instance_data(shared), function_index);
  }
 
  void RefAsNonNull(FullDecoder* decoder, const Value& arg, Value* result) {
    result->op =
        __ AssertNotNull(arg.op, arg.type, TrapId::kTrapNullDereference);
  }
 
  void Drop(FullDecoder* decoder) {}
 
  void LocalGet(FullDecoder* decoder, Value* result,
                const IndexImmediate& imm) {
    result->op = ssa_env_[imm.index];
  }
 
  void LocalSet(FullDecoder* decoder, const Value& value,
                const IndexImmediate& imm) {
    ssa_env_[imm.index] = value.op;
  }
 
  void LocalTee(FullDecoder* decoder, const Value& value, Value* result,
                const IndexImmediate& imm) {
    ssa_env_[imm.index] = result->op = value.op;
  }
 
  void GlobalGet(FullDecoder* decoder, Value* result,
                 const GlobalIndexImmediate& imm) {
    bool shared = decoder->module_->globals[imm.index].shared;
    result->op = __ GlobalGet(trusted_instance_data(shared), imm.global);
  }
 
  void GlobalSet(FullDecoder* decoder, const Value& value,
                 const GlobalIndexImmediate& imm) {
    bool shared = decoder->module_->globals[imm.index].shared;
    __ GlobalSet(trusted_instance_data(shared), value.op, imm.global);
  }
 
  void Trap(FullDecoder* decoder, TrapReason reason) {
    __ TrapIfNot(__ Word32Constant(0), GetTrapIdForTrap(reason));
    __ Unreachable();
  }
 
  void AssertNullTypecheck(FullDecoder* decoder, const Value& obj,
                           Value* result) {
    __ TrapIfNot(__ IsNull(obj.op, obj.type), TrapId::kTrapIllegalCast);
    Forward(decoder, obj, result);
  }
 
  void AssertNotNullTypecheck(FullDecoder* decoder, const Value& obj,
                              Value* result) {
    __ AssertNotNull(obj.op, obj.type, TrapId::kTrapIllegalCast);
    Forward(decoder, obj, result);
  }
 
  void NopForTestingUnsupportedInLiftoff(FullDecoder* decoder) {
    // This is just for testing bailouts in Liftoff, here it's just a nop.
  }
 
  void Select(FullDecoder* decoder, const Value& cond, const Value& fval,
              const Value& tval, Value* result) {
    using Implementation = compiler::turboshaft::SelectOp::Implementation;
    bool use_select = false;
    switch (tval.type.kind()) {
      case kI32:
        if (SupportedOperations::word32_select()) use_select = true;
        break;
      case kI64:
        if (SupportedOperations::word64_select()) use_select = true;
        break;
      case kF32:
        if (SupportedOperations::float32_select()) use_select = true;
        break;
      case kF64:
        if (SupportedOperations::float64_select()) use_select = true;
        break;
      case kRef:
      case kRefNull:
      case kS128:
        break;
      case kI8:
      case kI16:
      case kF16:
      case kVoid:
      case kTop:
      case kBottom:
        UNREACHABLE();
    }
    result->op = __ Select(
        cond.op, tval.op, fval.op, RepresentationFor(tval.type),
        BranchHint::kNone,
        use_select ? Implementation::kCMove : Implementation::kBranch);
  }
 
  OpIndex BuildChangeEndiannessStore(OpIndex node,
                                     MachineRepresentation mem_rep,
                                     wasm::ValueType wasmtype) {
    OpIndex result;
    OpIndex value = node;
    int value_size_in_bytes = wasmtype.value_kind_size();
    int value_size_in_bits = 8 * value_size_in_bytes;
    bool is_float = false;
 
    switch (wasmtype.kind()) {
      case wasm::kF64:
        value = __ BitcastFloat64ToWord64(node);
        is_float = true;
        [[fallthrough]];
      case wasm::kI64:
        result = __ Word64Constant(static_cast<uint64_t>(0));
        break;
      case wasm::kF32:
        value = __ BitcastFloat32ToWord32(node);
        is_float = true;
        [[fallthrough]];
      case wasm::kI32:
        result = __ Word32Constant(0);
        break;
      case wasm::kS128:
        DCHECK(ReverseBytesSupported(value_size_in_bytes));
        break;
      default:
        UNREACHABLE();
    }
 
    if (mem_rep == MachineRepresentation::kWord8) {
      // No need to change endianness for byte size, return original node
      return node;
    }
    if (wasmtype == wasm::kWasmI64 &&
        mem_rep < MachineRepresentation::kWord64) {
      // In case we store lower part of WasmI64 expression, we can truncate
      // upper 32bits.
      value_size_in_bytes = wasm::kWasmI32.value_kind_size();
      value_size_in_bits = 8 * value_size_in_bytes;
      if (mem_rep == MachineRepresentation::kWord16) {
        value = __ Word32ShiftLeft(value, 16);
      }
    } else if (wasmtype == wasm::kWasmI32 &&
               mem_rep == MachineRepresentation::kWord16) {
      value = __ Word32ShiftLeft(value, 16);
    }
 
    int i;
    uint32_t shift_count;
 
    if (ReverseBytesSupported(value_size_in_bytes)) {
      switch (value_size_in_bytes) {
        case 4:
          result = __ Word32ReverseBytes(V<Word32>::Cast(value));
          break;
        case 8:
          result = __ Word64ReverseBytes(V<Word64>::Cast(value));
          break;
        case 16:
          result = __ Simd128ReverseBytes(
              V<compiler::turboshaft::Simd128>::Cast(value));
          break;
        default:
          UNREACHABLE();
      }
    } else {
      for (i = 0, shift_count = value_size_in_bits - 8;
           i < value_size_in_bits / 2; i += 8, shift_count -= 16) {
        OpIndex shift_lower;
        OpIndex shift_higher;
        OpIndex lower_byte;
        OpIndex higher_byte;
 
        DCHECK_LT(0, shift_count);
        DCHECK_EQ(0, (shift_count + 8) % 16);
 
        if (value_size_in_bits > 32) {
          shift_lower = __ Word64ShiftLeft(value, shift_count);
          shift_higher = __ Word64ShiftRightLogical(value, shift_count);
          lower_byte = __ Word64BitwiseAnd(shift_lower,
                                           static_cast<uint64_t>(0xFF)
                                               << (value_size_in_bits - 8 - i));
          higher_byte = __ Word64BitwiseAnd(shift_higher,
                                            static_cast<uint64_t>(0xFF) << i);
          result = __ Word64BitwiseOr(result, lower_byte);
          result = __ Word64BitwiseOr(result, higher_byte);
        } else {
          shift_lower = __ Word32ShiftLeft(value, shift_count);
          shift_higher = __ Word32ShiftRightLogical(value, shift_count);
          lower_byte = __ Word32BitwiseAnd(shift_lower,
                                           static_cast<uint32_t>(0xFF)
                                               << (value_size_in_bits - 8 - i));
          higher_byte = __ Word32BitwiseAnd(shift_higher,
                                            static_cast<uint32_t>(0xFF) << i);
          result = __ Word32BitwiseOr(result, lower_byte);
          result = __ Word32BitwiseOr(result, higher_byte);
        }
      }
    }
 
    if (is_float) {
      switch (wasmtype.kind()) {
        case wasm::kF64:
          result = __ BitcastWord64ToFloat64(result);
          break;
        case wasm::kF32:
          result = __ BitcastWord32ToFloat32(result);
          break;
        default:
          UNREACHABLE();
      }
    }
 
    return result;
  }
 
  OpIndex BuildChangeEndiannessLoad(OpIndex node, MachineType memtype,
                                    wasm::ValueType wasmtype) {
    OpIndex result;
    OpIndex value = node;
    int value_size_in_bytes = ElementSizeInBytes(memtype.representation());
    int value_size_in_bits = 8 * value_size_in_bytes;
    bool is_float = false;
 
    switch (memtype.representation()) {
      case MachineRepresentation::kFloat64:
        value = __ BitcastFloat64ToWord64(node);
        is_float = true;
        [[fallthrough]];
      case MachineRepresentation::kWord64:
        result = __ Word64Constant(static_cast<uint64_t>(0));
        break;
      case MachineRepresentation::kFloat32:
        value = __ BitcastFloat32ToWord32(node);
        is_float = true;
        [[fallthrough]];
      case MachineRepresentation::kWord32:
      case MachineRepresentation::kWord16:
        result = __ Word32Constant(0);
        break;
      case MachineRepresentation::kWord8:
        // No need to change endianness for byte size, return original node.
        return node;
      case MachineRepresentation::kSimd128:
        DCHECK(ReverseBytesSupported(value_size_in_bytes));
        break;
      default:
        UNREACHABLE();
    }
 
    int i;
    uint32_t shift_count;
 
    if (ReverseBytesSupported(value_size_in_bytes < 4 ? 4
                                                      : value_size_in_bytes)) {
      switch (value_size_in_bytes) {
        case 2:
          result = __ Word32ReverseBytes(__ Word32ShiftLeft(value, 16));
          break;
        case 4:
          result = __ Word32ReverseBytes(value);
          break;
        case 8:
          result = __ Word64ReverseBytes(value);
          break;
        case 16:
          result = __ Simd128ReverseBytes(value);
          break;
        default:
          UNREACHABLE();
      }
    } else {
      for (i = 0, shift_count = value_size_in_bits - 8;
           i < value_size_in_bits / 2; i += 8, shift_count -= 16) {
        OpIndex shift_lower;
        OpIndex shift_higher;
        OpIndex lower_byte;
        OpIndex higher_byte;
 
        DCHECK_LT(0, shift_count);
        DCHECK_EQ(0, (shift_count + 8) % 16);
 
        if (value_size_in_bits > 32) {
          shift_lower = __ Word64ShiftLeft(value, shift_count);
          shift_higher = __ Word64ShiftRightLogical(value, shift_count);
          lower_byte = __ Word64BitwiseAnd(shift_lower,
                                           static_cast<uint64_t>(0xFF)
                                               << (value_size_in_bits - 8 - i));
          higher_byte = __ Word64BitwiseAnd(shift_higher,
                                            static_cast<uint64_t>(0xFF) << i);
          result = __ Word64BitwiseOr(result, lower_byte);
          result = __ Word64BitwiseOr(result, higher_byte);
        } else {
          shift_lower = __ Word32ShiftLeft(value, shift_count);
          shift_higher = __ Word32ShiftRightLogical(value, shift_count);
          lower_byte = __ Word32BitwiseAnd(shift_lower,
                                           static_cast<uint32_t>(0xFF)
                                               << (value_size_in_bits - 8 - i));
          higher_byte = __ Word32BitwiseAnd(shift_higher,
                                            static_cast<uint32_t>(0xFF) << i);
          result = __ Word32BitwiseOr(result, lower_byte);
          result = __ Word32BitwiseOr(result, higher_byte);
        }
      }
    }
 
    if (is_float) {
      switch (memtype.representation()) {
        case MachineRepresentation::kFloat64:
          result = __ BitcastWord64ToFloat64(result);
          break;
        case MachineRepresentation::kFloat32:
          result = __ BitcastWord32ToFloat32(result);
          break;
        default:
          UNREACHABLE();
      }
    }
 
    // We need to sign or zero extend the value.
    // Values with size >= 32-bits may need to be sign/zero extended after
    // calling this function.
    if (value_size_in_bits < 32) {
      DCHECK(!is_float);
      int shift_bit_count = 32 - value_size_in_bits;
      result = __ Word32ShiftLeft(result, shift_bit_count);
      if (memtype.IsSigned()) {
        result =
            __ Word32ShiftRightArithmeticShiftOutZeros(result, shift_bit_count);
      } else {
        result = __ Word32ShiftRightLogical(result, shift_bit_count);
      }
    }
 
    return result;
  }
 
  void LoadMem(FullDecoder* decoder, LoadType type,
               const MemoryAccessImmediate& imm, const Value& index,
               Value* result) {
    bool needs_f16_to_f32_conv = false;
    if (type.value() == LoadType::kF32LoadF16 &&
        !SupportedOperations::float16()) {
      needs_f16_to_f32_conv = true;
      type = LoadType::kI32Load16U;
    }
    MemoryRepresentation repr =
        MemoryRepresentation::FromMachineType(type.mem_type());
 
    auto [final_index, strategy] =
        BoundsCheckMem(imm.memory, repr, index.op, imm.offset,
                       compiler::EnforceBoundsCheck::kCanOmitBoundsCheck,
                       compiler::AlignmentCheck::kNo);
 
    V<WordPtr> mem_start = MemStart(imm.memory->index);
 
    LoadOp::Kind load_kind = GetMemoryAccessKind(repr, strategy);
 
    const bool offset_in_int_range =
        imm.offset <= std::numeric_limits<int32_t>::max();
    OpIndex base =
        offset_in_int_range ? mem_start : __ WordPtrAdd(mem_start, imm.offset);
    int32_t offset = offset_in_int_range ? static_cast<int32_t>(imm.offset) : 0;
    OpIndex load = __ Load(base, final_index, load_kind, repr, offset);
 
#if V8_TARGET_BIG_ENDIAN
    load = BuildChangeEndiannessLoad(load, type.mem_type(), type.value_type());
#endif
 
    if (type.value_type() == kWasmI64 && repr.SizeInBytes() < 8) {
      load = repr.IsSigned() ? __ ChangeInt32ToInt64(load)
                             : __ ChangeUint32ToUint64(load);
    }
 
    if (needs_f16_to_f32_conv) {
      load = CallCStackSlotToStackSlot(
          load, ExternalReference::wasm_float16_to_float32(),
          MemoryRepresentation::Uint16(), MemoryRepresentation::Float32());
    }
 
    if (v8_flags.trace_wasm_memory) {
      // TODO(14259): Implement memory tracing for multiple memories.
      CHECK_EQ(0, imm.memory->index);
      TraceMemoryOperation(decoder, false, repr, final_index, imm.offset);
    }
 
    result->op = load;
  }
 
  void LoadTransform(FullDecoder* decoder, LoadType type,
                     LoadTransformationKind transform,
                     const MemoryAccessImmediate& imm, const Value& index,
                     Value* result) {
    MemoryRepresentation repr =
        transform == LoadTransformationKind::kExtend
            ? MemoryRepresentation::Int64()
            : MemoryRepresentation::FromMachineType(type.mem_type());
 
    auto [final_index, strategy] =
        BoundsCheckMem(imm.memory, repr, index.op, imm.offset,
                       compiler::EnforceBoundsCheck::kCanOmitBoundsCheck,
                       compiler::AlignmentCheck::kNo);
 
    compiler::turboshaft::Simd128LoadTransformOp::LoadKind load_kind =
        GetMemoryAccessKind(repr, strategy);
 
    using TransformKind =
        compiler::turboshaft::Simd128LoadTransformOp::TransformKind;
 
    TransformKind transform_kind;
 
    if (transform == LoadTransformationKind::kExtend) {
      if (type.mem_type() == MachineType::Int8()) {
        transform_kind = TransformKind::k8x8S;
      } else if (type.mem_type() == MachineType::Uint8()) {
        transform_kind = TransformKind::k8x8U;
      } else if (type.mem_type() == MachineType::Int16()) {
        transform_kind = TransformKind::k16x4S;
      } else if (type.mem_type() == MachineType::Uint16()) {
        transform_kind = TransformKind::k16x4U;
      } else if (type.mem_type() == MachineType::Int32()) {
        transform_kind = TransformKind::k32x2S;
      } else if (type.mem_type() == MachineType::Uint32()) {
        transform_kind = TransformKind::k32x2U;
      } else {
        UNREACHABLE();
      }
    } else if (transform == LoadTransformationKind::kSplat) {
      if (type.mem_type() == MachineType::Int8()) {
        transform_kind = TransformKind::k8Splat;
      } else if (type.mem_type() == MachineType::Int16()) {
        transform_kind = TransformKind::k16Splat;
      } else if (type.mem_type() == MachineType::Int32()) {
        transform_kind = TransformKind::k32Splat;
      } else if (type.mem_type() == MachineType::Int64()) {
        transform_kind = TransformKind::k64Splat;
      } else {
        UNREACHABLE();
      }
    } else {
      if (type.mem_type() == MachineType::Int32()) {
        transform_kind = TransformKind::k32Zero;
      } else if (type.mem_type() == MachineType::Int64()) {
        transform_kind = TransformKind::k64Zero;
      } else {
        UNREACHABLE();
      }
    }
 
    V<compiler::turboshaft::Simd128> load = __ Simd128LoadTransform(
        __ WordPtrAdd(MemStart(imm.mem_index), imm.offset), final_index,
        load_kind, transform_kind, 0);
 
    if (v8_flags.trace_wasm_memory) {
      TraceMemoryOperation(decoder, false, repr, final_index, imm.offset);
    }
 
    result->op = load;
  }
 
  void LoadLane(FullDecoder* decoder, LoadType type, const Value& value,
                const Value& index, const MemoryAccessImmediate& imm,
                const uint8_t laneidx, Value* result) {
    using compiler::turboshaft::Simd128LaneMemoryOp;
 
    MemoryRepresentation repr =
        MemoryRepresentation::FromMachineType(type.mem_type());
 
    auto [final_index, strategy] =
        BoundsCheckMem(imm.memory, repr, index.op, imm.offset,
                       compiler::EnforceBoundsCheck::kCanOmitBoundsCheck,
                       compiler::AlignmentCheck::kNo);
    Simd128LaneMemoryOp::Kind kind = GetMemoryAccessKind(repr, strategy);
 
    Simd128LaneMemoryOp::LaneKind lane_kind;
 
    switch (repr) {
      case MemoryRepresentation::Int8():
        lane_kind = Simd128LaneMemoryOp::LaneKind::k8;
        break;
      case MemoryRepresentation::Int16():
        lane_kind = Simd128LaneMemoryOp::LaneKind::k16;
        break;
      case MemoryRepresentation::Int32():
        lane_kind = Simd128LaneMemoryOp::LaneKind::k32;
        break;
      case MemoryRepresentation::Int64():
        lane_kind = Simd128LaneMemoryOp::LaneKind::k64;
        break;
      default:
        UNREACHABLE();
    }
 
    // TODO(14108): If `offset` is in int range, use it as static offset, or
    // consider using a larger type as offset.
    OpIndex load = __ Simd128LaneMemory(
        __ WordPtrAdd(MemStart(imm.mem_index), imm.offset), final_index,
        value.op, Simd128LaneMemoryOp::Mode::kLoad, kind, lane_kind, laneidx,
        0);
 
    if (v8_flags.trace_wasm_memory) {
      TraceMemoryOperation(decoder, false, repr, final_index, imm.offset);
    }
 
    result->op = load;
  }
 
  void StoreMem(FullDecoder* decoder, StoreType type,
                const MemoryAccessImmediate& imm, const Value& index,
                const Value& value) {
    bool needs_f32_to_f16_conv = false;
    if (type.value() == StoreType::kF32StoreF16 &&
        !SupportedOperations::float16()) {
      needs_f32_to_f16_conv = true;
      type = StoreType::kI32Store16;
    }
    MemoryRepresentation repr =
        MemoryRepresentation::FromMachineRepresentation(type.mem_rep());
 
    compiler::EnforceBoundsCheck enforce_bounds_check =
        (wasm::kPartialOOBWritesAreNoops || type.size() == 1)
            ? compiler::EnforceBoundsCheck::kCanOmitBoundsCheck
            : compiler::EnforceBoundsCheck::kNeedsBoundsCheck;
 
    auto [final_index, strategy] =
        BoundsCheckMem(imm.memory, repr, index.op, imm.offset,
                       enforce_bounds_check, compiler::AlignmentCheck::kNo);
 
    V<WordPtr> mem_start = MemStart(imm.memory->index);
 
    StoreOp::Kind store_kind = GetMemoryAccessKind(repr, strategy);
 
    OpIndex store_value = value.op;
    if (value.type == kWasmI64 && repr.SizeInBytes() <= 4) {
      store_value = __ TruncateWord64ToWord32(store_value);
    }
    if (needs_f32_to_f16_conv) {
      store_value = CallCStackSlotToStackSlot(
          store_value, ExternalReference::wasm_float32_to_float16(),
          MemoryRepresentation::Float32(), MemoryRepresentation::Int16());
    }
 
#if defined(V8_TARGET_BIG_ENDIAN)
    store_value = BuildChangeEndiannessStore(store_value, type.mem_rep(),
                                             type.value_type());
#endif
    const bool offset_in_int_range =
        imm.offset <= std::numeric_limits<int32_t>::max();
    OpIndex base =
        offset_in_int_range ? mem_start : __ WordPtrAdd(mem_start, imm.offset);
    int32_t offset = offset_in_int_range ? static_cast<int32_t>(imm.offset) : 0;
    __ Store(base, final_index, store_value, store_kind, repr,
             compiler::kNoWriteBarrier, offset);
 
    if (v8_flags.trace_wasm_memory) {
      // TODO(14259): Implement memory tracing for multiple memories.
      CHECK_EQ(0, imm.memory->index);
      TraceMemoryOperation(decoder, true, repr, final_index, imm.offset);
    }
  }
 
  void StoreLane(FullDecoder* decoder, StoreType type,
                 const MemoryAccessImmediate& imm, const Value& index,
                 const Value& value, const uint8_t laneidx) {
    using compiler::turboshaft::Simd128LaneMemoryOp;
 
    MemoryRepresentation repr =
        MemoryRepresentation::FromMachineRepresentation(type.mem_rep());
 
    compiler::EnforceBoundsCheck enforce_bounds_check =
        (wasm::kPartialOOBWritesAreNoops || type.size() == 1)
            ? compiler::EnforceBoundsCheck::kCanOmitBoundsCheck
            : compiler::EnforceBoundsCheck::kNeedsBoundsCheck;
 
    auto [final_index, strategy] =
        BoundsCheckMem(imm.memory, repr, index.op, imm.offset,
                       enforce_bounds_check, compiler::AlignmentCheck::kNo);
    Simd128LaneMemoryOp::Kind kind = GetMemoryAccessKind(repr, strategy);
 
    Simd128LaneMemoryOp::LaneKind lane_kind;
 
    switch (repr) {
      // TODO(manoskouk): Why use unsigned representations here as opposed to
      // LoadLane?
      case MemoryRepresentation::Uint8():
        lane_kind = Simd128LaneMemoryOp::LaneKind::k8;
        break;
      case MemoryRepresentation::Uint16():
        lane_kind = Simd128LaneMemoryOp::LaneKind::k16;
        break;
      case MemoryRepresentation::Uint32():
        lane_kind = Simd128LaneMemoryOp::LaneKind::k32;
        break;
      case MemoryRepresentation::Uint64():
        lane_kind = Simd128LaneMemoryOp::LaneKind::k64;
        break;
      default:
        UNREACHABLE();
    }
 
    // TODO(14108): If `offset` is in int range, use it as static offset, or
    // consider using a larger type as offset.
    __ Simd128LaneMemory(__ WordPtrAdd(MemStart(imm.mem_index), imm.offset),
                         final_index, value.op,
                         Simd128LaneMemoryOp::Mode::kStore, kind, lane_kind,
                         laneidx, 0);
 
    if (v8_flags.trace_wasm_memory) {
      TraceMemoryOperation(decoder, true, repr, final_index, imm.offset);
    }
  }
 
  void CurrentMemoryPages(FullDecoder* decoder, const MemoryIndexImmediate& imm,
                          Value* result) {
    V<WordPtr> result_wordptr =
        __ WordPtrShiftRightArithmetic(MemSize(imm.index), kWasmPageSizeLog2);
    // In the 32-bit case, truncation happens implicitly.
    if (imm.memory->is_memory64()) {
      result->op = __ ChangeIntPtrToInt64(result_wordptr);
    } else {
      result->op = __ TruncateWordPtrToWord32(result_wordptr);
    }
  }
 
  void MemoryGrow(FullDecoder* decoder, const MemoryIndexImmediate& imm,
                  const Value& value, Value* result) {
    if (!imm.memory->is_memory64()) {
      result->op =
          CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmMemoryGrow>(
              decoder, {__ Word32Constant(imm.index), value.op});
    } else {
      Label<Word64> done(&asm_);
 
      IF (LIKELY(__ Uint64LessThanOrEqual(
              value.op, __ Word64Constant(static_cast<int64_t>(kMaxInt))))) {
        GOTO(done, __ ChangeInt32ToInt64(CallBuiltinThroughJumptable<
                                         BuiltinCallDescriptor::WasmMemoryGrow>(
                       decoder, {__ Word32Constant(imm.index),
                                 __ TruncateWord64ToWord32(value.op)})));
      } ELSE {
        GOTO(done, __ Word64Constant(int64_t{-1}));
      }
 
      BIND(done, result_64);
 
      result->op = result_64;
    }
    instance_cache_.ReloadCachedMemory();
  }
 
  V<Word32> IsExternRefString(const Value value) {
    compiler::WasmTypeCheckConfig config{value.type, kWasmRefExternString};
    V<Map> rtt = OpIndex::Invalid();
    return __ WasmTypeCheck(value.op, rtt, config);
  }
 
  V<String> ExternRefToString(const Value value, bool null_succeeds = false) {
    wasm::ValueType target_type =
        null_succeeds ? kWasmRefNullExternString : kWasmRefExternString;
    compiler::WasmTypeCheckConfig config{value.type, target_type};
    V<Map> rtt = OpIndex::Invalid();
    return V<String>::Cast(__ WasmTypeCast(value.op, rtt, config));
  }
 
  bool IsExplicitStringCast(const Value value) {
    if (__ generating_unreachable_operations()) return false;
    const WasmTypeCastOp* cast =
        __ output_graph().Get(value.op).TryCast<WasmTypeCastOp>();
    return cast && cast->config.to == kWasmRefExternString;
  }
 
  V<Word32> GetStringIndexOf(FullDecoder* decoder, V<String> string,
                             V<String> search, V<Word32> start) {
    // Clamp the start index.
    Label<Word32> clamped_start_label(&asm_);
    GOTO_IF(__ Int32LessThan(start, 0), clamped_start_label,
            __ Word32Constant(0));
    V<Word32> length = __ template LoadField<Word32>(
        string, compiler::AccessBuilder::ForStringLength());
    GOTO_IF(__ Int32LessThan(start, length), clamped_start_label, start);
    GOTO(clamped_start_label, length);
    BIND(clamped_start_label, clamped_start);
    start = clamped_start;
 
    // This can't overflow because we've clamped `start` above.
    V<Smi> start_smi = __ TagSmi(start);
    BuildModifyThreadInWasmFlag(decoder->zone(), false);
 
    V<Smi> result_value =
        CallBuiltinThroughJumptable<BuiltinCallDescriptor::StringIndexOf>(
            decoder, {string, search, start_smi});
    BuildModifyThreadInWasmFlag(decoder->zone(), true);
 
    return __ UntagSmi(result_value);
  }
 
#if V8_INTL_SUPPORT
  V<String> CallStringToLowercase(FullDecoder* decoder, V<String> string) {
    BuildModifyThreadInWasmFlag(decoder->zone(), false);
    OpIndex result = CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::StringToLowerCaseIntl>(
        decoder, __ NoContextConstant(), {string});
    BuildModifyThreadInWasmFlag(decoder->zone(), true);
    return result;
  }
#endif
 
  void SetDataViewOpForErrorMessage(DataViewOp op_type) {
    OpIndex isolate_root = __ LoadRootRegister();
    __ Store(isolate_root, __ Word32Constant(op_type),
             StoreOp::Kind::RawAligned(), MemoryRepresentation::Uint8(),
             compiler::kNoWriteBarrier, Isolate::error_message_param_offset());
  }
 
  void ThrowDataViewTypeError(FullDecoder* decoder, V<Object> dataview,
                              DataViewOp op_type) {
    SetDataViewOpForErrorMessage(op_type);
    CallBuiltinThroughJumptable<BuiltinCallDescriptor::ThrowDataViewTypeError>(
        decoder, {V<JSDataView>::Cast(dataview)});
    __ Unreachable();
  }
 
  void ThrowDataViewOutOfBoundsError(FullDecoder* decoder, DataViewOp op_type) {
    SetDataViewOpForErrorMessage(op_type);
    CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::ThrowDataViewOutOfBounds>(decoder, {});
    __ Unreachable();
  }
 
  void ThrowDataViewDetachedError(FullDecoder* decoder, DataViewOp op_type) {
    SetDataViewOpForErrorMessage(op_type);
    CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::ThrowDataViewDetachedError>(decoder, {});
    __ Unreachable();
  }
 
  void DataViewRangeCheck(FullDecoder* decoder, V<WordPtr> left,
                          V<WordPtr> right, DataViewOp op_type) {
    IF (UNLIKELY(__ IntPtrLessThan(left, right))) {
      ThrowDataViewOutOfBoundsError(decoder, op_type);
    }
  }
 
  void DataViewBoundsCheck(FullDecoder* decoder, V<WordPtr> left,
                           V<WordPtr> right, DataViewOp op_type) {
    IF (UNLIKELY(__ IntPtrLessThan(left, right))) {
      ThrowDataViewDetachedError(decoder, op_type);
    }
  }
 
  void DataViewDetachedBufferCheck(FullDecoder* decoder, V<Object> dataview,
                                   DataViewOp op_type) {
    IF (UNLIKELY(
            __ ArrayBufferIsDetached(V<JSArrayBufferView>::Cast(dataview)))) {
      ThrowDataViewDetachedError(decoder, op_type);
    }
  }
 
  V<WordPtr> GetDataViewByteLength(FullDecoder* decoder, V<Object> dataview,
                                   DataViewOp op_type) {
    DCHECK_EQ(op_type, DataViewOp::kByteLength);
    return GetDataViewByteLength(decoder, dataview, __ IntPtrConstant(0),
                                 op_type);
  }
 
  // Converts a Smi or HeapNumber to an intptr. The input is not validated.
  V<WordPtr> ChangeTaggedNumberToIntPtr(V<Object> tagged) {
    Label<> smi_label(&asm_);
    Label<> heapnumber_label(&asm_);
    Label<WordPtr> done_label(&asm_);
 
    GOTO_IF(LIKELY(__ IsSmi(tagged)), smi_label);
    GOTO(heapnumber_label);
 
    BIND(smi_label);
    V<WordPtr> smi_length =
        __ ChangeInt32ToIntPtr(__ UntagSmi(V<Smi>::Cast(tagged)));
    GOTO(done_label, smi_length);
 
    BIND(heapnumber_label);
    V<Float64> float_value = __ template LoadField<Float64>(
        tagged, AccessBuilder::ForHeapNumberValue());
    if constexpr (Is64()) {
      DCHECK_EQ(WordPtr::bits, Word64::bits);
      GOTO(done_label,
           V<WordPtr>::Cast(
               __ TruncateFloat64ToInt64OverflowUndefined(float_value)));
    } else {
      GOTO(done_label,
           __ ChangeInt32ToIntPtr(
               __ TruncateFloat64ToInt32OverflowUndefined(float_value)));
    }
 
    BIND(done_label, length);
    return length;
  }
 
  // An `ArrayBuffer` can be resizable, i.e. it can shrink or grow.
  // A `SharedArrayBuffer` can be growable, i.e. it can only grow. A `DataView`
  // can be length-tracking or non-legth-tracking . A length-tracking `DataView`
  // is tracking the length of the underlying buffer, i.e. it doesn't have a
  // `byteLength` specified, which means that the length of the `DataView` is
  // the length (or remaining length if `byteOffset != 0`) of the underlying
  // array buffer. On the other hand, a non-length-tracking `DataView` has a
  // `byteLength`.
  // Depending on whether the buffer is resizable or growable and the `DataView`
  // is length-tracking or non-length-tracking, getting the byte length has to
  // be handled differently.
  V<WordPtr> GetDataViewByteLength(FullDecoder* decoder, V<Object> dataview,
                                   V<WordPtr> offset, DataViewOp op_type) {
    Label<WordPtr> done_label(&asm_);
    Label<> type_error_label(&asm_);
 
    GOTO_IF(UNLIKELY(__ IsSmi(dataview)), type_error_label);
 
    // Case 1):
    //  - non-resizable ArrayBuffers, length-tracking and non-length-tracking
    //  - non-growable SharedArrayBuffers, length-tracking and non-length-tr.
    //  - growable SharedArrayBuffers, non-length-tracking
    IF (LIKELY(__ HasInstanceType(dataview, InstanceType::JS_DATA_VIEW_TYPE))) {
      if (op_type != DataViewOp::kByteLength) {
        DataViewRangeCheck(decoder, offset, __ IntPtrConstant(0), op_type);
      }
      DataViewDetachedBufferCheck(decoder, dataview, op_type);
      V<WordPtr> view_byte_length = __ LoadField<WordPtr>(
          dataview, AccessBuilder::ForJSArrayBufferViewByteLength());
      GOTO(done_label, view_byte_length);
    }
 
    // Case 2):
    // - resizable ArrayBuffers, length-tracking and non-length-tracking
    // - growable SharedArrayBuffers, length-tracking
    GOTO_IF_NOT(LIKELY(__ HasInstanceType(
                    dataview, InstanceType::JS_RAB_GSAB_DATA_VIEW_TYPE)),
                type_error_label);
    if (op_type != DataViewOp::kByteLength) {
      DataViewRangeCheck(decoder, offset, __ IntPtrConstant(0), op_type);
    }
    DataViewDetachedBufferCheck(decoder, dataview, op_type);
 
    V<Word32> bit_field = __ LoadField<Word32>(
        dataview, AccessBuilder::ForJSArrayBufferViewBitField());
    V<Word32> length_tracking = __ Word32BitwiseAnd(
        bit_field, JSArrayBufferView::IsLengthTrackingBit::kMask);
    V<Word32> backed_by_rab_bit = __ Word32BitwiseAnd(
        bit_field, JSArrayBufferView::IsBackedByRabBit::kMask);
 
    V<Object> buffer = __ LoadField<Object>(
        dataview, compiler::AccessBuilder::ForJSArrayBufferViewBuffer());
    V<WordPtr> buffer_byte_length = __ LoadField<WordPtr>(
        buffer, AccessBuilder::ForJSArrayBufferByteLength());
    V<WordPtr> view_byte_offset = __ LoadField<WordPtr>(
        dataview, AccessBuilder::ForJSArrayBufferViewByteOffset());
 
    // The final length for each case in Case 2) is calculated differently.
    // Case: resizable ArrayBuffers, LT and non-LT.
    IF (backed_by_rab_bit) {
      // DataViews with resizable ArrayBuffers can go out of bounds.
      IF (length_tracking) {
        ScopedVar<WordPtr> final_length(this, 0);
        IF (LIKELY(__ UintPtrLessThanOrEqual(view_byte_offset,
                                             buffer_byte_length))) {
          final_length = __ WordPtrSub(buffer_byte_length, view_byte_offset);
        }
        DataViewBoundsCheck(decoder, buffer_byte_length, view_byte_offset,
                            op_type);
        GOTO(done_label, final_length);
      } ELSE {
        V<WordPtr> view_byte_length = __ LoadField<WordPtr>(
            dataview, AccessBuilder::ForJSArrayBufferViewByteLength());
        DataViewBoundsCheck(decoder, buffer_byte_length,
                            __ WordPtrAdd(view_byte_offset, view_byte_length),
                            op_type);
 
        GOTO(done_label, view_byte_length);
      }
    }
    // Case: growable SharedArrayBuffers, LT.
    ELSE {
      V<Object> gsab_length_tagged = CallRuntime(
          decoder->zone(), Runtime::kGrowableSharedArrayBufferByteLength,
          {buffer}, __ NoContextConstant());
      V<WordPtr> gsab_length = ChangeTaggedNumberToIntPtr(gsab_length_tagged);
      ScopedVar<WordPtr> gsab_buffer_byte_length(this, 0);
      IF (LIKELY(__ UintPtrLessThanOrEqual(view_byte_offset, gsab_length))) {
        gsab_buffer_byte_length = __ WordPtrSub(gsab_length, view_byte_offset);
      }
      GOTO(done_label, gsab_buffer_byte_length);
    }
    __ Unreachable();
 
    BIND(type_error_label);
    ThrowDataViewTypeError(decoder, dataview, op_type);
 
    BIND(done_label, final_view_byte_length);
    return final_view_byte_length;
  }
 
  V<WordPtr> GetDataViewDataPtr(FullDecoder* decoder, V<Object> dataview,
                                V<WordPtr> offset, DataViewOp op_type) {
    V<WordPtr> view_byte_length =
        GetDataViewByteLength(decoder, dataview, offset, op_type);
    V<WordPtr> view_byte_length_minus_size =
        __ WordPtrSub(view_byte_length, GetTypeSize(op_type));
    DataViewRangeCheck(decoder, view_byte_length_minus_size, offset, op_type);
    return __ LoadField<WordPtr>(
        dataview, compiler::AccessBuilder::ForJSDataViewDataPointer());
  }
 
  OpIndex DataViewGetter(FullDecoder* decoder, const Value args[],
                         DataViewOp op_type) {
    V<Object> dataview = args[0].op;
    V<WordPtr> offset = __ ChangeInt32ToIntPtr(args[1].op);
    V<Word32> is_little_endian =
        (op_type == DataViewOp::kGetInt8 || op_type == DataViewOp::kGetUint8)
            ? __ Word32Constant(1)
            : args[2].op;
 
    V<WordPtr> data_ptr =
        GetDataViewDataPtr(decoder, dataview, offset, op_type);
    return __ LoadDataViewElement(dataview, data_ptr, offset, is_little_endian,
                                  GetExternalArrayType(op_type));
  }
 
  void DataViewSetter(FullDecoder* decoder, const Value args[],
                      DataViewOp op_type) {
    V<Object> dataview = args[0].op;
    V<WordPtr> offset = __ ChangeInt32ToIntPtr(args[1].op);
    V<Word32> value = args[2].op;
    V<Word32> is_little_endian =
        (op_type == DataViewOp::kSetInt8 || op_type == DataViewOp::kSetUint8)
            ? __ Word32Constant(1)
            : args[3].op;
 
    V<WordPtr> data_ptr =
        GetDataViewDataPtr(decoder, dataview, offset, op_type);
    __ StoreDataViewElement(dataview, data_ptr, offset, value, is_little_endian,
                            GetExternalArrayType(op_type));
  }
 
  // Adds a wasm type annotation to the graph and replaces any extern type with
  // the extern string type.
  template <typename T>
  V<T> AnnotateAsString(V<T> value, wasm::ValueType type) {
    DCHECK(type.is_reference_to(HeapType::kString) ||
           type.is_reference_to(HeapType::kExternString) ||
           type.is_reference_to(HeapType::kExtern));
    if (type.is_reference_to(HeapType::kExtern)) {
      type = ValueType::RefMaybeNull(kWasmRefExternString, type.nullability());
    }
    return __ AnnotateWasmType(value, type);
  }
 
  void WellKnown_FastApi(FullDecoder* decoder, const CallFunctionImmediate& imm,
                         const Value args[], Value returns[]) {
    uint32_t func_index = imm.index;
    V<Object> receiver = args[0].op;
    // TODO(14616): Fix this.
    V<FixedArray> imports_array = LOAD_IMMUTABLE_INSTANCE_FIELD(
        trusted_instance_data(false), WellKnownImports,
        MemoryRepresentation::TaggedPointer());
    V<Object> data = __ LoadFixedArrayElement(imports_array, func_index);
    V<Object> cached_map = __ Load(data, LoadOp::Kind::TaggedBase(),
                                   MemoryRepresentation::TaggedPointer(),
                                   WasmFastApiCallData::kCachedMapOffset);
 
    Label<> if_equal_maps(&asm_);
    Label<> if_unknown_receiver(&asm_);
    GOTO_IF(__ IsSmi(receiver), if_unknown_receiver);
 
    V<Map> map = __ LoadMapField(V<Object>::Cast(receiver));
 
    // Clear the weak bit.
    cached_map = __ BitcastWordPtrToTagged(__ WordPtrBitwiseAnd(
        __ BitcastTaggedToWordPtr(cached_map), ~kWeakHeapObjectMask));
    GOTO_IF(__ TaggedEqual(map, cached_map), if_equal_maps);
    GOTO(if_unknown_receiver);
 
    BIND(if_unknown_receiver);
    V<NativeContext> context = instance_cache_.native_context();
    CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::WasmFastApiCallTypeCheckAndUpdateIC>(
        decoder, context, {data, receiver});
    GOTO(if_equal_maps);
 
    BIND(if_equal_maps);
    OpIndex receiver_handle = __ AdaptLocalArgument(receiver);
 
    const wasm::FunctionSig* sig = decoder->module_->functions[func_index].sig;
    size_t param_count = sig->parameter_count();
    DCHECK_LE(sig->return_count(), 1);
 
    const MachineSignature* callback_sig =
        env_->fast_api_signatures[func_index];
    // All normal parameters + the options as additional parameter at the end.
    MachineSignature::Builder builder(decoder->zone(), sig->return_count(),
                                      param_count + 1);
    if (sig->return_count()) {
      builder.AddReturn(callback_sig->GetReturn());
    }
    // The first parameter is the receiver. Because of the fake handle on the
    // stack the type is `Pointer`.
    builder.AddParam(MachineType::Pointer());
 
    for (size_t i = 0; i < callback_sig->parameter_count(); ++i) {
      builder.AddParam(callback_sig->GetParam(i));
    }
    // Options object.
    builder.AddParam(MachineType::Pointer());
 
    base::SmallVector<OpIndex, 16> inputs(param_count + 1);
 
    inputs[0] = receiver_handle;
 
    Label<> value_out_of_range(&asm_);
    for (size_t i = 1; i < param_count; ++i) {
      if (sig->GetParam(i).is_reference()) {
        inputs[i] = __ AdaptLocalArgument(args[i].op);
      } else if (callback_sig->GetParam(i - 1).representation() ==
                 MachineRepresentation::kWord64) {
        if (sig->GetParam(i) == kWasmI64) {
          // If we already have an I64, then no conversion is needed neither for
          // int64 nor uint64.
          inputs[i] = args[i].op;
        } else if (callback_sig->GetParam(i - 1) == MachineType::Int64()) {
          if (sig->GetParam(i) == kWasmF64) {
            V<Tuple<Word64, Word32>> truncate =
                __ TryTruncateFloat64ToInt64(args[i].op);
            inputs[i] = __ template Projection<0>(truncate);
            GOTO_IF(UNLIKELY(
                        __ Word32Equal(__ template Projection<1>(truncate), 0)),
                    value_out_of_range);
          } else if (sig->GetParam(i) == kWasmI32) {
            inputs[i] = __ ChangeInt32ToInt64(args[i].op);
          } else {
            // TODO(ahaas): Handle values that are out of range of int64.
            CHECK_EQ(sig->GetParam(i), kWasmF32);
            V<Tuple<Word64, Word32>> truncate =
                __ TryTruncateFloat32ToInt64(args[i].op);
            inputs[i] = __ template Projection<0>(truncate);
            GOTO_IF(UNLIKELY(
                        __ Word32Equal(__ template Projection<1>(truncate), 0)),
                    value_out_of_range);
          }
        } else if (callback_sig->GetParam(i - 1) == MachineType::Uint64()) {
          if (sig->GetParam(i) == kWasmF64) {
            V<Tuple<Word64, Word32>> truncate =
                __ TryTruncateFloat64ToUint64(args[i].op);
            inputs[i] = __ template Projection<0>(truncate);
            GOTO_IF(UNLIKELY(
                        __ Word32Equal(__ template Projection<1>(truncate), 0)),
                    value_out_of_range);
          } else if (sig->GetParam(i) == kWasmI32) {
            inputs[i] = __ ChangeUint32ToUint64(args[i].op);
          } else {
            // TODO(ahaas): Handle values that are out of range of int64.
            CHECK_EQ(sig->GetParam(i), kWasmF32);
            V<Tuple<Word64, Word32>> truncate =
                __ TryTruncateFloat32ToUint64(args[i].op);
            inputs[i] = __ template Projection<0>(truncate);
            GOTO_IF(UNLIKELY(
                        __ Word32Equal(__ template Projection<1>(truncate), 0)),
                    value_out_of_range);
          }
        }
      } else {
        inputs[i] = args[i].op;
      }
    }
 
    OpIndex options_object;
    {
      const int kAlign = alignof(v8::FastApiCallbackOptions);
      const int kSize = sizeof(v8::FastApiCallbackOptions);
 
      options_object = __ StackSlot(kSize, kAlign);
 
      static_assert(
          sizeof(v8::FastApiCallbackOptions::isolate) == sizeof(intptr_t),
          "We expected 'isolate' to be pointer sized, but it is not.");
      __ StoreOffHeap(options_object,
                      __ IsolateField(IsolateFieldId::kIsolateAddress),
                      MemoryRepresentation::UintPtr(),
                      offsetof(v8::FastApiCallbackOptions, isolate));
 
      V<Object> callback_data =
          __ Load(data, LoadOp::Kind::TaggedBase(),
                  MemoryRepresentation::TaggedPointer(),
                  WasmFastApiCallData::kCallbackDataOffset);
      V<WordPtr> data_argument_to_pass = __ AdaptLocalArgument(callback_data);
 
      __ StoreOffHeap(options_object, data_argument_to_pass,
                      MemoryRepresentation::UintPtr(),
                      offsetof(v8::FastApiCallbackOptions, data));
    }
 
    inputs[param_count] = options_object;
 
    const CallDescriptor* call_descriptor =
        compiler::Linkage::GetSimplifiedCDescriptor(__ graph_zone(),
                                                    builder.Get());
    const TSCallDescriptor* ts_call_descriptor = TSCallDescriptor::Create(
        call_descriptor, compiler::CanThrow::kNo,
        compiler::LazyDeoptOnThrow::kNo, __ graph_zone());
    OpIndex target_address = __ ExternalConstant(ExternalReference::Create(
        env_->fast_api_targets[func_index].load(std::memory_order_relaxed),
        ExternalReference::FAST_C_CALL));
 
    V<Context> native_context = instance_cache_.native_context();
 
    __ Store(__ LoadRootRegister(),
             __ BitcastHeapObjectToWordPtr(native_context),
             StoreOp::Kind::RawAligned(), MemoryRepresentation::UintPtr(),
             compiler::kNoWriteBarrier, Isolate::context_offset());
    BuildModifyThreadInWasmFlag(__ graph_zone(), false);
    auto [old_sp, old_limit] = BuildSwitchToTheCentralStackIfNeeded();
    OpIndex ret_val = __ Call(target_address, OpIndex::Invalid(),
                              base::VectorOf(inputs), ts_call_descriptor);
    BuildSwitchBackFromCentralStack(old_sp, old_limit);
 
#if DEBUG
    // Reset the context again after the call, to make sure nobody is using the
    // leftover context in the isolate.
    __ Store(__ LoadRootRegister(),
             __ WordPtrConstant(Context::kInvalidContext),
             StoreOp::Kind::RawAligned(), MemoryRepresentation::UintPtr(),
             compiler::kNoWriteBarrier, Isolate::context_offset());
#endif
 
    V<Object> exception = __ Load(
        __ LoadRootRegister(), LoadOp::Kind::RawAligned(),
        MemoryRepresentation::UintPtr(), IsolateData::exception_offset());
 
    IF_NOT (LIKELY(
                __ TaggedEqual(exception, LOAD_ROOT(TheHoleValue)))) {
      CallBuiltinThroughJumptable<
          BuiltinCallDescriptor::WasmPropagateException>(
          decoder, {}, CheckForException::kCatchInThisFrame);
    }
    BuildModifyThreadInWasmFlag(__ graph_zone(), true);
 
    if (callback_sig->return_count() > 0) {
      if (callback_sig->GetReturn() == MachineType::Bool()) {
        ret_val = __ WordBitwiseAnd(ret_val, __ Word32Constant(0xff),
                                    WordRepresentation::Word32());
      } else if (callback_sig->GetReturn() == MachineType::Int64()) {
        if (sig->GetReturn() == kWasmF64) {
          ret_val = __ ChangeInt64ToFloat64(ret_val);
        } else if (sig->GetReturn() == kWasmI32) {
          ret_val = __ TruncateWord64ToWord32(ret_val);
        } else if (sig->GetReturn() == kWasmF32) {
          ret_val = __ ChangeInt64ToFloat32(ret_val);
        }
      } else if (callback_sig->GetReturn() == MachineType::Uint64()) {
        if (sig->GetReturn() == kWasmF64) {
          ret_val = __ ChangeUint64ToFloat64(ret_val);
        } else if (sig->GetReturn() == kWasmI32) {
          ret_val = __ TruncateWord64ToWord32(ret_val);
        } else if (sig->GetReturn() == kWasmF32) {
          ret_val = __ ChangeUint64ToFloat32(ret_val);
        }
      }
    }
    Label<> done(&asm_);
    GOTO(done);
    BIND(value_out_of_range);
    auto [target, implicit_arg] =
        BuildImportedFunctionTargetAndImplicitArg(decoder, imm.index);
    BuildWasmCall(decoder, imm.sig, target, implicit_arg, args, returns,
                  compiler::kWasmIndirectFunction);
    __ Unreachable();
    BIND(done);
    if (sig->return_count()) {
      returns[0].op = ret_val;
    }
  }
 
  bool HandleWellKnownImport(FullDecoder* decoder,
                             const CallFunctionImmediate& imm,
                             const Value args[], Value returns[]) {
    uint32_t index = imm.index;
    if (!decoder->module_) return false;  // Only needed for tests.
    const WellKnownImportsList& well_known_imports =
        decoder->module_->type_feedback.well_known_imports;
    using WKI = WellKnownImport;
    WKI imported_op = well_known_imports.get(index);
    OpIndex result;
    switch (imported_op) {
      case WKI::kUninstantiated:
      case WKI::kGeneric:
      case WKI::kLinkError:
        return false;
 
      // JS String Builtins proposal.
      case WKI::kStringCast: {
        result = ExternRefToString(args[0]);
        decoder->detected_->add_imported_strings();
        break;
      }
      case WKI::kStringTest: {
        result = IsExternRefString(args[0]);
        decoder->detected_->add_imported_strings();
        break;
      }
      case WKI::kStringCharCodeAt: {
        V<String> string = ExternRefToString(args[0]);
        V<String> view = __ StringAsWtf16(string);
        // TODO(14108): Annotate `view`'s type.
        result = GetCodeUnitImpl(decoder, view, args[1].op);
        decoder->detected_->add_imported_strings();
        break;
      }
      case WKI::kStringCodePointAt: {
        V<String> string = ExternRefToString(args[0]);
        V<String> view = __ StringAsWtf16(string);
        // TODO(14108): Annotate `view`'s type.
        result = StringCodePointAt(decoder, view, args[1].op);
        decoder->detected_->add_imported_strings();
        break;
      }
      case WKI::kStringCompare: {
        V<String> a_string = ExternRefToString(args[0]);
        V<String> b_string = ExternRefToString(args[1]);
        result = __ UntagSmi(
            CallBuiltinThroughJumptable<BuiltinCallDescriptor::StringCompare>(
                decoder, {a_string, b_string}));
        decoder->detected_->add_imported_strings();
        break;
      }
      case WKI::kStringConcat: {
        V<String> head_string = ExternRefToString(args[0]);
        V<String> tail_string = ExternRefToString(args[1]);
        V<HeapObject> native_context = instance_cache_.native_context();
        V<String> result_value = CallBuiltinThroughJumptable<
            BuiltinCallDescriptor::StringAdd_CheckNone>(
            decoder, V<Context>::Cast(native_context),
            {head_string, tail_string});
        result = __ AnnotateWasmType(result_value, kWasmRefExternString);
        decoder->detected_->add_imported_strings();
        break;
      }
      case WKI::kStringEquals: {
        // Using nullable type guards here because this instruction needs to
        // handle {null} without trapping.
        static constexpr bool kNullSucceeds = true;
        V<String> a_string = ExternRefToString(args[0], kNullSucceeds);
        V<String> b_string = ExternRefToString(args[1], kNullSucceeds);
        result = StringEqImpl(decoder, a_string, b_string, kWasmExternRef,
                              kWasmExternRef);
        decoder->detected_->add_imported_strings();
        break;
      }
      case WKI::kStringFromCharCode: {
        V<Word32> capped = __ Word32BitwiseAnd(args[0].op, 0xFFFF);
        V<String> result_value = CallBuiltinThroughJumptable<
            BuiltinCallDescriptor::WasmStringFromCodePoint>(decoder, {capped});
        result = __ AnnotateWasmType(result_value, kWasmRefExternString);
        decoder->detected_->add_imported_strings();
        break;
      }
      case WKI::kStringFromCodePoint: {
        V<String> result_value = CallBuiltinThroughJumptable<
            BuiltinCallDescriptor::WasmStringFromCodePoint>(decoder,
                                                            {args[0].op});
        result = __ AnnotateWasmType(result_value, kWasmRefExternString);
        decoder->detected_->add_imported_strings();
        break;
      }
      case WKI::kStringFromWtf16Array: {
        V<String> result_value = CallBuiltinThroughJumptable<
            BuiltinCallDescriptor::WasmStringNewWtf16Array>(
            decoder,
            {V<WasmArray>::Cast(NullCheck(args[0])), args[1].op, args[2].op});
        result = __ AnnotateWasmType(result_value, kWasmRefExternString);
        decoder->detected_->add_imported_strings();
        break;
      }
      case WKI::kStringFromUtf8Array:
        result = StringNewWtf8ArrayImpl(
            decoder, unibrow::Utf8Variant::kLossyUtf8, args[0], args[1],
            args[2], kWasmRefExternString);
        decoder->detected_->add_imported_strings();
        break;
      case WKI::kStringIntoUtf8Array: {
        V<String> string = ExternRefToString(args[0]);
        result = StringEncodeWtf8ArrayImpl(
            decoder, unibrow::Utf8Variant::kLossyUtf8, string,
            V<WasmArray>::Cast(NullCheck(args[1])), args[2].op);
        decoder->detected_->add_imported_strings();
        break;
      }
      case WKI::kStringToUtf8Array: {
        V<String> string = ExternRefToString(args[0]);
        V<WasmArray> result_value = CallBuiltinThroughJumptable<
            BuiltinCallDescriptor::WasmStringToUtf8Array>(decoder, {string});
        result = __ AnnotateWasmType(result_value, returns[0].type);
        decoder->detected_->add_imported_strings();
        break;
      }
      case WKI::kStringLength: {
        V<Object> string = ExternRefToString(args[0]);
        result = __ template LoadField<Word32>(
            string, compiler::AccessBuilder::ForStringLength());
        decoder->detected_->add_imported_strings();
        break;
      }
      case WKI::kStringMeasureUtf8: {
        V<String> string = ExternRefToString(args[0]);
        result = StringMeasureWtf8Impl(
            decoder, unibrow::Utf8Variant::kLossyUtf8, string);
        decoder->detected_->add_imported_strings();
        break;
      }
      case WKI::kStringSubstring: {
        V<String> string = ExternRefToString(args[0]);
        V<String> view = __ StringAsWtf16(string);
        // TODO(12868): Consider annotating {view}'s type when the typing story
        //              for string views has been settled.
        V<String> result_value = CallBuiltinThroughJumptable<
            BuiltinCallDescriptor::WasmStringViewWtf16Slice>(
            decoder, {view, args[1].op, args[2].op});
        result = __ AnnotateWasmType(result_value, kWasmRefExternString);
        decoder->detected_->add_imported_strings();
        break;
      }
      case WKI::kStringToWtf16Array: {
        V<String> string = ExternRefToString(args[0]);
        result = CallBuiltinThroughJumptable<
            BuiltinCallDescriptor::WasmStringEncodeWtf16Array>(
            decoder,
            {string, V<WasmArray>::Cast(NullCheck(args[1])), args[2].op});
        decoder->detected_->add_imported_strings();
        break;
      }
 
      // Other string-related imports.
      case WKI::kDoubleToString: {
        BuildModifyThreadInWasmFlag(decoder->zone(), false);
        V<String> result_value = CallBuiltinThroughJumptable<
            BuiltinCallDescriptor::WasmFloat64ToString>(decoder, {args[0].op});
        result = AnnotateAsString(result_value, returns[0].type);
        BuildModifyThreadInWasmFlag(decoder->zone(), true);
        decoder->detected_->Add(
            returns[0].type.is_reference_to(wasm::HeapType::kString)
                ? WasmDetectedFeature::stringref
                : WasmDetectedFeature::imported_strings);
        break;
      }
      case WKI::kIntToString: {
        BuildModifyThreadInWasmFlag(decoder->zone(), false);
        V<String> result_value =
            CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmIntToString>(
                decoder, {args[0].op, args[1].op});
        result = AnnotateAsString(result_value, returns[0].type);
        BuildModifyThreadInWasmFlag(decoder->zone(), true);
        decoder->detected_->Add(
            returns[0].type.is_reference_to(wasm::HeapType::kString)
                ? WasmDetectedFeature::stringref
                : WasmDetectedFeature::imported_strings);
        break;
      }
      case WKI::kParseFloat: {
        if (args[0].type.is_nullable()) {
          Label<Float64> done(&asm_);
          GOTO_IF(__ IsNull(args[0].op, args[0].type), done,
                  __ Float64Constant(std::numeric_limits<double>::quiet_NaN()));
 
          BuildModifyThreadInWasmFlag(decoder->zone(), false);
          V<Float64> not_null_res = CallBuiltinThroughJumptable<
              BuiltinCallDescriptor::WasmStringToDouble>(decoder, {args[0].op});
          BuildModifyThreadInWasmFlag(decoder->zone(), true);
          GOTO(done, not_null_res);
 
          BIND(done, result_f64);
          result = result_f64;
        } else {
          BuildModifyThreadInWasmFlag(decoder->zone(), false);
          result = CallBuiltinThroughJumptable<
              BuiltinCallDescriptor::WasmStringToDouble>(decoder, {args[0].op});
          BuildModifyThreadInWasmFlag(decoder->zone(), true);
        }
        decoder->detected_->add_stringref();
        break;
      }
      case WKI::kStringIndexOf: {
        V<String> string = args[0].op;
        V<String> search = args[1].op;
        V<Word32> start = args[2].op;
 
        // If string is null, throw.
        if (args[0].type.is_nullable()) {
          IF (__ IsNull(string, args[0].type)) {
            CallBuiltinThroughJumptable<
                BuiltinCallDescriptor::ThrowIndexOfCalledOnNull>(decoder, {});
            __ Unreachable();
          }
        }
 
        // If search is null, replace it with "null".
        if (args[1].type.is_nullable()) {
          Label<String> search_done_label(&asm_);
          GOTO_IF_NOT(__ IsNull(search, args[1].type), search_done_label,
                      search);
          GOTO(search_done_label, LOAD_ROOT(null_string));
          BIND(search_done_label, search_value);
          search = search_value;
        }
 
        result = GetStringIndexOf(decoder, string, search, start);
        decoder->detected_->add_stringref();
        break;
      }
      case WKI::kStringIndexOfImported: {
        // As the `string` and `search` parameters are externrefs, we have to
        // make sure they are strings. To enforce this, we inline only if a
        // (successful) `"js-string":"cast"` was performed before.
        if (!(IsExplicitStringCast(args[0]) && IsExplicitStringCast(args[1]))) {
          return false;
        }
        V<String> string = args[0].op;
        V<String> search = args[1].op;
        V<Word32> start = args[2].op;
 
        result = GetStringIndexOf(decoder, string, search, start);
        decoder->detected_->add_imported_strings();
        break;
      }
      case WKI::kStringToLocaleLowerCaseStringref:
        // TODO(14108): Implement.
        return false;
      case WKI::kStringToLowerCaseStringref: {
#if V8_INTL_SUPPORT
        V<String> string = args[0].op;
        if (args[0].type.is_nullable()) {
          IF (__ IsNull(string, args[0].type)) {
            CallBuiltinThroughJumptable<
                BuiltinCallDescriptor::ThrowToLowerCaseCalledOnNull>(decoder,
                                                                     {});
            __ Unreachable();
          }
        }
        V<String> result_value = CallStringToLowercase(decoder, string);
        result = __ AnnotateWasmType(result_value, kWasmRefString);
        decoder->detected_->add_stringref();
        break;
#else
        return false;
#endif
      }
      case WKI::kStringToLowerCaseImported: {
        // We have to make sure that the externref `string` parameter is a
        // string. To enforce this, we inline only if a (successful)
        // `"js-string":"cast"` was performed before.
#if V8_INTL_SUPPORT
        if (!IsExplicitStringCast(args[0])) {
          return false;
        }
        V<String> string = args[0].op;
        V<String> result_value = CallStringToLowercase(decoder, string);
        result = __ AnnotateWasmType(result_value, kWasmRefExternString);
        decoder->detected_->add_imported_strings();
        break;
#else
        return false;
#endif
      }
 
      // DataView related imports.
      // Note that we don't support DataView imports for resizable ArrayBuffers.
      case WKI::kDataViewGetBigInt64: {
        result = DataViewGetter(decoder, args, DataViewOp::kGetBigInt64);
        break;
      }
      case WKI::kDataViewGetBigUint64:
        result = DataViewGetter(decoder, args, DataViewOp::kGetBigUint64);
        break;
      case WKI::kDataViewGetFloat32:
        result = DataViewGetter(decoder, args, DataViewOp::kGetFloat32);
        break;
      case WKI::kDataViewGetFloat64:
        result = DataViewGetter(decoder, args, DataViewOp::kGetFloat64);
        break;
      case WKI::kDataViewGetInt8:
        result = DataViewGetter(decoder, args, DataViewOp::kGetInt8);
        break;
      case WKI::kDataViewGetInt16:
        result = DataViewGetter(decoder, args, DataViewOp::kGetInt16);
        break;
      case WKI::kDataViewGetInt32:
        result = DataViewGetter(decoder, args, DataViewOp::kGetInt32);
        break;
      case WKI::kDataViewGetUint8:
        result = DataViewGetter(decoder, args, DataViewOp::kGetUint8);
        break;
      case WKI::kDataViewGetUint16:
        result = DataViewGetter(decoder, args, DataViewOp::kGetUint16);
        break;
      case WKI::kDataViewGetUint32:
        result = DataViewGetter(decoder, args, DataViewOp::kGetUint32);
        break;
      case WKI::kDataViewSetBigInt64:
        DataViewSetter(decoder, args, DataViewOp::kSetBigInt64);
        break;
      case WKI::kDataViewSetBigUint64:
        DataViewSetter(decoder, args, DataViewOp::kSetBigUint64);
        break;
      case WKI::kDataViewSetFloat32:
        DataViewSetter(decoder, args, DataViewOp::kSetFloat32);
        break;
      case WKI::kDataViewSetFloat64:
        DataViewSetter(decoder, args, DataViewOp::kSetFloat64);
        break;
      case WKI::kDataViewSetInt8:
        DataViewSetter(decoder, args, DataViewOp::kSetInt8);
        break;
      case WKI::kDataViewSetInt16:
        DataViewSetter(decoder, args, DataViewOp::kSetInt16);
        break;
      case WKI::kDataViewSetInt32:
        DataViewSetter(decoder, args, DataViewOp::kSetInt32);
        break;
      case WKI::kDataViewSetUint8:
        DataViewSetter(decoder, args, DataViewOp::kSetUint8);
        break;
      case WKI::kDataViewSetUint16:
        DataViewSetter(decoder, args, DataViewOp::kSetUint16);
        break;
      case WKI::kDataViewSetUint32:
        DataViewSetter(decoder, args, DataViewOp::kSetUint32);
        break;
      case WKI::kDataViewByteLength: {
        V<Object> dataview = args[0].op;
        V<WordPtr> view_byte_length =
            GetDataViewByteLength(decoder, dataview, DataViewOp::kByteLength);
        if constexpr (Is64()) {
          result =
              __ ChangeInt64ToFloat64(__ ChangeIntPtrToInt64(view_byte_length));
        } else {
          result = __ ChangeInt32ToFloat64(
              __ TruncateWordPtrToWord32(view_byte_length));
        }
        break;
      }
 
        // Math functions.
      case WKI::kMathF64Acos:
        result = __ Float64Acos(args[0].op);
        break;
      case WKI::kMathF64Asin:
        result = __ Float64Asin(args[0].op);
        break;
      case WKI::kMathF64Atan:
        result = __ Float64Atan(args[0].op);
        break;
      case WKI::kMathF64Atan2:
        result = __ Float64Atan2(args[0].op, args[1].op);
        break;
      case WKI::kMathF64Cos:
        result = __ Float64Cos(args[0].op);
        break;
      case WKI::kMathF64Sin:
        result = __ Float64Sin(args[0].op);
        break;
      case WKI::kMathF64Tan:
        result = __ Float64Tan(args[0].op);
        break;
      case WKI::kMathF64Exp:
        result = __ Float64Exp(args[0].op);
        break;
      case WKI::kMathF64Log:
        result = __ Float64Log(args[0].op);
        break;
      case WKI::kMathF64Pow:
        result = __ Float64Power(args[0].op, args[1].op);
        break;
      case WKI::kMathF64Sqrt:
        result = __ Float64Sqrt(args[0].op);
        break;
 
        // Fast API calls.
      case WKI::kFastAPICall: {
        WellKnown_FastApi(decoder, imm, args, returns);
        result = returns[0].op;
        break;
      }
    }
    if (v8_flags.trace_wasm_inlining) {
      PrintF("[function %d: call to %d is well-known %s]\n", func_index_, index,
             WellKnownImportName(imported_op));
    }
    if (!*assumptions_) *assumptions_ = std::make_unique<AssumptionsJournal>();
    (*assumptions_)->RecordAssumption(index, imported_op);
    returns[0].op = result;
    return true;
  }
 
  void CallDirect(FullDecoder* decoder, const CallFunctionImmediate& imm,
                  const Value args[], Value returns[]) {
    feedback_slot_++;
    if (imm.index < decoder->module_->num_imported_functions) {
      if (HandleWellKnownImport(decoder, imm, args, returns)) {
        return;
      }
      auto [target, implicit_arg] =
          BuildImportedFunctionTargetAndImplicitArg(decoder, imm.index);
      BuildWasmCall(decoder, imm.sig, target, implicit_arg, args, returns,
                    compiler::kWasmIndirectFunction);
    } else {
      // Locally defined function.
      if (should_inline(decoder, feedback_slot_,
                        decoder->module_->functions[imm.index].code.length())) {
        if (v8_flags.trace_wasm_inlining) {
          PrintF("[function %d%s: inlining direct call #%d to function %d]\n",
                 func_index_, mode_ == kRegular ? "" : " (inlined)",
                 feedback_slot_, imm.index);
        }
        InlineWasmCall(decoder, imm.index, imm.sig, 0, false, args, returns);
      } else {
        V<WordPtr> callee =
            __ RelocatableConstant(imm.index, RelocInfo::WASM_CALL);
        BuildWasmCall(decoder, imm.sig, callee,
                      trusted_instance_data(
                          decoder->module_->function_is_shared(imm.index)),
                      args, returns);
      }
    }
  }
 
  void ReturnCall(FullDecoder* decoder, const CallFunctionImmediate& imm,
                  const Value args[]) {
    feedback_slot_++;
    if (imm.index < decoder->module_->num_imported_functions) {
      auto [target, implicit_arg] =
          BuildImportedFunctionTargetAndImplicitArg(decoder, imm.index);
      BuildWasmMaybeReturnCall(decoder, imm.sig, target, implicit_arg, args,
                               compiler::kWasmIndirectFunction);
    } else {
      // Locally defined function.
      if (should_inline(decoder, feedback_slot_,
                        decoder->module_->functions[imm.index].code.length())) {
        if (v8_flags.trace_wasm_inlining) {
          PrintF(
              "[function %d%s: inlining direct tail call #%d to function %d]\n",
              func_index_, mode_ == kRegular ? "" : " (inlined)",
              feedback_slot_, imm.index);
        }
        InlineWasmCall(decoder, imm.index, imm.sig, 0, true, args, nullptr);
      } else {
        BuildWasmMaybeReturnCall(
            decoder, imm.sig,
            __ RelocatableConstant(imm.index, RelocInfo::WASM_CALL),
            trusted_instance_data(
                decoder->module_->function_is_shared(imm.index)),
            args);
      }
    }
  }
 
  void CallIndirect(FullDecoder* decoder, const Value& index,
                    const CallIndirectImmediate& imm, const Value args[],
                    Value returns[]) {
    if (v8_flags.wasm_inlining_call_indirect) {
      CHECK(v8_flags.wasm_inlining);
      feedback_slot_++;
      // In case of being unreachable, skip it because it tries to access nodes
      // which might be non-existent (OpIndex::Invalid()) in unreachable code.
      if (__ generating_unreachable_operations()) return;
 
      if (should_inline(decoder, feedback_slot_,
                        std::numeric_limits<int>::max())) {
        V<WordPtr> index_wordptr = TableAddressToUintPtrOrOOBTrap(
            imm.table_imm.table->address_type, index.op);
 
        DCHECK(!shared_);
        // Load the instance here even though it's only used below, in the hope
        // that load elimination can use it when fetching the target next.
        V<WasmTrustedInstanceData> instance = trusted_instance_data(kNotShared);
 
        // We are only interested in the target here for comparison against
        // the inlined call target below.
        // In particular, we don't need a dynamic type or null check: If the
        // actual call target (at runtime) is equal to the inlined call target,
        // we know already from the static check on the inlinee (see below) that
        // the inlined code has the right signature.
        constexpr bool kNeedsTypeOrNullCheck = false;
        auto [target, implicit_arg] = BuildIndirectCallTargetAndImplicitArg(
            decoder, index_wordptr, imm, kNeedsTypeOrNullCheck);
 
        size_t return_count = imm.sig->return_count();
        base::Vector<InliningTree*> feedback_cases =
            inlining_decisions_->function_calls()[feedback_slot_];
        std::vector<base::SmallVector<OpIndex, 2>> case_returns(return_count);
        // The slow path is the non-inlined generic `call_indirect`,
        // or a deopt node if that is enabled.
        constexpr int kSlowpathCase = 1;
        base::SmallVector<TSBlock*, wasm::kMaxPolymorphism + kSlowpathCase>
            case_blocks;
        for (size_t i = 0; i < feedback_cases.size() + kSlowpathCase; i++) {
          case_blocks.push_back(__ NewBlock());
        }
        // Block for the slowpath, i.e., the not-inlined call or deopt.
        TSBlock* no_inline_block = case_blocks.back();
        // Block for merging results after the inlined code.
        TSBlock* merge = __ NewBlock();
 
        // Always create a frame state, but rely on DCE to remove it in case we
        // end up not using deopts. This allows us to share the frame state
        // between a deopt due to wrong instance and deopt due to wrong target.
        V<FrameState> frame_state =
            CreateFrameState(decoder, imm.sig, &index, args);
        bool use_deopt_slowpath = deopts_enabled();
        DCHECK_IMPLIES(use_deopt_slowpath, frame_state.valid());
        if (use_deopt_slowpath &&
            inlining_decisions_->has_non_inlineable_targets()[feedback_slot_]) {
          if (v8_flags.trace_wasm_inlining) {
            PrintF(
                "[function %d%s: Not emitting deopt slow-path for "
                "call_indirect #%d as feedback contains non-inlineable "
                "targets]\n",
                func_index_, mode_ == kRegular ? "" : " (inlined)",
                feedback_slot_);
          }
          use_deopt_slowpath = false;
        }
 
        // Wasm functions are semantically closures over the instance, but
        // when we inline a target in the following, we implicitly assume the
        // inlinee instance is the same as the caller's instance.
        // Directly jump to the non-inlined slowpath if that's violated.
        // Note that for `call_ref` this isn't necessary, because the funcref
        // equality check already captures both code and instance equality.
        constexpr BranchHint kUnlikelyCrossInstanceCall = BranchHint::kTrue;
        // Note that the `implicit_arg` can never be a `WasmImportData`,
        // since we don't inline imported functions right now.
        __ Branch({__ TaggedEqual(implicit_arg, instance),
                   kUnlikelyCrossInstanceCall},
                  case_blocks[0], no_inline_block);
 
        for (size_t i = 0; i < feedback_cases.size(); i++) {
          __ Bind(case_blocks[i]);
          InliningTree* tree = feedback_cases[i];
          if (!tree || !tree->is_inlined()) {
            // Fall through to the next case.
            __ Goto(case_blocks[i + 1]);
            // Do not use the deopt slowpath if we decided to not inline (at
            // least) one call target.
            // Otherwise, this could lead to a deopt loop.
            use_deopt_slowpath = false;
            continue;
          }
          uint32_t inlined_index = tree->function_index();
          // Ensure that we only inline if the inlinee's signature is compatible
          // with the call_indirect. In other words, perform the type check that
          // would normally be done dynamically (see above
          // `BuildIndirectCallTargetAndImplicitArg`) statically on the inlined
          // target. This can fail, e.g., because the mapping of feedback back
          // to function indices may produce spurious targets, or because the
          // feedback in the JS heap has been corrupted by a vulnerability.
          if (!InlineTargetIsTypeCompatible(
                  decoder->module_, imm.sig,
                  decoder->module_->functions[inlined_index].sig)) {
            __ Goto(case_blocks[i + 1]);
            continue;
          }
 
          V<Word32> inlined_target =
              __ RelocatableWasmIndirectCallTarget(inlined_index);
 
          bool is_last_feedback_case = (i == feedback_cases.size() - 1);
          if (use_deopt_slowpath && is_last_feedback_case) {
            DeoptIfNot(decoder, __ Word32Equal(target, inlined_target),
                       frame_state);
            } else {
            TSBlock* inline_block = __ NewBlock();
            BranchHint hint =
                is_last_feedback_case ? BranchHint::kTrue : BranchHint::kNone;
            __ Branch({__ Word32Equal(target, inlined_target), hint},
                      inline_block, case_blocks[i + 1]);
            __ Bind(inline_block);
          }
 
          SmallZoneVector<Value, 4> direct_returns(return_count,
                                                   decoder->zone_);
          if (v8_flags.trace_wasm_inlining) {
            PrintF(
                "[function %d%s: Speculatively inlining call_indirect #%d, "
                "case #%zu, to function %d]\n",
                func_index_, mode_ == kRegular ? "" : " (inlined)",
                feedback_slot_, i, inlined_index);
          }
          InlineWasmCall(decoder, inlined_index, imm.sig,
                         static_cast<uint32_t>(i), false, args,
                         direct_returns.data());
 
          if (__ current_block() != nullptr) {
            // Only add phi inputs and a Goto to {merge} if the current_block is
            // not nullptr. If the current_block is nullptr, it means that the
            // inlined body unconditionally exits early (likely an unconditional
            // trap or throw).
            for (size_t ret = 0; ret < direct_returns.size(); ret++) {
              case_returns[ret].push_back(direct_returns[ret].op);
            }
            __ Goto(merge);
          }
        }
 
        __ Bind(no_inline_block);
        if (use_deopt_slowpath) {
          // We need this unconditional deopt only for the "instance check",
          // as the last "target check" already uses a `DeoptIfNot` node.
          Deopt(decoder, frame_state);
        } else {
          auto [call_target, call_implicit_arg] =
              BuildIndirectCallTargetAndImplicitArg(decoder, index_wordptr,
                                                    imm);
          SmallZoneVector<Value, 4> indirect_returns(return_count,
                                                     decoder->zone_);
          BuildWasmCall(decoder, imm.sig, call_target, call_implicit_arg, args,
                        indirect_returns.data(),
                        compiler::kWasmIndirectFunction);
          for (size_t ret = 0; ret < indirect_returns.size(); ret++) {
            case_returns[ret].push_back(indirect_returns[ret].op);
          }
          __ Goto(merge);
        }
 
        __ Bind(merge);
        for (size_t i = 0; i < case_returns.size(); i++) {
          returns[i].op = __ Phi(base::VectorOf(case_returns[i]),
                                 RepresentationFor(imm.sig->GetReturn(i)));
        }
 
        return;
      }  // should_inline
    }    // v8_flags.wasm_inlining_call_indirect
 
    // Didn't inline.
    V<WordPtr> index_wordptr = TableAddressToUintPtrOrOOBTrap(
        imm.table_imm.table->address_type, index.op);
    auto [target, implicit_arg] =
        BuildIndirectCallTargetAndImplicitArg(decoder, index_wordptr, imm);
    BuildWasmCall(decoder, imm.sig, target, implicit_arg, args, returns,
                  compiler::kWasmIndirectFunction);
  }
 
  void ReturnCallIndirect(FullDecoder* decoder, const Value& index,
                          const CallIndirectImmediate& imm,
                          const Value args[]) {
    if (v8_flags.wasm_inlining_call_indirect) {
      CHECK(v8_flags.wasm_inlining);
      feedback_slot_++;
 
      if (should_inline(decoder, feedback_slot_,
                        std::numeric_limits<int>::max())) {
        V<WordPtr> index_wordptr = TableAddressToUintPtrOrOOBTrap(
            imm.table_imm.table->address_type, index.op);
 
        DCHECK(!shared_);
        // Load the instance here even though it's only used below, in the hope
        // that load elimination can use it when fetching the target next.
        V<WasmTrustedInstanceData> instance = trusted_instance_data(kNotShared);
 
        // We are only interested in the target here for comparison against
        // the inlined call target below.
        // In particular, we don't need a dynamic type or null check: If the
        // actual call target (at runtime) is equal to the inlined call target,
        // we know already from the static check on the inlinee (see below) that
        // the inlined code has the right signature.
        constexpr bool kNeedsTypeOrNullCheck = false;
        auto [target, implicit_arg] = BuildIndirectCallTargetAndImplicitArg(
            decoder, index_wordptr, imm, kNeedsTypeOrNullCheck);
 
        base::Vector<InliningTree*> feedback_cases =
            inlining_decisions_->function_calls()[feedback_slot_];
        constexpr int kSlowpathCase = 1;
        base::SmallVector<TSBlock*, wasm::kMaxPolymorphism + kSlowpathCase>
            case_blocks;
        for (size_t i = 0; i < feedback_cases.size() + kSlowpathCase; i++) {
          case_blocks.push_back(__ NewBlock());
        }
        // Block for the slowpath, i.e., the not-inlined call.
        TSBlock* no_inline_block = case_blocks.back();
 
        // Wasm functions are semantically closures over the instance, but
        // when we inline a target in the following, we implicitly assume the
        // inlinee instance is the same as the caller's instance.
        // Directly jump to the non-inlined slowpath if that's violated.
        // Note that for `call_ref` this isn't necessary, because the funcref
        // equality check already captures both code and instance equality.
        constexpr BranchHint kUnlikelyCrossInstanceCall = BranchHint::kTrue;
        // Note that the `implicit_arg` can never be a `WasmImportData`,
        // since we don't inline imported functions right now.
        __ Branch({__ TaggedEqual(implicit_arg, instance),
                   kUnlikelyCrossInstanceCall},
                  case_blocks[0], no_inline_block);
 
        for (size_t i = 0; i < feedback_cases.size(); i++) {
          __ Bind(case_blocks[i]);
          InliningTree* tree = feedback_cases[i];
          if (!tree || !tree->is_inlined()) {
            // Fall through to the next case.
            __ Goto(case_blocks[i + 1]);
            continue;
          }
          uint32_t inlined_index = tree->function_index();
          // Ensure that we only inline if the inlinee's signature is compatible
          // with the call_indirect. In other words, perform the type check that
          // would normally be done dynamically (see above
          // `BuildIndirectCallTargetAndImplicitArg`) statically on the inlined
          // target. This can fail, e.g., because the mapping of feedback back
          // to function indices may produce spurious targets, or because the
          // feedback in the JS heap has been corrupted by a vulnerability.
          if (!InlineTargetIsTypeCompatible(
                  decoder->module_, imm.sig,
                  decoder->module_->functions[inlined_index].sig)) {
            __ Goto(case_blocks[i + 1]);
            continue;
          }
 
          V<Word32> inlined_target =
              __ RelocatableWasmIndirectCallTarget(inlined_index);
 
          TSBlock* inline_block = __ NewBlock();
          bool is_last_case = (i == feedback_cases.size() - 1);
          BranchHint hint =
              is_last_case ? BranchHint::kTrue : BranchHint::kNone;
          __ Branch({__ Word32Equal(target, inlined_target), hint},
                    inline_block, case_blocks[i + 1]);
          __ Bind(inline_block);
          if (v8_flags.trace_wasm_inlining) {
            PrintF(
                "[function %d%s: Speculatively inlining return_call_indirect "
                "#%d, case #%zu, to function %d]\n",
                func_index_, mode_ == kRegular ? "" : " (inlined)",
                feedback_slot_, i, inlined_index);
          }
          InlineWasmCall(decoder, inlined_index, imm.sig,
                         static_cast<uint32_t>(i), true, args, nullptr);
 
          // An inlined tail call should still terminate execution.
          DCHECK_NULL(__ current_block());
        }
 
        __ Bind(no_inline_block);
      }  // should_inline
    }    // v8_flags.wasm_inlining_call_indirect
 
    // Didn't inline.
    V<WordPtr> index_wordptr = TableAddressToUintPtrOrOOBTrap(
        imm.table_imm.table->address_type, index.op);
    auto [target, implicit_arg] =
        BuildIndirectCallTargetAndImplicitArg(decoder, index_wordptr, imm);
    BuildWasmMaybeReturnCall(decoder, imm.sig, target, implicit_arg, args,
                             compiler::kWasmIndirectFunction);
  }
 
  void CallRef(FullDecoder* decoder, const Value& func_ref,
               const FunctionSig* sig, const Value args[], Value returns[]) {
    // TODO(14108): As the slot needs to be aligned with Liftoff, ideally the
    // stack slot index would be provided by the decoder and passed to both
    // Liftoff and Turbofan.
    feedback_slot_++;
    // In case of being unreachable, skip it because it tries to access nodes
    // which might be non-existent (OpIndex::Invalid()) in unreachable code.
    if (__ generating_unreachable_operations()) return;
 
    if (should_inline(decoder, feedback_slot_,
                      std::numeric_limits<int>::max())) {
      DCHECK(!shared_);
      V<FixedArray> func_refs = LOAD_IMMUTABLE_INSTANCE_FIELD(
          trusted_instance_data(kNotShared), FuncRefs,
          MemoryRepresentation::TaggedPointer());
 
      size_t return_count = sig->return_count();
      base::Vector<InliningTree*> feedback_cases =
          inlining_decisions_->function_calls()[feedback_slot_];
      std::vector<base::SmallVector<OpIndex, 2>> case_returns(return_count);
      // The slow path is the non-inlined generic `call_ref`,
      // or a deopt node if that is enabled.
      constexpr int kSlowpathCase = 1;
      base::SmallVector<TSBlock*, wasm::kMaxPolymorphism + kSlowpathCase>
          case_blocks;
      for (size_t i = 0; i < feedback_cases.size() + kSlowpathCase; i++) {
        case_blocks.push_back(__ NewBlock());
      }
      TSBlock* merge = __ NewBlock();
      __ Goto(case_blocks[0]);
 
      bool use_deopt_slowpath = deopts_enabled();
      for (size_t i = 0; i < feedback_cases.size(); i++) {
        __ Bind(case_blocks[i]);
        InliningTree* tree = feedback_cases[i];
        if (!tree || !tree->is_inlined()) {
          // Fall through to the next case.
          __ Goto(case_blocks[i + 1]);
          // Do not use the deopt slowpath if we decided to not inline (at
          // least) one call target. Otherwise, this could lead to a deopt loop.
          use_deopt_slowpath = false;
          continue;
        }
        uint32_t inlined_index = tree->function_index();
        DCHECK(!decoder->module_->function_is_shared(inlined_index));
        V<Object> inlined_func_ref =
            __ LoadFixedArrayElement(func_refs, inlined_index);
 
        bool is_last_feedback_case = (i == feedback_cases.size() - 1);
        if (use_deopt_slowpath && is_last_feedback_case) {
          if (inlining_decisions_
                  ->has_non_inlineable_targets()[feedback_slot_]) {
            if (v8_flags.trace_wasm_inlining) {
              PrintF(
                  "[function %d%s: Not emitting deopt slow-path for "
                  "call_ref #%d as feedback contains non-inlineable "
                  "targets]\n",
                  func_index_, mode_ == kRegular ? "" : " (inlined)",
                  feedback_slot_);
            }
            use_deopt_slowpath = false;
          }
        }
        bool emit_deopt = use_deopt_slowpath && is_last_feedback_case;
        if (emit_deopt) {
          V<FrameState> frame_state =
              CreateFrameState(decoder, sig, &func_ref, args);
          if (frame_state.valid()) {
            DeoptIfNot(decoder, __ TaggedEqual(func_ref.op, inlined_func_ref),
                       frame_state);
          } else {
            emit_deopt = false;
            use_deopt_slowpath = false;
          }
        }
        if (!emit_deopt) {
          TSBlock* inline_block = __ NewBlock();
          BranchHint hint =
              is_last_feedback_case ? BranchHint::kTrue : BranchHint::kNone;
          __ Branch({__ TaggedEqual(func_ref.op, inlined_func_ref), hint},
                    inline_block, case_blocks[i + 1]);
          __ Bind(inline_block);
        }
 
        SmallZoneVector<Value, 4> direct_returns(return_count, decoder->zone_);
        if (v8_flags.trace_wasm_inlining) {
          PrintF(
              "[function %d%s: Speculatively inlining call_ref #%d, case #%zu, "
              "to function %d]\n",
              func_index_, mode_ == kRegular ? "" : " (inlined)",
              feedback_slot_, i, inlined_index);
        }
        InlineWasmCall(decoder, inlined_index, sig, static_cast<uint32_t>(i),
                       false, args, direct_returns.data());
 
        if (__ current_block() != nullptr) {
          // Only add phi inputs and a Goto to {merge} if the current_block is
          // not nullptr. If the current_block is nullptr, it means that the
          // inlined body unconditionally exits early (likely an unconditional
          // trap or throw).
          for (size_t ret = 0; ret < direct_returns.size(); ret++) {
            case_returns[ret].push_back(direct_returns[ret].op);
          }
          __ Goto(merge);
        }
      }
 
      if (!use_deopt_slowpath) {
        TSBlock* no_inline_block = case_blocks.back();
        __ Bind(no_inline_block);
        auto [target, implicit_arg] =
            BuildFunctionReferenceTargetAndImplicitArg(func_ref.op,
                                                       func_ref.type);
        SmallZoneVector<Value, 4> ref_returns(return_count, decoder->zone_);
        BuildWasmCall(decoder, sig, target, implicit_arg, args,
                      ref_returns.data(), compiler::kWasmIndirectFunction);
        for (size_t ret = 0; ret < ref_returns.size(); ret++) {
          case_returns[ret].push_back(ref_returns[ret].op);
        }
        __ Goto(merge);
      }
 
      __ Bind(merge);
      for (size_t i = 0; i < case_returns.size(); i++) {
        returns[i].op = __ Phi(base::VectorOf(case_returns[i]),
                               RepresentationFor(sig->GetReturn(i)));
      }
    } else {
      auto [target, implicit_arg] = BuildFunctionReferenceTargetAndImplicitArg(
          func_ref.op, func_ref.type);
      BuildWasmCall(decoder, sig, target, implicit_arg, args, returns,
                    compiler::kWasmIndirectFunction);
    }
  }
 
  void ReturnCallRef(FullDecoder* decoder, const Value& func_ref,
                     const FunctionSig* sig, const Value args[]) {
    feedback_slot_++;
 
    if (should_inline(decoder, feedback_slot_,
                      std::numeric_limits<int>::max())) {
      DCHECK(!shared_);
      V<FixedArray> func_refs = LOAD_IMMUTABLE_INSTANCE_FIELD(
          trusted_instance_data(kNotShared), FuncRefs,
          MemoryRepresentation::TaggedPointer());
 
      base::Vector<InliningTree*> feedback_cases =
          inlining_decisions_->function_calls()[feedback_slot_];
      constexpr int kSlowpathCase = 1;
      base::SmallVector<TSBlock*, wasm::kMaxPolymorphism + kSlowpathCase>
          case_blocks;
 
      for (size_t i = 0; i < feedback_cases.size() + kSlowpathCase; i++) {
        case_blocks.push_back(__ NewBlock());
      }
      __ Goto(case_blocks[0]);
 
      for (size_t i = 0; i < feedback_cases.size(); i++) {
        __ Bind(case_blocks[i]);
        InliningTree* tree = feedback_cases[i];
        if (!tree || !tree->is_inlined()) {
          // Fall through to the next case.
          __ Goto(case_blocks[i + 1]);
          continue;
        }
        uint32_t inlined_index = tree->function_index();
        DCHECK(!decoder->module_->function_is_shared(inlined_index));
        V<Object> inlined_func_ref =
            __ LoadFixedArrayElement(func_refs, inlined_index);
 
        TSBlock* inline_block = __ NewBlock();
        bool is_last_case = (i == feedback_cases.size() - 1);
        BranchHint hint = is_last_case ? BranchHint::kTrue : BranchHint::kNone;
        __ Branch({__ TaggedEqual(func_ref.op, inlined_func_ref), hint},
                  inline_block, case_blocks[i + 1]);
        __ Bind(inline_block);
        if (v8_flags.trace_wasm_inlining) {
          PrintF(
              "[function %d%s: Speculatively inlining return_call_ref #%d, "
              "case #%zu, to function %d]\n",
              func_index_, mode_ == kRegular ? "" : " (inlined)",
              feedback_slot_, i, inlined_index);
        }
        InlineWasmCall(decoder, inlined_index, sig, static_cast<uint32_t>(i),
                       true, args, nullptr);
 
        // An inlined tail call should still terminate execution.
        DCHECK_NULL(__ current_block());
      }
 
      TSBlock* no_inline_block = case_blocks.back();
      __ Bind(no_inline_block);
    }
    auto [target, implicit_arg] =
        BuildFunctionReferenceTargetAndImplicitArg(func_ref.op, func_ref.type);
    BuildWasmMaybeReturnCall(decoder, sig, target, implicit_arg, args,
                             compiler::kWasmIndirectFunction);
  }
 
  void BrOnNull(FullDecoder* decoder, const Value& ref_object, uint32_t depth,
                bool pass_null_along_branch, Value* result_on_fallthrough) {
    result_on_fallthrough->op = ref_object.op;
    IF (UNLIKELY(__ IsNull(ref_object.op, ref_object.type))) {
      int drop_values = pass_null_along_branch ? 0 : 1;
      BrOrRet(decoder, depth, drop_values);
    }
  }
 
  void BrOnNonNull(FullDecoder* decoder, const Value& ref_object, Value* result,
                   uint32_t depth, bool /* drop_null_on_fallthrough */) {
    result->op = ref_object.op;
    IF_NOT (UNLIKELY(__ IsNull(ref_object.op, ref_object.type))) {
      BrOrRet(decoder, depth);
    }
  }
 
  void SimdOp(FullDecoder* decoder, WasmOpcode opcode, const Value* args,
              Value* result) {
    switch (opcode) {
#define HANDLE_BINARY_OPCODE(kind)                                            \
  case kExpr##kind:                                                           \
    result->op =                                                              \
        __ Simd128Binop(V<compiler::turboshaft::Simd128>::Cast(args[0].op),   \
                        V<compiler::turboshaft::Simd128>::Cast(args[1].op),   \
                        compiler::turboshaft::Simd128BinopOp::Kind::k##kind); \
    break;
      FOREACH_SIMD_128_BINARY_MANDATORY_OPCODE(HANDLE_BINARY_OPCODE)
#undef HANDLE_BINARY_OPCODE
#define HANDLE_F16X8_BIN_OPTIONAL_OPCODE(kind, extern_ref)                     \
  case kExprF16x8##kind:                                                       \
    if (SupportedOperations::float16()) {                                      \
      result->op = __ Simd128Binop(                                            \
          V<compiler::turboshaft::Simd128>::Cast(args[0].op),                  \
          V<compiler::turboshaft::Simd128>::Cast(args[1].op),                  \
          compiler::turboshaft::Simd128BinopOp::Kind::kF16x8##kind);           \
    } else {                                                                   \
      result->op = CallCStackSlotToStackSlot(args[0].op, args[1].op,           \
                                             ExternalReference::extern_ref(),  \
                                             MemoryRepresentation::Simd128()); \
    }                                                                          \
    break;
 
      HANDLE_F16X8_BIN_OPTIONAL_OPCODE(Add, wasm_f16x8_add)
      HANDLE_F16X8_BIN_OPTIONAL_OPCODE(Sub, wasm_f16x8_sub)
      HANDLE_F16X8_BIN_OPTIONAL_OPCODE(Mul, wasm_f16x8_mul)
      HANDLE_F16X8_BIN_OPTIONAL_OPCODE(Div, wasm_f16x8_div)
      HANDLE_F16X8_BIN_OPTIONAL_OPCODE(Min, wasm_f16x8_min)
      HANDLE_F16X8_BIN_OPTIONAL_OPCODE(Max, wasm_f16x8_max)
      HANDLE_F16X8_BIN_OPTIONAL_OPCODE(Pmin, wasm_f16x8_pmin)
      HANDLE_F16X8_BIN_OPTIONAL_OPCODE(Pmax, wasm_f16x8_pmax)
      HANDLE_F16X8_BIN_OPTIONAL_OPCODE(Eq, wasm_f16x8_eq)
      HANDLE_F16X8_BIN_OPTIONAL_OPCODE(Ne, wasm_f16x8_ne)
      HANDLE_F16X8_BIN_OPTIONAL_OPCODE(Lt, wasm_f16x8_lt)
      HANDLE_F16X8_BIN_OPTIONAL_OPCODE(Le, wasm_f16x8_le)
#undef HANDLE_F16X8_BIN_OPCODE
 
#define HANDLE_F16X8_INVERSE_COMPARISON(kind, ts_kind, extern_ref)             \
  case kExprF16x8##kind:                                                       \
    if (SupportedOperations::float16()) {                                      \
      result->op = __ Simd128Binop(                                            \
          V<compiler::turboshaft::Simd128>::Cast(args[1].op),                  \
          V<compiler::turboshaft::Simd128>::Cast(args[0].op),                  \
          compiler::turboshaft::Simd128BinopOp::Kind::kF16x8##ts_kind);        \
    } else {                                                                   \
      result->op = CallCStackSlotToStackSlot(args[1].op, args[0].op,           \
                                             ExternalReference::extern_ref(),  \
                                             MemoryRepresentation::Simd128()); \
    }                                                                          \
    break;
 
      HANDLE_F16X8_INVERSE_COMPARISON(Gt, Lt, wasm_f16x8_lt)
      HANDLE_F16X8_INVERSE_COMPARISON(Ge, Le, wasm_f16x8_le)
#undef HANDLE_F16X8_INVERSE_COMPARISON
 
#define HANDLE_INVERSE_COMPARISON(wasm_kind, ts_kind)            \
  case kExpr##wasm_kind:                                         \
    result->op = __ Simd128Binop(                                \
        V<compiler::turboshaft::Simd128>::Cast(args[1].op),      \
        V<compiler::turboshaft::Simd128>::Cast(args[0].op),      \
        compiler::turboshaft::Simd128BinopOp::Kind::k##ts_kind); \
    break;
 
      HANDLE_INVERSE_COMPARISON(I8x16LtS, I8x16GtS)
      HANDLE_INVERSE_COMPARISON(I8x16LtU, I8x16GtU)
      HANDLE_INVERSE_COMPARISON(I8x16LeS, I8x16GeS)
      HANDLE_INVERSE_COMPARISON(I8x16LeU, I8x16GeU)
 
      HANDLE_INVERSE_COMPARISON(I16x8LtS, I16x8GtS)
      HANDLE_INVERSE_COMPARISON(I16x8LtU, I16x8GtU)
      HANDLE_INVERSE_COMPARISON(I16x8LeS, I16x8GeS)
      HANDLE_INVERSE_COMPARISON(I16x8LeU, I16x8GeU)
 
      HANDLE_INVERSE_COMPARISON(I32x4LtS, I32x4GtS)
      HANDLE_INVERSE_COMPARISON(I32x4LtU, I32x4GtU)
      HANDLE_INVERSE_COMPARISON(I32x4LeS, I32x4GeS)
      HANDLE_INVERSE_COMPARISON(I32x4LeU, I32x4GeU)
 
      HANDLE_INVERSE_COMPARISON(I64x2LtS, I64x2GtS)
      HANDLE_INVERSE_COMPARISON(I64x2LeS, I64x2GeS)
 
      HANDLE_INVERSE_COMPARISON(F32x4Gt, F32x4Lt)
      HANDLE_INVERSE_COMPARISON(F32x4Ge, F32x4Le)
      HANDLE_INVERSE_COMPARISON(F64x2Gt, F64x2Lt)
      HANDLE_INVERSE_COMPARISON(F64x2Ge, F64x2Le)
 
#undef HANDLE_INVERSE_COMPARISON
 
#define HANDLE_UNARY_NON_OPTIONAL_OPCODE(kind)                                \
  case kExpr##kind:                                                           \
    result->op =                                                              \
        __ Simd128Unary(V<compiler::turboshaft::Simd128>::Cast(args[0].op),   \
                        compiler::turboshaft::Simd128UnaryOp::Kind::k##kind); \
    break;
      FOREACH_SIMD_128_UNARY_NON_OPTIONAL_OPCODE(
          HANDLE_UNARY_NON_OPTIONAL_OPCODE)
#undef HANDLE_UNARY_NON_OPTIONAL_OPCODE
 
#define HANDLE_UNARY_OPTIONAL_OPCODE(kind, feature, external_ref) \
  case kExpr##kind:                                               \
    if (SupportedOperations::feature()) {                         \
      result->op = __ Simd128Unary(                               \
          V<compiler::turboshaft::Simd128>::Cast(args[0].op),     \
          compiler::turboshaft::Simd128UnaryOp::Kind::k##kind);   \
    } else {                                                      \
      result->op = CallCStackSlotToStackSlot(                     \
          args[0].op, ExternalReference::external_ref(),          \
          MemoryRepresentation::Simd128());                       \
    }                                                             \
    break;
      HANDLE_UNARY_OPTIONAL_OPCODE(F16x8Abs, float16, wasm_f16x8_abs)
      HANDLE_UNARY_OPTIONAL_OPCODE(F16x8Neg, float16, wasm_f16x8_neg)
      HANDLE_UNARY_OPTIONAL_OPCODE(F16x8Sqrt, float16, wasm_f16x8_sqrt)
      HANDLE_UNARY_OPTIONAL_OPCODE(F16x8Ceil, float16, wasm_f16x8_ceil)
      HANDLE_UNARY_OPTIONAL_OPCODE(F16x8Floor, float16, wasm_f16x8_floor)
      HANDLE_UNARY_OPTIONAL_OPCODE(F16x8Trunc, float16, wasm_f16x8_trunc)
      HANDLE_UNARY_OPTIONAL_OPCODE(F16x8NearestInt, float16,
                                   wasm_f16x8_nearest_int)
      HANDLE_UNARY_OPTIONAL_OPCODE(I16x8SConvertF16x8, float16,
                                   wasm_i16x8_sconvert_f16x8)
      HANDLE_UNARY_OPTIONAL_OPCODE(I16x8UConvertF16x8, float16,
                                   wasm_i16x8_uconvert_f16x8)
      HANDLE_UNARY_OPTIONAL_OPCODE(F16x8SConvertI16x8, float16,
                                   wasm_f16x8_sconvert_i16x8)
      HANDLE_UNARY_OPTIONAL_OPCODE(F16x8UConvertI16x8, float16,
                                   wasm_f16x8_uconvert_i16x8)
      HANDLE_UNARY_OPTIONAL_OPCODE(F16x8DemoteF32x4Zero, float16,
                                   wasm_f16x8_demote_f32x4_zero)
      HANDLE_UNARY_OPTIONAL_OPCODE(F16x8DemoteF64x2Zero,
                                   float64_to_float16_raw_bits,
                                   wasm_f16x8_demote_f64x2_zero)
      HANDLE_UNARY_OPTIONAL_OPCODE(F32x4PromoteLowF16x8, float16,
                                   wasm_f32x4_promote_low_f16x8)
      HANDLE_UNARY_OPTIONAL_OPCODE(F32x4Ceil, float32_round_up, wasm_f32x4_ceil)
      HANDLE_UNARY_OPTIONAL_OPCODE(F32x4Floor, float32_round_down,
                                   wasm_f32x4_floor)
      HANDLE_UNARY_OPTIONAL_OPCODE(F32x4Trunc, float32_round_to_zero,
                                   wasm_f32x4_trunc)
      HANDLE_UNARY_OPTIONAL_OPCODE(F32x4NearestInt, float32_round_ties_even,
                                   wasm_f32x4_nearest_int)
      HANDLE_UNARY_OPTIONAL_OPCODE(F64x2Ceil, float64_round_up, wasm_f64x2_ceil)
      HANDLE_UNARY_OPTIONAL_OPCODE(F64x2Floor, float64_round_down,
                                   wasm_f64x2_floor)
      HANDLE_UNARY_OPTIONAL_OPCODE(F64x2Trunc, float64_round_to_zero,
                                   wasm_f64x2_trunc)
      HANDLE_UNARY_OPTIONAL_OPCODE(F64x2NearestInt, float64_round_ties_even,
                                   wasm_f64x2_nearest_int)
#undef HANDLE_UNARY_OPTIONAL_OPCODE
 
#define HANDLE_SHIFT_OPCODE(kind)                                             \
  case kExpr##kind:                                                           \
    result->op =                                                              \
        __ Simd128Shift(V<compiler::turboshaft::Simd128>::Cast(args[0].op),   \
                        V<Word32>::Cast(args[1].op),                          \
                        compiler::turboshaft::Simd128ShiftOp::Kind::k##kind); \
    break;
      FOREACH_SIMD_128_SHIFT_OPCODE(HANDLE_SHIFT_OPCODE)
#undef HANDLE_SHIFT_OPCODE
 
#define HANDLE_TEST_OPCODE(kind)                                            \
  case kExpr##kind:                                                         \
    result->op =                                                            \
        __ Simd128Test(V<compiler::turboshaft::Simd128>::Cast(args[0].op),  \
                       compiler::turboshaft::Simd128TestOp::Kind::k##kind); \
    break;
      FOREACH_SIMD_128_TEST_OPCODE(HANDLE_TEST_OPCODE)
#undef HANDLE_TEST_OPCODE
 
#define HANDLE_SPLAT_OPCODE(kind)                                             \
  case kExpr##kind##Splat:                                                    \
    result->op =                                                              \
        __ Simd128Splat(V<Any>::Cast(args[0].op),                             \
                        compiler::turboshaft::Simd128SplatOp::Kind::k##kind); \
    break;
      FOREACH_SIMD_128_SPLAT_MANDATORY_OPCODE(HANDLE_SPLAT_OPCODE)
#undef HANDLE_SPLAT_OPCODE
      case kExprF16x8Splat:
        if (SupportedOperations::float16()) {
          result->op = __ Simd128Splat(
              V<Any>::Cast(args[0].op),
              compiler::turboshaft::Simd128SplatOp::Kind::kF16x8);
        } else {
          auto f16 = CallCStackSlotToStackSlot(
              args[0].op, ExternalReference::wasm_float32_to_float16(),
              MemoryRepresentation::Float32(), MemoryRepresentation::Int16());
          result->op = __ Simd128Splat(
              V<Any>::Cast(f16),
              compiler::turboshaft::Simd128SplatOp::Kind::kI16x8);
        }
        break;
 
// Ternary mask operators put the mask as first input.
#define HANDLE_TERNARY_MASK_OPCODE(kind)                        \
  case kExpr##kind:                                             \
    result->op = __ Simd128Ternary(                             \
        V<compiler::turboshaft::Simd128>::Cast(args[2].op),     \
        V<compiler::turboshaft::Simd128>::Cast(args[0].op),     \
        V<compiler::turboshaft::Simd128>::Cast(args[1].op),     \
        compiler::turboshaft::Simd128TernaryOp::Kind::k##kind); \
    break;
      FOREACH_SIMD_128_TERNARY_MASK_OPCODE(HANDLE_TERNARY_MASK_OPCODE)
#undef HANDLE_TERNARY_MASK_OPCODE
 
#define HANDLE_TERNARY_OTHER_OPCODE(kind)                       \
  case kExpr##kind:                                             \
    result->op = __ Simd128Ternary(                             \
        V<compiler::turboshaft::Simd128>::Cast(args[0].op),     \
        V<compiler::turboshaft::Simd128>::Cast(args[1].op),     \
        V<compiler::turboshaft::Simd128>::Cast(args[2].op),     \
        compiler::turboshaft::Simd128TernaryOp::Kind::k##kind); \
    break;
      FOREACH_SIMD_128_TERNARY_OTHER_OPCODE(HANDLE_TERNARY_OTHER_OPCODE)
#undef HANDLE_TERNARY_OTHER_OPCODE
 
#define HANDLE_F16X8_TERN_OPCODE(kind, extern_ref)                          \
  case kExpr##kind:                                                         \
    if (SupportedOperations::float16()) {                                   \
      result->op = __ Simd128Ternary(                                       \
          V<compiler::turboshaft::Simd128>::Cast(args[0].op),               \
          V<compiler::turboshaft::Simd128>::Cast(args[1].op),               \
          V<compiler::turboshaft::Simd128>::Cast(args[2].op),               \
          compiler::turboshaft::Simd128TernaryOp::Kind::k##kind);           \
    } else {                                                                \
      result->op = CallCStackSlotToStackSlot(                               \
          ExternalReference::extern_ref(), MemoryRepresentation::Simd128(), \
          {{args[0].op, MemoryRepresentation::Simd128()},                   \
           {args[1].op, MemoryRepresentation::Simd128()},                   \
           {args[2].op, MemoryRepresentation::Simd128()}});                 \
    }                                                                       \
    break;
        HANDLE_F16X8_TERN_OPCODE(F16x8Qfma, wasm_f16x8_qfma)
        HANDLE_F16X8_TERN_OPCODE(F16x8Qfms, wasm_f16x8_qfms)
#undef HANDLE_F16X8_TERN_OPCODE
      default:
        UNREACHABLE();
    }
  }
 
  void SimdLaneOp(FullDecoder* decoder, WasmOpcode opcode,
                  const SimdLaneImmediate& imm,
                  base::Vector<const Value> inputs, Value* result) {
    using compiler::turboshaft::Simd128ExtractLaneOp;
    using compiler::turboshaft::Simd128ReplaceLaneOp;
    using Simd128 = compiler::turboshaft::Simd128;
    V<Simd128> input_val = V<Simd128>::Cast(inputs[0].op);
    switch (opcode) {
      case kExprI8x16ExtractLaneS:
        result->op = __ Simd128ExtractLane(
            input_val, Simd128ExtractLaneOp::Kind::kI8x16S, imm.lane);
        break;
      case kExprI8x16ExtractLaneU:
        result->op = __ Simd128ExtractLane(
            input_val, Simd128ExtractLaneOp::Kind::kI8x16U, imm.lane);
        break;
      case kExprI16x8ExtractLaneS:
        result->op = __ Simd128ExtractLane(
            input_val, Simd128ExtractLaneOp::Kind::kI16x8S, imm.lane);
        break;
      case kExprI16x8ExtractLaneU:
        result->op = __ Simd128ExtractLane(
            input_val, Simd128ExtractLaneOp::Kind::kI16x8U, imm.lane);
        break;
      case kExprI32x4ExtractLane:
        result->op = __ Simd128ExtractLane(
            input_val, Simd128ExtractLaneOp::Kind::kI32x4, imm.lane);
        break;
      case kExprI64x2ExtractLane:
        result->op = __ Simd128ExtractLane(
            input_val, Simd128ExtractLaneOp::Kind::kI64x2, imm.lane);
        break;
      case kExprF16x8ExtractLane:
        if (SupportedOperations::float16()) {
          result->op = __ Simd128ExtractLane(
              input_val, Simd128ExtractLaneOp::Kind::kF16x8, imm.lane);
        } else {
          auto f16 = __ Simd128ExtractLane(
              input_val, Simd128ExtractLaneOp::Kind::kI16x8S, imm.lane);
          result->op = CallCStackSlotToStackSlot(
              f16, ExternalReference::wasm_float16_to_float32(),
              MemoryRepresentation::Int16(), MemoryRepresentation::Float32());
        }
        break;
      case kExprF32x4ExtractLane:
        result->op = __ Simd128ExtractLane(
            input_val, Simd128ExtractLaneOp::Kind::kF32x4, imm.lane);
        break;
      case kExprF64x2ExtractLane:
        result->op = __ Simd128ExtractLane(
            input_val, Simd128ExtractLaneOp::Kind::kF64x2, imm.lane);
        break;
      case kExprI8x16ReplaceLane:
        result->op =
            __ Simd128ReplaceLane(input_val, V<Any>::Cast(inputs[1].op),
                                  Simd128ReplaceLaneOp::Kind::kI8x16, imm.lane);
        break;
      case kExprI16x8ReplaceLane:
        result->op =
            __ Simd128ReplaceLane(input_val, V<Simd128>::Cast(inputs[1].op),
                                  Simd128ReplaceLaneOp::Kind::kI16x8, imm.lane);
        break;
      case kExprI32x4ReplaceLane:
        result->op =
            __ Simd128ReplaceLane(input_val, V<Any>::Cast(inputs[1].op),
                                  Simd128ReplaceLaneOp::Kind::kI32x4, imm.lane);
        break;
      case kExprI64x2ReplaceLane:
        result->op =
            __ Simd128ReplaceLane(input_val, V<Any>::Cast(inputs[1].op),
                                  Simd128ReplaceLaneOp::Kind::kI64x2, imm.lane);
        break;
      case kExprF16x8ReplaceLane:
        if (SupportedOperations::float16()) {
          result->op = __ Simd128ReplaceLane(
              input_val, V<Any>::Cast(inputs[1].op),
              Simd128ReplaceLaneOp::Kind::kF16x8, imm.lane);
        } else {
          auto f16 = CallCStackSlotToStackSlot(
              inputs[1].op, ExternalReference::wasm_float32_to_float16(),
              MemoryRepresentation::Float32(), MemoryRepresentation::Int16());
          result->op = __ Simd128ReplaceLane(input_val, V<Any>::Cast(f16),
                                             Simd128ReplaceLaneOp::Kind::kI16x8,
                                             imm.lane);
        }
        break;
      case kExprF32x4ReplaceLane:
        result->op =
            __ Simd128ReplaceLane(input_val, V<Any>::Cast(inputs[1].op),
                                  Simd128ReplaceLaneOp::Kind::kF32x4, imm.lane);
        break;
      case kExprF64x2ReplaceLane:
        result->op =
            __ Simd128ReplaceLane(input_val, V<Any>::Cast(inputs[1].op),
                                  Simd128ReplaceLaneOp::Kind::kF64x2, imm.lane);
        break;
      default:
        UNREACHABLE();
    }
  }
 
  void Simd8x16ShuffleOp(FullDecoder* decoder, const Simd128Immediate& imm,
                         const Value& input0, const Value& input1,
                         Value* result) {
    result->op = __ Simd128Shuffle(
        V<compiler::turboshaft::Simd128>::Cast(input0.op),
        V<compiler::turboshaft::Simd128>::Cast(input1.op),
        compiler::turboshaft::Simd128ShuffleOp::Kind::kI8x16, imm.value);
  }
 
  void Try(FullDecoder* decoder, Control* block) {
    block->false_or_loop_or_catch_block = NewBlockWithPhis(decoder, nullptr);
    block->merge_block = NewBlockWithPhis(decoder, block->br_merge());
  }
 
  void Throw(FullDecoder* decoder, const TagIndexImmediate& imm,
             const Value arg_values[]) {
    size_t count = imm.tag->sig->parameter_count();
    SmallZoneVector<OpIndex, 16> values(count, decoder->zone_);
    for (size_t index = 0; index < count; index++) {
      values[index] = arg_values[index].op;
    }
 
    uint32_t encoded_size = WasmExceptionPackage::GetEncodedSize(imm.tag);
 
    V<FixedArray> values_array = CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::WasmAllocateFixedArray>(
        decoder, {__ IntPtrConstant(encoded_size)});
    uint32_t index = 0;
    const wasm::WasmTagSig* sig = imm.tag->sig;
 
    // Encode the exception values in {values_array}.
    for (size_t i = 0; i < count; i++) {
      OpIndex value = values[i];
      switch (sig->GetParam(i).kind()) {
        case kF32:
          value = __ BitcastFloat32ToWord32(value);
          [[fallthrough]];
        case kI32:
          BuildEncodeException32BitValue(values_array, index, value);
          // We need 2 Smis to encode a 32-bit value.
          index += 2;
          break;
        case kF64:
          value = __ BitcastFloat64ToWord64(value);
          [[fallthrough]];
        case kI64: {
          OpIndex upper_half =
              __ TruncateWord64ToWord32(__ Word64ShiftRightLogical(value, 32));
          BuildEncodeException32BitValue(values_array, index, upper_half);
          index += 2;
          OpIndex lower_half = __ TruncateWord64ToWord32(value);
          BuildEncodeException32BitValue(values_array, index, lower_half);
          index += 2;
          break;
        }
        case wasm::kRef:
        case wasm::kRefNull:
          __ StoreFixedArrayElement(values_array, index, value,
                                    compiler::kFullWriteBarrier);
          index++;
          break;
        case kS128: {
          using Simd128 = compiler::turboshaft::Simd128;
          V<Simd128> value_s128 = V<Simd128>::Cast(value);
          using Kind = compiler::turboshaft::Simd128ExtractLaneOp::Kind;
          BuildEncodeException32BitValue(values_array, index,
                                         V<Word32>::Cast(__ Simd128ExtractLane(
                                             value_s128, Kind::kI32x4, 0)));
          index += 2;
          BuildEncodeException32BitValue(values_array, index,
                                         V<Word32>::Cast(__ Simd128ExtractLane(
                                             value_s128, Kind::kI32x4, 1)));
          index += 2;
          BuildEncodeException32BitValue(values_array, index,
                                         V<Word32>::Cast(__ Simd128ExtractLane(
                                             value_s128, Kind::kI32x4, 2)));
          index += 2;
          BuildEncodeException32BitValue(values_array, index,
                                         V<Word32>::Cast(__ Simd128ExtractLane(
                                             value_s128, Kind::kI32x4, 3)));
          index += 2;
          break;
        }
        case kI8:
        case kI16:
        case kF16:
        case kVoid:
        case kTop:
        case kBottom:
          UNREACHABLE();
      }
    }
 
    // TODO(14616): Support shared tags.
    V<FixedArray> instance_tags =
        LOAD_IMMUTABLE_INSTANCE_FIELD(trusted_instance_data(false), TagsTable,
                                      MemoryRepresentation::TaggedPointer());
    auto tag = V<WasmTagObject>::Cast(
        __ LoadFixedArrayElement(instance_tags, imm.index));
 
    CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmThrow>(
        decoder, {tag, values_array}, CheckForException::kCatchInThisFrame);
    __ Unreachable();
  }
 
  void Rethrow(FullDecoder* decoder, Control* block) {
    CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmRethrow>(
        decoder, {block->exception}, CheckForException::kCatchInThisFrame);
    __ Unreachable();
  }
 
  void CatchException(FullDecoder* decoder, const TagIndexImmediate& imm,
                      Control* block, base::Vector<Value> values) {
    if (deopts_enabled()) {
      if (v8_flags.trace_wasm_inlining) {
        PrintF(
            "[function %d%s: Disabling deoptimizations for speculative "
            "inlining due to legacy exception handling usage]\n",
            func_index_, mode_ == kRegular ? "" : " (inlined)");
      }
      disable_deopts();
    }
 
    BindBlockAndGeneratePhis(decoder, block->false_or_loop_or_catch_block,
                             nullptr, &block->exception);
    V<NativeContext> native_context = instance_cache_.native_context();
    V<WasmTagObject> caught_tag = V<WasmTagObject>::Cast(
        CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmGetOwnProperty>(
            decoder, native_context,
            {block->exception, LOAD_ROOT(wasm_exception_tag_symbol)}));
    // TODO(14616): Support shared tags.
    V<FixedArray> instance_tags =
        LOAD_IMMUTABLE_INSTANCE_FIELD(trusted_instance_data(false), TagsTable,
                                      MemoryRepresentation::TaggedPointer());
    auto expected_tag = V<WasmTagObject>::Cast(
        __ LoadFixedArrayElement(instance_tags, imm.index));
    TSBlock* if_no_catch = NewBlockWithPhis(decoder, nullptr);
    SetupControlFlowEdge(decoder, if_no_catch);
 
    // If the tags don't match we continue with the next tag by setting the
    // no-catch environment as the new {block->false_or_loop_or_catch_block}
    // here.
    block->false_or_loop_or_catch_block = if_no_catch;
 
    if (imm.tag->sig->parameter_count() == 1 &&
        imm.tag->sig->GetParam(0).is_reference_to(HeapType::kExtern)) {
      // Check for the special case where the tag is WebAssembly.JSTag and the
      // exception is not a WebAssembly.Exception. In this case the exception is
      // caught and pushed on the operand stack.
      // Only perform this check if the tag signature is the same as
      // the JSTag signature, i.e. a single externref or (ref extern), otherwise
      // we know statically that it cannot be the JSTag.
      V<Word32> caught_tag_undefined =
          __ TaggedEqual(caught_tag, LOAD_ROOT(UndefinedValue));
      Label<Object> if_catch(&asm_);
      Label<> no_catch_merge(&asm_);
 
      IF (UNLIKELY(caught_tag_undefined)) {
        V<Object> tag_object = __ Load(
            native_context, LoadOp::Kind::TaggedBase(),
            MemoryRepresentation::TaggedPointer(),
            NativeContext::OffsetOfElementAt(Context::WASM_JS_TAG_INDEX));
        V<Object> js_tag = __ Load(tag_object, LoadOp::Kind::TaggedBase(),
                                   MemoryRepresentation::TaggedPointer(),
                                   WasmTagObject::kTagOffset);
        GOTO_IF(__ TaggedEqual(expected_tag, js_tag), if_catch,
                block->exception);
        GOTO(no_catch_merge);
      } ELSE {
        IF (__ TaggedEqual(caught_tag, expected_tag)) {
          UnpackWasmException(decoder, block->exception, values);
          GOTO(if_catch, values[0].op);
        }
        GOTO(no_catch_merge);
      }
 
      BIND(no_catch_merge);
      __ Goto(if_no_catch);
 
      BIND(if_catch, caught_exception);
      // The first unpacked value is the exception itself in the case of a JS
      // exception.
      values[0].op = caught_exception;
    } else {
      TSBlock* if_catch = __ NewBlock();
      __ Branch(ConditionWithHint(__ TaggedEqual(caught_tag, expected_tag)),
                if_catch, if_no_catch);
      __ Bind(if_catch);
      UnpackWasmException(decoder, block->exception, values);
    }
  }
 
  void Delegate(FullDecoder* decoder, uint32_t depth, Control* block) {
    BindBlockAndGeneratePhis(decoder, block->false_or_loop_or_catch_block,
                             nullptr, &block->exception);
    if (depth == decoder->control_depth() - 1) {
      if (mode_ == kInlinedWithCatch) {
        if (block->exception.valid()) {
          return_phis_->AddIncomingException(block->exception);
        }
        __ Goto(return_catch_block_);
      } else {
        // We just throw to the caller, no need to handle the exception in this
        // frame.
        CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmRethrow>(
            decoder, {block->exception});
        __ Unreachable();
      }
    } else {
      DCHECK(decoder->control_at(depth)->is_try());
      TSBlock* target_catch =
          decoder->control_at(depth)->false_or_loop_or_catch_block;
      SetupControlFlowEdge(decoder, target_catch, 0, block->exception);
      __ Goto(target_catch);
    }
  }
 
  void CatchAll(FullDecoder* decoder, Control* block) {
    DCHECK(block->is_try_catchall() || block->is_try_catch());
    DCHECK_EQ(decoder->control_at(0), block);
 
    if (deopts_enabled()) {
      if (v8_flags.trace_wasm_inlining) {
        // TODO(42204618): Would it be worthwhile to add support for this?
        // The difficulty is the handling of the exception which is handled as a
        // value on the value stack in Liftoff but handled very differently in
        // Turboshaft (and it would need to be passed on in the FrameState).
        PrintF(
            "[function %d%s: Disabling deoptimizations for speculative "
            "inlining due to legacy exception handling usage]\n",
            func_index_, mode_ == kRegular ? "" : " (inlined)");
      }
      disable_deopts();
    }
 
    BindBlockAndGeneratePhis(decoder, block->false_or_loop_or_catch_block,
                             nullptr, &block->exception);
  }
 
  void TryTable(FullDecoder* decoder, Control* block) { Try(decoder, block); }
 
  void CatchCase(FullDecoder* decoder, Control* block,
                 const CatchCase& catch_case, base::Vector<Value> values) {
    // If this is the first catch case, {block->false_or_loop_or_catch_block} is
    // the block that was created on block entry, and is where all throwing
    // instructions in the try-table jump to if they throw.
    // Otherwise, {block->false_or_loop_or_catch_block} has been overwritten by
    // the previous handler, and is where we jump to if we did not catch the
    // exception yet.
    BindBlockAndGeneratePhis(decoder, block->false_or_loop_or_catch_block,
                             nullptr, &block->exception);
    if (catch_case.kind == kCatchAll || catch_case.kind == kCatchAllRef) {
      if (catch_case.kind == kCatchAllRef) {
        DCHECK_EQ(values.size(), 1);
        values.last().op = block->exception;
      }
      BrOrRet(decoder, catch_case.br_imm.depth);
      return;
    }
    V<NativeContext> native_context = instance_cache_.native_context();
    V<WasmTagObject> caught_tag = V<WasmTagObject>::Cast(
        CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmGetOwnProperty>(
            decoder, native_context,
            {block->exception, LOAD_ROOT(wasm_exception_tag_symbol)}));
    // TODO(14616): Support shared tags.
    V<FixedArray> instance_tags =
        LOAD_IMMUTABLE_INSTANCE_FIELD(trusted_instance_data(false), TagsTable,
                                      MemoryRepresentation::TaggedPointer());
    auto expected_tag = V<WasmTagObject>::Cast(__ LoadFixedArrayElement(
        instance_tags, catch_case.maybe_tag.tag_imm.index));
    TSBlock* if_no_catch = NewBlockWithPhis(decoder, nullptr);
    SetupControlFlowEdge(decoder, if_no_catch);
 
    // If the tags don't match we continue with the next tag by setting the
    // no-catch environment as the new {block->false_or_loop_or_catch_block}
    // here.
    block->false_or_loop_or_catch_block = if_no_catch;
 
    if (catch_case.maybe_tag.tag_imm.tag->sig->parameter_count() == 1 &&
        catch_case.maybe_tag.tag_imm.tag->sig->GetParam(0) == kWasmExternRef) {
      // Check for the special case where the tag is WebAssembly.JSTag and the
      // exception is not a WebAssembly.Exception. In this case the exception is
      // caught and pushed on the operand stack.
      // Only perform this check if the tag signature is the same as
      // the JSTag signature, i.e. a single externref, otherwise
      // we know statically that it cannot be the JSTag.
      V<Word32> caught_tag_undefined =
          __ TaggedEqual(caught_tag, LOAD_ROOT(UndefinedValue));
      Label<Object> if_catch(&asm_);
      Label<> no_catch_merge(&asm_);
 
      IF (UNLIKELY(caught_tag_undefined)) {
        V<Object> tag_object = __ Load(
            native_context, LoadOp::Kind::TaggedBase(),
            MemoryRepresentation::TaggedPointer(),
            NativeContext::OffsetOfElementAt(Context::WASM_JS_TAG_INDEX));
        V<Object> js_tag = __ Load(tag_object, LoadOp::Kind::TaggedBase(),
                                   MemoryRepresentation::TaggedPointer(),
                                   WasmTagObject::kTagOffset);
        GOTO_IF(__ TaggedEqual(expected_tag, js_tag), if_catch,
                block->exception);
        GOTO(no_catch_merge);
      } ELSE {
        IF (__ TaggedEqual(caught_tag, expected_tag)) {
          if (catch_case.kind == kCatchRef) {
            UnpackWasmException(decoder, block->exception,
                                values.SubVector(0, values.size() - 1));
            values.last().op = block->exception;
          } else {
            UnpackWasmException(decoder, block->exception, values);
          }
          GOTO(if_catch, values[0].op);
        }
        GOTO(no_catch_merge);
      }
 
      BIND(no_catch_merge);
      __ Goto(if_no_catch);
 
      BIND(if_catch, caught_exception);
      // The first unpacked value is the exception itself in the case of a JS
      // exception.
      values[0].op = caught_exception;
    } else {
      TSBlock* if_catch = __ NewBlock();
      __ Branch(ConditionWithHint(__ TaggedEqual(caught_tag, expected_tag)),
                if_catch, if_no_catch);
      __ Bind(if_catch);
      if (catch_case.kind == kCatchRef) {
        UnpackWasmException(decoder, block->exception,
                            values.SubVector(0, values.size() - 1));
        values.last().op = block->exception;
      } else {
        UnpackWasmException(decoder, block->exception, values);
      }
    }
 
    BrOrRet(decoder, catch_case.br_imm.depth);
 
    bool is_last = &catch_case == &block->catch_cases.last();
    if (is_last && !decoder->HasCatchAll(block)) {
      BindBlockAndGeneratePhis(decoder, block->false_or_loop_or_catch_block,
                               nullptr, &block->exception);
      ThrowRef(decoder, block->exception);
    }
  }
 
  void ThrowRef(FullDecoder* decoder, Value* value) {
    ThrowRef(decoder, value->op);
  }
 
  void AtomicNotify(FullDecoder* decoder, const MemoryAccessImmediate& imm,
                    OpIndex index, OpIndex num_waiters_to_wake, Value* result) {
    V<WordPtr> converted_index;
    compiler::BoundsCheckResult bounds_check_result;
    std::tie(converted_index, bounds_check_result) = BoundsCheckMem(
        imm.memory, MemoryRepresentation::Int32(), index, imm.offset,
        compiler::EnforceBoundsCheck::kNeedsBoundsCheck,
        compiler::AlignmentCheck::kYes);
 
    OpIndex effective_offset = __ WordPtrAdd(converted_index, imm.offset);
    OpIndex addr = __ WordPtrAdd(MemStart(imm.mem_index), effective_offset);
 
    auto sig = FixedSizeSignature<MachineType>::Returns(MachineType::Int32())
                   .Params(MachineType::Pointer(), MachineType::Uint32());
    result->op = CallC(&sig, ExternalReference::wasm_atomic_notify(),
                       {addr, num_waiters_to_wake});
  }
 
  void AtomicWait(FullDecoder* decoder, WasmOpcode opcode,
                  const MemoryAccessImmediate& imm, OpIndex index,
                  OpIndex expected, V<Word64> timeout, Value* result) {
    constexpr StubCallMode kStubMode = StubCallMode::kCallWasmRuntimeStub;
    V<WordPtr> converted_index;
    compiler::BoundsCheckResult bounds_check_result;
    std::tie(converted_index, bounds_check_result) = BoundsCheckMem(
        imm.memory,
        opcode == kExprI32AtomicWait ? MemoryRepresentation::Int32()
                                     : MemoryRepresentation::Int64(),
        index, imm.offset, compiler::EnforceBoundsCheck::kNeedsBoundsCheck,
        compiler::AlignmentCheck::kYes);
 
    OpIndex effective_offset = __ WordPtrAdd(converted_index, imm.offset);
    V<BigInt> bigint_timeout = BuildChangeInt64ToBigInt(timeout, kStubMode);
 
    if (opcode == kExprI32AtomicWait) {
      result->op =
          CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmI32AtomicWait>(
              decoder, {__ Word32Constant(imm.memory->index), effective_offset,
                        expected, bigint_timeout});
      return;
    }
    DCHECK_EQ(opcode, kExprI64AtomicWait);
    V<BigInt> bigint_expected = BuildChangeInt64ToBigInt(expected, kStubMode);
    result->op =
        CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmI64AtomicWait>(
            decoder, {__ Word32Constant(imm.memory->index), effective_offset,
                      bigint_expected, bigint_timeout});
  }
 
  void AtomicOp(FullDecoder* decoder, WasmOpcode opcode, const Value args[],
                const size_t argc, const MemoryAccessImmediate& imm,
                Value* result) {
    if (opcode == WasmOpcode::kExprAtomicNotify) {
      return AtomicNotify(decoder, imm, args[0].op, args[1].op, result);
    }
    if (opcode == WasmOpcode::kExprI32AtomicWait ||
        opcode == WasmOpcode::kExprI64AtomicWait) {
      return AtomicWait(decoder, opcode, imm, args[0].op, args[1].op,
                        args[2].op, result);
    }
    using Binop = compiler::turboshaft::AtomicRMWOp::BinOp;
    enum OpType { kBinop, kLoad, kStore };
    struct AtomicOpInfo {
      OpType op_type;
      // Initialize with a default value, to allow constexpr constructors.
      Binop bin_op = Binop::kAdd;
      RegisterRepresentation in_out_rep;
      MemoryRepresentation memory_rep;
 
      constexpr AtomicOpInfo(Binop bin_op, RegisterRepresentation in_out_rep,
                             MemoryRepresentation memory_rep)
          : op_type(kBinop),
            bin_op(bin_op),
            in_out_rep(in_out_rep),
            memory_rep(memory_rep) {}
 
      constexpr AtomicOpInfo(OpType op_type, RegisterRepresentation in_out_rep,
                             MemoryRepresentation memory_rep)
          : op_type(op_type), in_out_rep(in_out_rep), memory_rep(memory_rep) {}
 
      static constexpr AtomicOpInfo Get(wasm::WasmOpcode opcode) {
        switch (opcode) {
#define CASE_BINOP(OPCODE, BINOP, RESULT, INPUT)                           \
  case WasmOpcode::kExpr##OPCODE:                                          \
    return AtomicOpInfo(Binop::k##BINOP, RegisterRepresentation::RESULT(), \
                        MemoryRepresentation::INPUT());
#define RMW_OPERATION(V)                                          \
  V(I32AtomicAdd, Add, Word32, Uint32)                            \
  V(I32AtomicAdd8U, Add, Word32, Uint8)                           \
  V(I32AtomicAdd16U, Add, Word32, Uint16)                         \
  V(I32AtomicSub, Sub, Word32, Uint32)                            \
  V(I32AtomicSub8U, Sub, Word32, Uint8)                           \
  V(I32AtomicSub16U, Sub, Word32, Uint16)                         \
  V(I32AtomicAnd, And, Word32, Uint32)                            \
  V(I32AtomicAnd8U, And, Word32, Uint8)                           \
  V(I32AtomicAnd16U, And, Word32, Uint16)                         \
  V(I32AtomicOr, Or, Word32, Uint32)                              \
  V(I32AtomicOr8U, Or, Word32, Uint8)                             \
  V(I32AtomicOr16U, Or, Word32, Uint16)                           \
  V(I32AtomicXor, Xor, Word32, Uint32)                            \
  V(I32AtomicXor8U, Xor, Word32, Uint8)                           \
  V(I32AtomicXor16U, Xor, Word32, Uint16)                         \
  V(I32AtomicExchange, Exchange, Word32, Uint32)                  \
  V(I32AtomicExchange8U, Exchange, Word32, Uint8)                 \
  V(I32AtomicExchange16U, Exchange, Word32, Uint16)               \
  V(I32AtomicCompareExchange, CompareExchange, Word32, Uint32)    \
  V(I32AtomicCompareExchange8U, CompareExchange, Word32, Uint8)   \
  V(I32AtomicCompareExchange16U, CompareExchange, Word32, Uint16) \
  V(I64AtomicAdd, Add, Word64, Uint64)                            \
  V(I64AtomicAdd8U, Add, Word64, Uint8)                           \
  V(I64AtomicAdd16U, Add, Word64, Uint16)                         \
  V(I64AtomicAdd32U, Add, Word64, Uint32)                         \
  V(I64AtomicSub, Sub, Word64, Uint64)                            \
  V(I64AtomicSub8U, Sub, Word64, Uint8)                           \
  V(I64AtomicSub16U, Sub, Word64, Uint16)                         \
  V(I64AtomicSub32U, Sub, Word64, Uint32)                         \
  V(I64AtomicAnd, And, Word64, Uint64)                            \
  V(I64AtomicAnd8U, And, Word64, Uint8)                           \
  V(I64AtomicAnd16U, And, Word64, Uint16)                         \
  V(I64AtomicAnd32U, And, Word64, Uint32)                         \
  V(I64AtomicOr, Or, Word64, Uint64)                              \
  V(I64AtomicOr8U, Or, Word64, Uint8)                             \
  V(I64AtomicOr16U, Or, Word64, Uint16)                           \
  V(I64AtomicOr32U, Or, Word64, Uint32)                           \
  V(I64AtomicXor, Xor, Word64, Uint64)                            \
  V(I64AtomicXor8U, Xor, Word64, Uint8)                           \
  V(I64AtomicXor16U, Xor, Word64, Uint16)                         \
  V(I64AtomicXor32U, Xor, Word64, Uint32)                         \
  V(I64AtomicExchange, Exchange, Word64, Uint64)                  \
  V(I64AtomicExchange8U, Exchange, Word64, Uint8)                 \
  V(I64AtomicExchange16U, Exchange, Word64, Uint16)               \
  V(I64AtomicExchange32U, Exchange, Word64, Uint32)               \
  V(I64AtomicCompareExchange, CompareExchange, Word64, Uint64)    \
  V(I64AtomicCompareExchange8U, CompareExchange, Word64, Uint8)   \
  V(I64AtomicCompareExchange16U, CompareExchange, Word64, Uint16) \
  V(I64AtomicCompareExchange32U, CompareExchange, Word64, Uint32)
 
          RMW_OPERATION(CASE_BINOP)
#undef RMW_OPERATION
#undef CASE
#define CASE_LOAD(OPCODE, RESULT, INPUT)                         \
  case WasmOpcode::kExpr##OPCODE:                                \
    return AtomicOpInfo(kLoad, RegisterRepresentation::RESULT(), \
                        MemoryRepresentation::INPUT());
#define LOAD_OPERATION(V)             \
  V(I32AtomicLoad, Word32, Uint32)    \
  V(I32AtomicLoad16U, Word32, Uint16) \
  V(I32AtomicLoad8U, Word32, Uint8)   \
  V(I64AtomicLoad, Word64, Uint64)    \
  V(I64AtomicLoad32U, Word64, Uint32) \
  V(I64AtomicLoad16U, Word64, Uint16) \
  V(I64AtomicLoad8U, Word64, Uint8)
          LOAD_OPERATION(CASE_LOAD)
#undef LOAD_OPERATION
#undef CASE_LOAD
#define CASE_STORE(OPCODE, INPUT, OUTPUT)                        \
  case WasmOpcode::kExpr##OPCODE:                                \
    return AtomicOpInfo(kStore, RegisterRepresentation::INPUT(), \
                        MemoryRepresentation::OUTPUT());
#define STORE_OPERATION(V)             \
  V(I32AtomicStore, Word32, Uint32)    \
  V(I32AtomicStore16U, Word32, Uint16) \
  V(I32AtomicStore8U, Word32, Uint8)   \
  V(I64AtomicStore, Word64, Uint64)    \
  V(I64AtomicStore32U, Word64, Uint32) \
  V(I64AtomicStore16U, Word64, Uint16) \
  V(I64AtomicStore8U, Word64, Uint8)
          STORE_OPERATION(CASE_STORE)
#undef STORE_OPERATION_OPERATION
#undef CASE_STORE
          default:
            UNREACHABLE();
        }
      }
    };
 
    AtomicOpInfo info = AtomicOpInfo::Get(opcode);
    V<WordPtr> index;
    compiler::BoundsCheckResult bounds_check_result;
    std::tie(index, bounds_check_result) =
        BoundsCheckMem(imm.memory, info.memory_rep, args[0].op, imm.offset,
                       compiler::EnforceBoundsCheck::kCanOmitBoundsCheck,
                       compiler::AlignmentCheck::kYes);
    // MemoryAccessKind::kUnaligned is impossible due to explicit aligment
    // check.
    MemoryAccessKind access_kind =
        bounds_check_result == compiler::BoundsCheckResult::kTrapHandler
            ? MemoryAccessKind::kProtectedByTrapHandler
            : MemoryAccessKind::kNormal;
 
    if (info.op_type == kBinop) {
      if (info.bin_op == Binop::kCompareExchange) {
        result->op = __ AtomicCompareExchange(
            MemBuffer(imm.memory->index, imm.offset), index, args[1].op,
            args[2].op, info.in_out_rep, info.memory_rep, access_kind);
        return;
      }
      result->op = __ AtomicRMW(MemBuffer(imm.memory->index, imm.offset), index,
                                args[1].op, info.bin_op, info.in_out_rep,
                                info.memory_rep, access_kind);
      return;
    }
    if (info.op_type == kStore) {
      OpIndex value = args[1].op;
      if (info.in_out_rep == RegisterRepresentation::Word64() &&
          info.memory_rep != MemoryRepresentation::Uint64()) {
        value = __ TruncateWord64ToWord32(value);
      }
#ifdef V8_TARGET_BIG_ENDIAN
      // Reverse the value bytes before storing.
      DCHECK(info.in_out_rep == RegisterRepresentation::Word32() ||
             info.in_out_rep == RegisterRepresentation::Word64());
      wasm::ValueType wasm_type =
          info.in_out_rep == RegisterRepresentation::Word32() ? wasm::kWasmI32
                                                              : wasm::kWasmI64;
      value = BuildChangeEndiannessStore(
          value, info.memory_rep.ToMachineType().representation(), wasm_type);
#endif
      __ Store(MemBuffer(imm.memory->index, imm.offset), index, value,
               access_kind == MemoryAccessKind::kProtectedByTrapHandler
                   ? LoadOp::Kind::Protected().Atomic()
                   : LoadOp::Kind::RawAligned().Atomic(),
               info.memory_rep, compiler::kNoWriteBarrier);
      return;
    }
    DCHECK_EQ(info.op_type, kLoad);
    RegisterRepresentation loaded_value_rep = info.in_out_rep;
#if V8_TARGET_BIG_ENDIAN
    // Do not sign-extend / zero-extend the value to 64 bits as the bytes need
    // to be reversed first to keep little-endian load / store semantics. Still
    // extend for 1 byte loads as it doesn't require reversing any bytes.
    bool needs_zero_extension_64 = false;
    if (info.in_out_rep == RegisterRepresentation::Word64() &&
        info.memory_rep.SizeInBytes() < 8 &&
        info.memory_rep.SizeInBytes() != 1) {
      needs_zero_extension_64 = true;
      loaded_value_rep = RegisterRepresentation::Word32();
    }
#endif
    result->op =
        __ Load(MemBuffer(imm.memory->index, imm.offset), index,
                access_kind == MemoryAccessKind::kProtectedByTrapHandler
                    ? LoadOp::Kind::Protected().Atomic()
                    : LoadOp::Kind::RawAligned().Atomic(),
                info.memory_rep, loaded_value_rep);
 
#ifdef V8_TARGET_BIG_ENDIAN
    // Reverse the value bytes after load.
    DCHECK(info.in_out_rep == RegisterRepresentation::Word32() ||
           info.in_out_rep == RegisterRepresentation::Word64());
    wasm::ValueType wasm_type =
        info.in_out_rep == RegisterRepresentation::Word32() ? wasm::kWasmI32
                                                            : wasm::kWasmI64;
    result->op = BuildChangeEndiannessLoad(
        result->op, info.memory_rep.ToMachineType(), wasm_type);
 
    if (needs_zero_extension_64) {
      result->op = __ ChangeUint32ToUint64(result->op);
    }
#endif
  }
 
  void AtomicFence(FullDecoder* decoder) {
    __ MemoryBarrier(AtomicMemoryOrder::kSeqCst);
  }
 
  void MemoryInit(FullDecoder* decoder, const MemoryInitImmediate& imm,
                  const Value& dst, const Value& src, const Value& size) {
    V<WordPtr> dst_uintptr = MemoryAddressToUintPtrOrOOBTrap(
        imm.memory.memory->address_type, dst.op);
    DCHECK_EQ(size.type, kWasmI32);
    auto sig = FixedSizeSignature<MachineType>::Returns(MachineType::Int32())
                   .Params(MachineType::Pointer(), MachineType::Uint32(),
                           MachineType::UintPtr(), MachineType::Uint32(),
                           MachineType::Uint32(), MachineType::Uint32());
    // TODO(14616): Fix sharedness.
    V<Word32> result =
        CallC(&sig, ExternalReference::wasm_memory_init(),
              {__ BitcastHeapObjectToWordPtr(trusted_instance_data(false)),
               __ Word32Constant(imm.memory.index), dst_uintptr, src.op,
               __ Word32Constant(imm.data_segment.index), size.op});
    __ TrapIfNot(result, TrapId::kTrapMemOutOfBounds);
  }
 
  void MemoryCopy(FullDecoder* decoder, const MemoryCopyImmediate& imm,
                  const Value& dst, const Value& src, const Value& size) {
    const WasmMemory* dst_memory = imm.memory_dst.memory;
    const WasmMemory* src_memory = imm.memory_src.memory;
    V<WordPtr> dst_uintptr =
        MemoryAddressToUintPtrOrOOBTrap(dst_memory->address_type, dst.op);
    V<WordPtr> src_uintptr =
        MemoryAddressToUintPtrOrOOBTrap(src_memory->address_type, src.op);
    AddressType min_address_type =
        dst_memory->is_memory64() && src_memory->is_memory64()
            ? AddressType::kI64
            : AddressType::kI32;
    V<WordPtr> size_uintptr =
        MemoryAddressToUintPtrOrOOBTrap(min_address_type, size.op);
    auto sig = FixedSizeSignature<MachineType>::Returns(MachineType::Int32())
                   .Params(MachineType::Pointer(), MachineType::Uint32(),
                           MachineType::Uint32(), MachineType::UintPtr(),
                           MachineType::UintPtr(), MachineType::UintPtr());
    // TODO(14616): Fix sharedness.
    V<Word32> result =
        CallC(&sig, ExternalReference::wasm_memory_copy(),
              {__ BitcastHeapObjectToWordPtr(trusted_instance_data(false)),
               __ Word32Constant(imm.memory_dst.index),
               __ Word32Constant(imm.memory_src.index), dst_uintptr,
               src_uintptr, size_uintptr});
    __ TrapIfNot(result, TrapId::kTrapMemOutOfBounds);
  }
 
  void MemoryFill(FullDecoder* decoder, const MemoryIndexImmediate& imm,
                  const Value& dst, const Value& value, const Value& size) {
    AddressType address_type = imm.memory->address_type;
    V<WordPtr> dst_uintptr =
        MemoryAddressToUintPtrOrOOBTrap(address_type, dst.op);
    V<WordPtr> size_uintptr =
        MemoryAddressToUintPtrOrOOBTrap(address_type, size.op);
    auto sig = FixedSizeSignature<MachineType>::Returns(MachineType::Int32())
                   .Params(MachineType::Pointer(), MachineType::Uint32(),
                           MachineType::UintPtr(), MachineType::Uint8(),
                           MachineType::UintPtr());
    // TODO(14616): Fix sharedness.
    V<Word32> result = CallC(
        &sig, ExternalReference::wasm_memory_fill(),
        {__ BitcastHeapObjectToWordPtr(trusted_instance_data(false)),
         __ Word32Constant(imm.index), dst_uintptr, value.op, size_uintptr});
 
    __ TrapIfNot(result, TrapId::kTrapMemOutOfBounds);
  }
 
  void DataDrop(FullDecoder* decoder, const IndexImmediate& imm) {
    // TODO(14616): Data segments aren't available during streaming compilation.
    // Discussion: github.com/WebAssembly/shared-everything-threads/issues/83
    bool shared = decoder->enabled_.has_shared() &&
                  decoder->module_->data_segments[imm.index].shared;
    V<FixedUInt32Array> data_segment_sizes = LOAD_IMMUTABLE_INSTANCE_FIELD(
        trusted_instance_data(shared), DataSegmentSizes,
        MemoryRepresentation::TaggedPointer());
    __ Store(data_segment_sizes, __ Word32Constant(0),
             StoreOp::Kind::TaggedBase(), MemoryRepresentation::Int32(),
             compiler::kNoWriteBarrier,
             FixedUInt32Array::OffsetOfElementAt(imm.index));
  }
 
  void TableGet(FullDecoder* decoder, const Value& index, Value* result,
                const TableIndexImmediate& imm) {
    V<WasmTableObject> table = LoadTable(decoder, imm);
    V<Smi> size_smi = __ Load(table, LoadOp::Kind::TaggedBase(),
                              MemoryRepresentation::TaggedSigned(),
                              WasmTableObject::kCurrentLengthOffset);
    V<WordPtr> index_wordptr =
        TableAddressToUintPtrOrOOBTrap(imm.table->address_type, index.op);
    DCHECK_GE(kSmiMaxValue, wasm::max_table_size());
    V<Word32> in_bounds = __ UintPtrLessThan(
        index_wordptr, __ ChangeUint32ToUintPtr(__ UntagSmi(size_smi)));
    __ TrapIfNot(in_bounds, TrapId::kTrapTableOutOfBounds);
    V<FixedArray> entries = __ Load(table, LoadOp::Kind::TaggedBase(),
                                    MemoryRepresentation::TaggedPointer(),
                                    WasmTableObject::kEntriesOffset);
    OpIndex entry = __ LoadFixedArrayElement(entries, index_wordptr);
 
    if (imm.table->type.ref_type_kind() == RefTypeKind::kFunction) {
      // If the entry has map type Tuple2, call WasmFunctionTableGet which will
      // initialize the function table entry.
      Label<Object> resolved(&asm_);
      Label<> call_runtime(&asm_);
      // The entry is a WasmFuncRef, WasmNull, or Tuple2. Hence
      // it is safe to cast it to HeapObject.
      V<Map> entry_map = __ LoadMapField(V<HeapObject>::Cast(entry));
      V<Word32> instance_type = __ LoadInstanceTypeField(entry_map);
      GOTO_IF(
          UNLIKELY(__ Word32Equal(instance_type, InstanceType::TUPLE2_TYPE)),
          call_runtime);
      // Otherwise the entry is WasmFuncRef or WasmNull; we are done.
      GOTO(resolved, entry);
 
      BIND(call_runtime);
      bool extract_shared_data = !shared_ && imm.table->shared;
      GOTO(resolved,
           CallBuiltinThroughJumptable<
               BuiltinCallDescriptor::WasmFunctionTableGet>(
               decoder, {__ IntPtrConstant(imm.index), index_wordptr,
                         __ Word32Constant(extract_shared_data ? 1 : 0)}));
 
      BIND(resolved, resolved_entry);
      result->op = resolved_entry;
    } else {
      result->op = entry;
    }
    result->op = AnnotateResultIfReference(result->op, imm.table->type);
  }
 
  void TableSet(FullDecoder* decoder, const Value& index, const Value& value,
                const TableIndexImmediate& imm) {
    bool extract_shared_data = !shared_ && imm.table->shared;
 
    V<WordPtr> index_wordptr =
        TableAddressToUintPtrOrOOBTrap(imm.table->address_type, index.op);
 
    if (imm.table->type.ref_type_kind() == RefTypeKind::kFunction) {
      CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmTableSetFuncRef>(
          decoder, {__ IntPtrConstant(imm.index),
                    __ Word32Constant(extract_shared_data ? 1 : 0),
                    index_wordptr, value.op});
    } else {
      CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmTableSet>(
          decoder, {__ IntPtrConstant(imm.index),
                    __ Word32Constant(extract_shared_data ? 1 : 0),
                    index_wordptr, value.op});
    }
  }
 
  void TableInit(FullDecoder* decoder, const TableInitImmediate& imm,
                 const Value& dst_val, const Value& src_val,
                 const Value& size_val) {
    const WasmTable* table = imm.table.table;
    V<WordPtr> dst_wordptr =
        TableAddressToUintPtrOrOOBTrap(table->address_type, dst_val.op);
    V<Word32> src = src_val.op;
    V<Word32> size = size_val.op;
    DCHECK_EQ(table->shared, table->shared);
    CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmTableInit>(
        decoder, {
                     dst_wordptr,
                     src,
                     size,
                     __ NumberConstant(imm.table.index),
                     __ NumberConstant(imm.element_segment.index),
                     __ NumberConstant((!shared_ && table->shared) ? 1 : 0),
                 });
  }
 
  void TableCopy(FullDecoder* decoder, const TableCopyImmediate& imm,
                 const Value& dst_val, const Value& src_val,
                 const Value& size_val) {
    const WasmTable* dst_table = imm.table_dst.table;
    const WasmTable* src_table = imm.table_src.table;
    V<WordPtr> dst_wordptr =
        TableAddressToUintPtrOrOOBTrap(dst_table->address_type, dst_val.op);
    V<WordPtr> src_wordptr =
        TableAddressToUintPtrOrOOBTrap(src_table->address_type, src_val.op);
    AddressType min_address_type =
        dst_table->is_table64() && src_table->is_table64() ? AddressType::kI64
                                                           : AddressType::kI32;
    V<WordPtr> size_wordptr =
        TableAddressToUintPtrOrOOBTrap(min_address_type, size_val.op);
    bool table_is_shared = imm.table_dst.table->shared;
    // TODO(14616): Is this too restrictive?
    DCHECK_EQ(table_is_shared, imm.table_src.table->shared);
    CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmTableCopy>(
        decoder, {dst_wordptr, src_wordptr, size_wordptr,
                  __ NumberConstant(imm.table_dst.index),
                  __ NumberConstant(imm.table_src.index),
                  __ NumberConstant((!shared_ && table_is_shared) ? 1 : 0)});
  }
 
  void TableGrow(FullDecoder* decoder, const TableIndexImmediate& imm,
                 const Value& value, const Value& delta, Value* result) {
    Label<Word32> end(&asm_);
    V<WordPtr> delta_wordptr;
 
    // If `delta` is OOB, return -1.
    if (!imm.table->is_table64()) {
      delta_wordptr = __ ChangeUint32ToUintPtr(delta.op);
    } else if constexpr (Is64()) {
      delta_wordptr = delta.op;
    } else {
      GOTO_IF(UNLIKELY(__ TruncateWord64ToWord32(
                  __ Word64ShiftRightLogical(delta.op, 32))),
              end, __ Word32Constant(-1));
      delta_wordptr = V<WordPtr>::Cast(__ TruncateWord64ToWord32(delta.op));
    }
 
    bool extract_shared_data = !shared_ && imm.table->shared;
    DCHECK_GE(kSmiMaxValue, wasm::max_table_size());
    V<Word32> call_result = __ UntagSmi(
        CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmTableGrow>(
            decoder, {__ NumberConstant(imm.index), delta_wordptr,
                      __ Word32Constant(extract_shared_data), value.op}));
    GOTO(end, call_result);
 
    BIND(end, result_i32);
    if (imm.table->is_table64()) {
      result->op = __ ChangeInt32ToInt64(result_i32);
    } else {
      result->op = result_i32;
    }
  }
 
  void TableFill(FullDecoder* decoder, const TableIndexImmediate& imm,
                 const Value& start, const Value& value, const Value& count) {
    V<WordPtr> start_wordptr =
        TableAddressToUintPtrOrOOBTrap(imm.table->address_type, start.op);
    V<WordPtr> count_wordptr =
        TableAddressToUintPtrOrOOBTrap(imm.table->address_type, count.op);
    bool extract_shared_data = !shared_ && imm.table->shared;
    CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmTableFill>(
        decoder,
        {start_wordptr, count_wordptr, __ Word32Constant(extract_shared_data),
         __ NumberConstant(imm.index), value.op});
  }
 
  V<WasmTableObject> LoadTable(FullDecoder* decoder,
                               const TableIndexImmediate& imm) {
    V<FixedArray> tables = LOAD_IMMUTABLE_INSTANCE_FIELD(
        trusted_instance_data(imm.table->shared), Tables,
        MemoryRepresentation::TaggedPointer());
    return V<WasmTableObject>::Cast(
        __ LoadFixedArrayElement(tables, imm.index));
  }
 
  void TableSize(FullDecoder* decoder, const TableIndexImmediate& imm,
                 Value* result) {
    V<WasmTableObject> table = LoadTable(decoder, imm);
    V<Word32> size_word32 = __ UntagSmi(__ Load(
        table, LoadOp::Kind::TaggedBase(), MemoryRepresentation::TaggedSigned(),
        WasmTableObject::kCurrentLengthOffset));
    if (imm.table->is_table64()) {
      result->op = __ ChangeUint32ToUint64(size_word32);
    } else {
      result->op = size_word32;
    }
  }
 
  void ElemDrop(FullDecoder* decoder, const IndexImmediate& imm) {
    // Note: Contrary to data segments, elem segments occur before the code
    // section, so we can be sure that they're available even during streaming
    // compilation.
    bool shared = decoder->module_->elem_segments[imm.index].shared;
    V<FixedArray> elem_segments = LOAD_IMMUTABLE_INSTANCE_FIELD(
        trusted_instance_data(shared), ElementSegments,
        MemoryRepresentation::TaggedPointer());
    __ StoreFixedArrayElement(elem_segments, imm.index,
                              LOAD_ROOT(EmptyFixedArray),
                              compiler::kFullWriteBarrier);
  }
 
  void StructNew(FullDecoder* decoder, const StructIndexImmediate& imm,
                 const Value& descriptor, const Value args[], Value* result) {
    uint32_t field_count = imm.struct_type->field_count();
    SmallZoneVector<OpIndex, 16> args_vector(field_count, decoder->zone_);
    for (uint32_t i = 0; i < field_count; ++i) {
      args_vector[i] = args[i].op;
    }
    result->op = StructNewImpl(decoder, imm, descriptor, args_vector.data());
  }
 
  void StructNewDefault(FullDecoder* decoder, const StructIndexImmediate& imm,
                        const Value& descriptor, Value* result) {
    uint32_t field_count = imm.struct_type->field_count();
    SmallZoneVector<OpIndex, 16> args(field_count, decoder->zone_);
    for (uint32_t i = 0; i < field_count; i++) {
      ValueType field_type = imm.struct_type->field(i);
      args[i] = DefaultValue(field_type);
    }
    result->op = StructNewImpl(decoder, imm, descriptor, args.data());
  }
 
  void StructGet(FullDecoder* decoder, const Value& struct_object,
                 const FieldImmediate& field, bool is_signed, Value* result) {
    result->op = __ StructGet(
        V<WasmStructNullable>::Cast(struct_object.op),
        field.struct_imm.struct_type, field.struct_imm.index,
        field.field_imm.index, is_signed,
        struct_object.type.is_nullable() ? compiler::kWithNullCheck
                                         : compiler::kWithoutNullCheck);
  }
 
  void StructSet(FullDecoder* decoder, const Value& struct_object,
                 const FieldImmediate& field, const Value& field_value) {
    __ StructSet(V<WasmStructNullable>::Cast(struct_object.op), field_value.op,
                 field.struct_imm.struct_type, field.struct_imm.index,
                 field.field_imm.index,
                 struct_object.type.is_nullable()
                     ? compiler::kWithNullCheck
                     : compiler::kWithoutNullCheck);
  }
 
  void ArrayNew(FullDecoder* decoder, const ArrayIndexImmediate& imm,
                const Value& length, const Value& initial_value,
                Value* result) {
    result->op = ArrayNewImpl(decoder, imm.index, imm.array_type,
                              V<Word32>::Cast(length.op),
                              V<Any>::Cast(initial_value.op));
  }
 
  void ArrayNewDefault(FullDecoder* decoder, const ArrayIndexImmediate& imm,
                       const Value& length, Value* result) {
    V<Any> initial_value = DefaultValue(imm.array_type->element_type());
    result->op = ArrayNewImpl(decoder, imm.index, imm.array_type,
                              V<Word32>::Cast(length.op), initial_value);
  }
 
  void ArrayGet(FullDecoder* decoder, const Value& array_obj,
                const ArrayIndexImmediate& imm, const Value& index,
                bool is_signed, Value* result) {
    auto array_value = V<WasmArrayNullable>::Cast(array_obj.op);
    BoundsCheckArray(array_value, index.op, array_obj.type);
    result->op = __ ArrayGet(array_value, V<Word32>::Cast(index.op),
                             imm.array_type, is_signed);
  }
 
  void ArraySet(FullDecoder* decoder, const Value& array_obj,
                const ArrayIndexImmediate& imm, const Value& index,
                const Value& value) {
    auto array_value = V<WasmArrayNullable>::Cast(array_obj.op);
    BoundsCheckArray(array_value, index.op, array_obj.type);
    __ ArraySet(array_value, V<Word32>::Cast(index.op), V<Any>::Cast(value.op),
                imm.array_type->element_type());
  }
 
  void ArrayLen(FullDecoder* decoder, const Value& array_obj, Value* result) {
    result->op = __ ArrayLength(V<WasmArrayNullable>::Cast(array_obj.op),
                                array_obj.type.is_nullable()
                                    ? compiler::kWithNullCheck
                                    : compiler::kWithoutNullCheck);
  }
 
  void ArrayCopy(FullDecoder* decoder, const Value& dst, const Value& dst_index,
                 const Value& src, const Value& src_index,
                 const ArrayIndexImmediate& src_imm, const Value& length) {
    V<WasmArrayNullable> src_array = V<WasmArrayNullable>::Cast(src.op);
    V<WasmArrayNullable> dst_array = V<WasmArrayNullable>::Cast(dst.op);
    BoundsCheckArrayWithLength(dst_array, dst_index.op, length.op,
                               dst.type.is_nullable()
                                   ? compiler::kWithNullCheck
                                   : compiler::kWithoutNullCheck);
    BoundsCheckArrayWithLength(src_array, src_index.op, length.op,
                               src.type.is_nullable()
                                   ? compiler::kWithNullCheck
                                   : compiler::kWithoutNullCheck);
 
    ValueType element_type = src_imm.array_type->element_type();
 
    IF_NOT (__ Word32Equal(length.op, 0)) {
      // Values determined by test/mjsunit/wasm/array-copy-benchmark.js on x64.
      int array_copy_max_loop_length;
      switch (element_type.kind()) {
        case wasm::kI32:
        case wasm::kI64:
        case wasm::kI8:
        case wasm::kI16:
          array_copy_max_loop_length = 20;
          break;
        case wasm::kF16:  // TODO(irezvov): verify the threshold for F16.
        case wasm::kF32:
        case wasm::kF64:
          array_copy_max_loop_length = 35;
          break;
        case wasm::kS128:
          array_copy_max_loop_length = 100;
          break;
        case wasm::kRef:
        case wasm::kRefNull:
          array_copy_max_loop_length = 15;
          break;
        case wasm::kVoid:
        case kTop:
        case wasm::kBottom:
          UNREACHABLE();
      }
 
      IF (__ Uint32LessThan(array_copy_max_loop_length, length.op)) {
        // Builtin
        MachineType arg_types[]{MachineType::TaggedPointer(),
                                MachineType::Uint32(),
                                MachineType::TaggedPointer(),
                                MachineType::Uint32(), MachineType::Uint32()};
        MachineSignature sig(0, 5, arg_types);
 
        CallC(&sig, ExternalReference::wasm_array_copy(),
              {dst_array, dst_index.op, src_array, src_index.op, length.op});
      } ELSE {
        V<Word32> src_end_index =
            __ Word32Sub(__ Word32Add(src_index.op, length.op), 1);
 
        IF (__ Uint32LessThan(src_index.op, dst_index.op)) {
          // Reverse
          V<Word32> dst_end_index =
              __ Word32Sub(__ Word32Add(dst_index.op, length.op), 1);
          ScopedVar<Word32> src_index_loop(this, src_end_index);
          ScopedVar<Word32> dst_index_loop(this, dst_end_index);
 
          WHILE(__ Word32Constant(1)) {
            V<Any> value = __ ArrayGet(src_array, src_index_loop,
                                       src_imm.array_type, true);
            __ ArraySet(dst_array, dst_index_loop, value, element_type);
 
            IF_NOT (__ Uint32LessThan(src_index.op, src_index_loop)) BREAK;
 
            src_index_loop = __ Word32Sub(src_index_loop, 1);
            dst_index_loop = __ Word32Sub(dst_index_loop, 1);
          }
        } ELSE {
          ScopedVar<Word32> src_index_loop(this, src_index.op);
          ScopedVar<Word32> dst_index_loop(this, dst_index.op);
 
          WHILE(__ Word32Constant(1)) {
            V<Any> value = __ ArrayGet(src_array, src_index_loop,
                                       src_imm.array_type, true);
            __ ArraySet(dst_array, dst_index_loop, value, element_type);
 
            IF_NOT (__ Uint32LessThan(src_index_loop, src_end_index)) BREAK;
 
            src_index_loop = __ Word32Add(src_index_loop, 1);
            dst_index_loop = __ Word32Add(dst_index_loop, 1);
          }
        }
      }
    }
  }
 
  void ArrayFill(FullDecoder* decoder, ArrayIndexImmediate& imm,
                 const Value& array, const Value& index, const Value& value,
                 const Value& length) {
    const bool emit_write_barrier =
        imm.array_type->element_type().is_reference();
    auto array_value = V<WasmArrayNullable>::Cast(array.op);
    V<WasmArray> array_not_null = BoundsCheckArrayWithLength(
        array_value, index.op, length.op,
        array.type.is_nullable() ? compiler::kWithNullCheck
                                 : compiler::kWithoutNullCheck);
    ArrayFillImpl(array_not_null, V<Word32>::Cast(index.op),
                  V<Any>::Cast(value.op), V<Word32>::Cast(length.op),
                  imm.array_type, emit_write_barrier);
  }
 
  void ArrayNewFixed(FullDecoder* decoder, const ArrayIndexImmediate& array_imm,
                     const IndexImmediate& length_imm, const Value elements[],
                     Value* result) {
    const wasm::ArrayType* type = array_imm.array_type;
    wasm::ValueType element_type = type->element_type();
    int element_count = length_imm.index;
    // Initialize the array header.
    bool shared = decoder->module_->type(array_imm.index).is_shared;
    V<Map> rtt = __ RttCanon(managed_object_maps(shared), array_imm.index);
    V<WasmArray> array = __ WasmAllocateArray(rtt, element_count, type);
    // Initialize all elements.
    for (int i = 0; i < element_count; i++) {
      __ ArraySet(array, __ Word32Constant(i), elements[i].op, element_type);
    }
    result->op = array;
  }
 
  void ArrayNewSegment(FullDecoder* decoder,
                       const ArrayIndexImmediate& array_imm,
                       const IndexImmediate& segment_imm, const Value& offset,
                       const Value& length, Value* result) {
    bool is_element = array_imm.array_type->element_type().is_reference();
    // TODO(14616): Data segments aren't available during streaming compilation.
    // Discussion: github.com/WebAssembly/shared-everything-threads/issues/83
    bool segment_is_shared =
        decoder->enabled_.has_shared() &&
        (is_element
             ? decoder->module_->elem_segments[segment_imm.index].shared
             : decoder->module_->data_segments[segment_imm.index].shared);
    // TODO(14616): Add DCHECK that array sharedness is equal to `shared`?
    V<WasmArray> result_value =
        CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmArrayNewSegment>(
            decoder,
            {__ Word32Constant(segment_imm.index), offset.op, length.op,
             __ SmiConstant(Smi::FromInt(is_element ? 1 : 0)),
             __ SmiConstant(Smi::FromInt(!shared_ && segment_is_shared)),
             __ RttCanon(managed_object_maps(segment_is_shared),
                         array_imm.index)});
    result->op = __ AnnotateWasmType(result_value, result->type);
  }
 
  void ArrayInitSegment(FullDecoder* decoder,
                        const ArrayIndexImmediate& array_imm,
                        const IndexImmediate& segment_imm, const Value& array,
                        const Value& array_index, const Value& segment_offset,
                        const Value& length) {
    bool is_element = array_imm.array_type->element_type().is_reference();
    // TODO(14616): Segments aren't available during streaming compilation.
    bool segment_is_shared =
        decoder->enabled_.has_shared() &&
        (is_element
             ? decoder->module_->elem_segments[segment_imm.index].shared
             : decoder->module_->data_segments[segment_imm.index].shared);
    // TODO(14616): Is this too restrictive?
    DCHECK_EQ(segment_is_shared,
              decoder->module_->type(array_imm.index).is_shared);
    CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmArrayInitSegment>(
        decoder,
        {array_index.op, segment_offset.op, length.op,
         __ SmiConstant(Smi::FromInt(segment_imm.index)),
         __ SmiConstant(Smi::FromInt(is_element ? 1 : 0)),
         __ SmiConstant(Smi::FromInt((!shared_ && segment_is_shared) ? 1 : 0)),
         array.op});
  }
 
  void RefI31(FullDecoder* decoder, const Value& input, Value* result) {
    if constexpr (SmiValuesAre31Bits()) {
      V<Word32> shifted =
          __ Word32ShiftLeft(input.op, kSmiTagSize + kSmiShiftSize);
      if constexpr (Is64()) {
        // The uppermost bits don't matter.
        result->op = __ BitcastWord32ToWord64(shifted);
      } else {
        result->op = shifted;
      }
    } else {
      // Set the topmost bit to sign-extend the second bit. This way,
      // interpretation in JS (if this value escapes there) will be the same as
      // i31.get_s.
      V<WordPtr> input_wordptr = __ ChangeUint32ToUintPtr(input.op);
      result->op = __ WordPtrShiftRightArithmetic(
          __ WordPtrShiftLeft(input_wordptr, kSmiShiftSize + kSmiTagSize + 1),
          1);
    }
    result->op =
        __ AnnotateWasmType(__ BitcastWordPtrToSmi(result->op), kWasmRefI31);
  }
 
  void I31GetS(FullDecoder* decoder, const Value& input, Value* result) {
    V<Object> input_non_null = NullCheck(input);
    if constexpr (SmiValuesAre31Bits()) {
      result->op = __ Word32ShiftRightArithmeticShiftOutZeros(
          __ TruncateWordPtrToWord32(__ BitcastTaggedToWordPtr(input_non_null)),
          kSmiTagSize + kSmiShiftSize);
    } else {
      // Topmost bit is already sign-extended.
      result->op = __ TruncateWordPtrToWord32(
          __ WordPtrShiftRightArithmeticShiftOutZeros(
              __ BitcastTaggedToWordPtr(input_non_null),
              kSmiTagSize + kSmiShiftSize));
    }
  }
 
  void I31GetU(FullDecoder* decoder, const Value& input, Value* result) {
    V<Object> input_non_null = NullCheck(input);
    if constexpr (SmiValuesAre31Bits()) {
      result->op = __ Word32ShiftRightLogical(
          __ TruncateWordPtrToWord32(__ BitcastTaggedToWordPtr(input_non_null)),
          kSmiTagSize + kSmiShiftSize);
    } else {
      // Topmost bit is sign-extended, remove it.
      result->op = __ TruncateWordPtrToWord32(__ WordPtrShiftRightLogical(
          __ WordPtrShiftLeft(__ BitcastTaggedToWordPtr(input_non_null), 1),
          kSmiTagSize + kSmiShiftSize + 1));
    }
  }
 
  void RefGetDesc(FullDecoder* decoder, const Value& ref_val, Value* result) {
    // Implicit null checks don't cover the map load.
    V<Object> ref = NullCheck(ref_val);
    V<Map> map = __ LoadMapField(ref);
    result->op = __ Load(map, LoadOp::Kind::TaggedBase().Immutable(),
                         MemoryRepresentation::TaggedPointer(),
                         Map::kInstanceDescriptorsOffset);
  }
 
  compiler::ExactOrSubtype GetExactness(FullDecoder* decoder, HeapType target) {
    // For exact target types, an exact match is needed for correctness;
    // for final target types, it's a performance optimization.
    if (target.is_exact() ||
        decoder->module_->type(target.ref_index()).is_final) {
      return compiler::kExactMatchOnly;
    }
    return compiler::kMayBeSubtype;
  }
 
  void RefTest(FullDecoder* decoder, HeapType target, const Value& object,
               Value* result, bool null_succeeds) {
    V<Map> rtt = __ RttCanon(managed_object_maps(target.is_shared()),
                             target.ref_index());
    compiler::WasmTypeCheckConfig config{
        object.type,
        ValueType::RefMaybeNull(target,
                                null_succeeds ? kNullable : kNonNullable),
        GetExactness(decoder, target)};
    result->op = __ WasmTypeCheck(object.op, rtt, config);
  }
 
  void RefTestAbstract(FullDecoder* decoder, const Value& object, HeapType type,
                       Value* result, bool null_succeeds) {
    compiler::WasmTypeCheckConfig config{
        object.type, ValueType::RefMaybeNull(
                         type, null_succeeds ? kNullable : kNonNullable)};
    V<Map> rtt = OpIndex::Invalid();
    result->op = __ WasmTypeCheck(object.op, rtt, config);
  }
 
  void RefCast(FullDecoder* decoder, const Value& object, Value* result) {
    if (v8_flags.experimental_wasm_assume_ref_cast_succeeds) {
      // TODO(14108): Implement type guards.
      Forward(decoder, object, result);
      return;
    }
    ValueType target = result->type;
    V<Map> rtt = __ RttCanon(managed_object_maps(target.is_shared()),
                             target.ref_index());
    compiler::WasmTypeCheckConfig config{
        object.type, target, GetExactness(decoder, target.heap_type())};
    result->op = __ WasmTypeCast(object.op, rtt, config);
  }
 
  void RefCastDesc(FullDecoder* decoder, const Value& object, const Value& desc,
                   Value* result) {
    if (v8_flags.experimental_wasm_assume_ref_cast_succeeds) {
      // TODO(14108): Implement type guards.
      Forward(decoder, object, result);
      return;
    }
    V<Map> rtt = GetRttFromDescriptor(desc);
    compiler::WasmTypeCheckConfig config{object.type, result->type,
                                         compiler::kExactMatchOnly};
    result->op = __ WasmTypeCast(object.op, rtt, config);
  }
 
  void RefCastAbstract(FullDecoder* decoder, const Value& object, HeapType type,
                       Value* result, bool null_succeeds) {
    if (v8_flags.experimental_wasm_assume_ref_cast_succeeds) {
      // TODO(14108): Implement type guards.
      Forward(decoder, object, result);
      return;
    }
    // TODO(jkummerow): {type} is redundant.
    DCHECK_IMPLIES(null_succeeds, result->type.is_nullable());
    DCHECK_EQ(type, result->type.heap_type());
    compiler::WasmTypeCheckConfig config{
        object.type, ValueType::RefMaybeNull(
                         type, null_succeeds ? kNullable : kNonNullable)};
    V<Map> rtt = OpIndex::Invalid();
    result->op = __ WasmTypeCast(object.op, rtt, config);
  }
 
  // TODO(jkummerow): Pass {target} to the interface.
  void BrOnCast(FullDecoder* decoder, HeapType target_type, const Value& object,
                Value* value_on_branch, uint32_t br_depth, bool null_succeeds) {
    // Note: we cannot just take {ref_index} and {null_succeeds} from
    // {value_on_branch->type} because of
    // https://github.com/WebAssembly/gc/issues/516.
    ValueType target = ValueType::RefMaybeNull(
        target_type, null_succeeds ? kNullable : kNonNullable);
    V<Map> rtt = __ RttCanon(managed_object_maps(target.is_shared()),
                             target_type.ref_index());
    compiler::WasmTypeCheckConfig config{object.type, target,
                                         GetExactness(decoder, target_type)};
    return BrOnCastImpl(decoder, rtt, config, object, value_on_branch, br_depth,
                        null_succeeds);
  }
 
  void BrOnCastDesc(FullDecoder* decoder, HeapType target_type,
                    const Value& object, const Value& descriptor,
                    Value* value_on_branch, uint32_t br_depth,
                    bool null_succeeds) {
    ValueType target = ValueType::RefMaybeNull(
        target_type, null_succeeds ? kNullable : kNonNullable);
    V<Map> rtt = GetRttFromDescriptor(descriptor);
    compiler::WasmTypeCheckConfig config{object.type, target,
                                         compiler::kExactMatchOnly};
    return BrOnCastImpl(decoder, rtt, config, object, value_on_branch, br_depth,
                        null_succeeds);
  }
 
  void BrOnCastAbstract(FullDecoder* decoder, const Value& object,
                        HeapType type, Value* value_on_branch,
                        uint32_t br_depth, bool null_succeeds) {
    V<Map> rtt = OpIndex::Invalid();
    compiler::WasmTypeCheckConfig config{
        object.type, ValueType::RefMaybeNull(
                         type, null_succeeds ? kNullable : kNonNullable)};
    return BrOnCastImpl(decoder, rtt, config, object, value_on_branch, br_depth,
                        null_succeeds);
  }
 
  void BrOnCastFail(FullDecoder* decoder, HeapType target_type,
                    const Value& object, Value* value_on_fallthrough,
                    uint32_t br_depth, bool null_succeeds) {
    ValueType target = ValueType::RefMaybeNull(
        target_type, null_succeeds ? kNullable : kNonNullable);
    V<Map> rtt = __ RttCanon(managed_object_maps(target.is_shared()),
                             target_type.ref_index());
    compiler::WasmTypeCheckConfig config{object.type, target,
                                         GetExactness(decoder, target_type)};
    return BrOnCastFailImpl(decoder, rtt, config, object, value_on_fallthrough,
                            br_depth, null_succeeds);
  }
 
  void BrOnCastDescFail(FullDecoder* decoder, HeapType target_type,
                        const Value& object, const Value& descriptor,
                        Value* value_on_fallthrough, uint32_t br_depth,
                        bool null_succeeds) {
    ValueType target = ValueType::RefMaybeNull(
        target_type, null_succeeds ? kNullable : kNonNullable);
    V<Map> rtt = GetRttFromDescriptor(descriptor);
    compiler::WasmTypeCheckConfig config{object.type, target,
                                         compiler::kExactMatchOnly};
    return BrOnCastFailImpl(decoder, rtt, config, object, value_on_fallthrough,
                            br_depth, null_succeeds);
  }
 
  void BrOnCastFailAbstract(FullDecoder* decoder, const Value& object,
                            HeapType type, Value* value_on_fallthrough,
                            uint32_t br_depth, bool null_succeeds) {
    V<Map> rtt = OpIndex::Invalid();
    compiler::WasmTypeCheckConfig config{
        object.type, ValueType::RefMaybeNull(
                         type, null_succeeds ? kNullable : kNonNullable)};
    return BrOnCastFailImpl(decoder, rtt, config, object, value_on_fallthrough,
                            br_depth, null_succeeds);
  }
 
  void StringNewWtf8(FullDecoder* decoder, const MemoryIndexImmediate& imm,
                     const unibrow::Utf8Variant variant, const Value& offset,
                     const Value& size, Value* result) {
    V<Word32> memory = __ Word32Constant(imm.index);
    V<Smi> variant_smi =
        __ SmiConstant(Smi::FromInt(static_cast<int>(variant)));
    V<WordPtr> index =
        MemoryAddressToUintPtrOrOOBTrap(imm.memory->address_type, offset.op);
    V<WasmStringRefNullable> result_value =
        CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmStringNewWtf8>(
            decoder, {index, size.op, memory, variant_smi});
    result->op = __ AnnotateWasmType(result_value, result->type);
  }
 
  // TODO(jkummerow): This check would be more elegant if we made
  // {ArrayNewSegment} a high-level node that's lowered later.
  // Returns the call on success, nullptr otherwise (like `TryCast`).
  const CallOp* IsArrayNewSegment(V<Object> array) {
    DCHECK_IMPLIES(!array.valid(), __ generating_unreachable_operations());
    if (__ generating_unreachable_operations()) return nullptr;
    if (const WasmTypeAnnotationOp* annotation =
            __ output_graph().Get(array).TryCast<WasmTypeAnnotationOp>()) {
      array = annotation->value();
    }
    if (const DidntThrowOp* didnt_throw =
            __ output_graph().Get(array).TryCast<DidntThrowOp>()) {
      array = didnt_throw->throwing_operation();
    }
    const CallOp* call = __ output_graph().Get(array).TryCast<CallOp>();
    if (call == nullptr) return nullptr;
    uint64_t stub_id{};
    if (!OperationMatcher(__ output_graph())
             .MatchWasmStubCallConstant(call->callee(), &stub_id)) {
      return nullptr;
    }
    DCHECK_LT(stub_id, static_cast<uint64_t>(Builtin::kFirstBytecodeHandler));
    if (stub_id == static_cast<uint64_t>(Builtin::kWasmArrayNewSegment)) {
      return call;
    }
    return nullptr;
  }
 
  V<HeapObject> StringNewWtf8ArrayImpl(FullDecoder* decoder,
                                       const unibrow::Utf8Variant variant,
                                       const Value& array, const Value& start,
                                       const Value& end,
                                       ValueType result_type) {
    // Special case: shortcut a sequence "array from data segment" + "string
    // from wtf8 array" to directly create a string from the segment.
    V<internal::UnionOf<String, WasmNull, Null>> call;
    if (const CallOp* array_new = IsArrayNewSegment(array.op)) {
      // We can only pass 3 untagged parameters to the builtin (on 32-bit
      // platforms). The segment index is easy to tag: if it validated, it must
      // be in Smi range.
      OpIndex segment_index = array_new->input(1);
      int32_t index_val;
      OperationMatcher(__ output_graph())
          .MatchIntegralWord32Constant(segment_index, &index_val);
      V<Smi> index_smi = __ SmiConstant(Smi::FromInt(index_val));
      // Arbitrary choice for the second tagged parameter: the segment offset.
      OpIndex segment_offset = array_new->input(2);
      __ TrapIfNot(
          __ Uint32LessThan(segment_offset, __ Word32Constant(Smi::kMaxValue)),
          OpIndex::Invalid(), TrapId::kTrapDataSegmentOutOfBounds);
      V<Smi> offset_smi = __ TagSmi(segment_offset);
      OpIndex segment_length = array_new->input(3);
      V<Smi> variant_smi =
          __ SmiConstant(Smi::FromInt(static_cast<int32_t>(variant)));
      call = CallBuiltinThroughJumptable<
          BuiltinCallDescriptor::WasmStringFromDataSegment>(
          decoder, {segment_length, start.op, end.op, index_smi, offset_smi,
                    variant_smi});
    } else {
      // Regular path if the shortcut wasn't taken.
      call = CallBuiltinThroughJumptable<
          BuiltinCallDescriptor::WasmStringNewWtf8Array>(
          decoder,
          {start.op, end.op, V<WasmArray>::Cast(NullCheck(array)),
           __ SmiConstant(Smi::FromInt(static_cast<int32_t>(variant)))});
    }
    DCHECK_IMPLIES(variant == unibrow::Utf8Variant::kUtf8NoTrap,
                   result_type.is_nullable());
    // The builtin returns a WasmNull for kUtf8NoTrap, so nullable values in
    // combination with extern strings are not supported.
    DCHECK_NE(result_type, wasm::kWasmExternRef);
    return AnnotateAsString(call, result_type);
  }
 
  void StringNewWtf8Array(FullDecoder* decoder,
                          const unibrow::Utf8Variant variant,
                          const Value& array, const Value& start,
                          const Value& end, Value* result) {
    result->op = StringNewWtf8ArrayImpl(decoder, variant, array, start, end,
                                        result->type);
  }
 
  void StringNewWtf16(FullDecoder* decoder, const MemoryIndexImmediate& imm,
                      const Value& offset, const Value& size, Value* result) {
    V<WordPtr> index =
        MemoryAddressToUintPtrOrOOBTrap(imm.memory->address_type, offset.op);
    V<String> result_value =
        CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmStringNewWtf16>(
            decoder, {__ Word32Constant(imm.index), index, size.op});
    result->op = __ AnnotateWasmType(result_value, result->type);
  }
 
  void StringNewWtf16Array(FullDecoder* decoder, const Value& array,
                           const Value& start, const Value& end,
                           Value* result) {
    V<String> result_value = CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::WasmStringNewWtf16Array>(
        decoder, {V<WasmArray>::Cast(NullCheck(array)), start.op, end.op});
    result->op = __ AnnotateWasmType(result_value, result->type);
  }
 
  void StringConst(FullDecoder* decoder, const StringConstImmediate& imm,
                   Value* result) {
    V<String> result_value =
        CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmStringConst>(
            decoder, {__ Word32Constant(imm.index)});
    result->op = __ AnnotateWasmType(result_value, result->type);
  }
 
  void StringMeasureWtf8(FullDecoder* decoder,
                         const unibrow::Utf8Variant variant, const Value& str,
                         Value* result) {
    result->op = StringMeasureWtf8Impl(decoder, variant,
                                       V<String>::Cast(NullCheck(str)));
  }
 
  OpIndex StringMeasureWtf8Impl(FullDecoder* decoder,
                                const unibrow::Utf8Variant variant,
                                V<String> string) {
    switch (variant) {
      case unibrow::Utf8Variant::kUtf8:
        return CallBuiltinThroughJumptable<
            BuiltinCallDescriptor::WasmStringMeasureUtf8>(decoder, {string});
      case unibrow::Utf8Variant::kLossyUtf8:
      case unibrow::Utf8Variant::kWtf8:
        return CallBuiltinThroughJumptable<
            BuiltinCallDescriptor::WasmStringMeasureWtf8>(decoder, {string});
      case unibrow::Utf8Variant::kUtf8NoTrap:
        UNREACHABLE();
    }
  }
 
  V<Word32> LoadStringLength(V<Object> string) {
    return __ template LoadField<Word32>(
        string, compiler::AccessBuilder::ForStringLength());
  }
 
  void StringMeasureWtf16(FullDecoder* decoder, const Value& str,
                          Value* result) {
    result->op = LoadStringLength(NullCheck(str));
  }
 
  void StringEncodeWtf8(FullDecoder* decoder,
                        const MemoryIndexImmediate& memory,
                        const unibrow::Utf8Variant variant, const Value& str,
                        const Value& offset, Value* result) {
    V<WordPtr> address =
        MemoryAddressToUintPtrOrOOBTrap(memory.memory->address_type, offset.op);
    V<Word32> mem_index = __ Word32Constant(memory.index);
    V<Word32> utf8 = __ Word32Constant(static_cast<int32_t>(variant));
    result->op = CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::WasmStringEncodeWtf8>(
        decoder, {address, mem_index, utf8, V<String>::Cast(NullCheck(str))});
  }
 
  void StringEncodeWtf8Array(FullDecoder* decoder,
                             const unibrow::Utf8Variant variant,
                             const Value& str, const Value& array,
                             const Value& start, Value* result) {
    result->op = StringEncodeWtf8ArrayImpl(
        decoder, variant, V<String>::Cast(NullCheck(str)),
        V<WasmArray>::Cast(NullCheck(array)), start.op);
  }
 
  OpIndex StringEncodeWtf8ArrayImpl(FullDecoder* decoder,
                                    const unibrow::Utf8Variant variant,
                                    V<String> str, V<WasmArray> array,
                                    V<Word32> start) {
    V<Smi> utf8 = __ SmiConstant(Smi::FromInt(static_cast<int32_t>(variant)));
    return CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::WasmStringEncodeWtf8Array>(
        decoder, {str, array, start, utf8});
  }
 
  void StringEncodeWtf16(FullDecoder* decoder, const MemoryIndexImmediate& imm,
                         const Value& str, const Value& offset, Value* result) {
    V<WordPtr> address =
        MemoryAddressToUintPtrOrOOBTrap(imm.memory->address_type, offset.op);
    V<Word32> mem_index = __ Word32Constant(static_cast<int32_t>(imm.index));
    result->op = CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::WasmStringEncodeWtf16>(
        decoder, {V<String>::Cast(NullCheck(str)), address, mem_index});
  }
 
  void StringEncodeWtf16Array(FullDecoder* decoder, const Value& str,
                              const Value& array, const Value& start,
                              Value* result) {
    result->op = CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::WasmStringEncodeWtf16Array>(
        decoder, {V<String>::Cast(NullCheck(str)),
                  V<WasmArray>::Cast(NullCheck(array)), start.op});
  }
 
  void StringConcat(FullDecoder* decoder, const Value& head, const Value& tail,
                    Value* result) {
    V<NativeContext> native_context = instance_cache_.native_context();
    V<String> result_value =
        CallBuiltinThroughJumptable<BuiltinCallDescriptor::StringAdd_CheckNone>(
            decoder, native_context,
            {V<String>::Cast(NullCheck(head)),
             V<String>::Cast(NullCheck(tail))});
    result->op = __ AnnotateWasmType(result_value, result->type);
  }
 
  V<Word32> StringEqImpl(FullDecoder* decoder, V<String> a, V<String> b,
                         ValueType a_type, ValueType b_type) {
    Label<Word32> done(&asm_);
    // Covers "identical string pointer" and "both are null" cases.
    GOTO_IF(__ TaggedEqual(a, b), done, __ Word32Constant(1));
    if (a_type.is_nullable()) {
      GOTO_IF(__ IsNull(a, a_type), done, __ Word32Constant(0));
    }
    if (b_type.is_nullable()) {
      GOTO_IF(__ IsNull(b, b_type), done, __ Word32Constant(0));
    }
 
    // Strings with unequal lengths can't be equal.
    V<Word32> a_length =
        __ LoadField<Word32>(a, AccessBuilder::ForStringLength());
    V<Word32> b_length =
        __ LoadField<Word32>(b, AccessBuilder::ForStringLength());
    GOTO_IF_NOT(__ Word32Equal(a_length, b_length), done, __ Word32Constant(0));
 
    // Comparing the strings' hashes before calling the builtin has been tried,
    // but isn't likely to be useful unless Wasm gains an ability to trigger
    // string hash computation (or internalization, for that matter).
 
    V<WordPtr> length = __ ChangeInt32ToIntPtr(a_length);
    V<Boolean> result =
        CallBuiltinByPointer<BuiltinCallDescriptor::StringEqual>(
            decoder, {a, b, length});
 
    GOTO(done, __ IsRootConstant(result, RootIndex::kTrueValue));
 
    BIND(done, eq_result);
    return eq_result;
  }
 
  void StringEq(FullDecoder* decoder, const Value& a, const Value& b,
                Value* result) {
    result->op = StringEqImpl(decoder, a.op, b.op, a.type, b.type);
  }
 
  void StringIsUSVSequence(FullDecoder* decoder, const Value& str,
                           Value* result) {
    result->op = CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::WasmStringIsUSVSequence>(
        decoder, {V<String>::Cast(NullCheck(str))});
  }
 
  void StringAsWtf8(FullDecoder* decoder, const Value& str, Value* result) {
    V<ByteArray> result_value =
        CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmStringAsWtf8>(
            decoder, {V<String>::Cast(NullCheck(str))});
    result->op = __ AnnotateWasmType(result_value, result->type);
  }
 
  void StringViewWtf8Advance(FullDecoder* decoder, const Value& view,
                             const Value& pos, const Value& bytes,
                             Value* result) {
    result->op = CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::WasmStringViewWtf8Advance>(
        decoder, {V<ByteArray>::Cast(NullCheck(view)), pos.op, bytes.op});
  }
 
  void StringViewWtf8Encode(FullDecoder* decoder,
                            const MemoryIndexImmediate& memory,
                            const unibrow::Utf8Variant variant,
                            const Value& view, const Value& addr,
                            const Value& pos, const Value& bytes,
                            Value* next_pos, Value* bytes_written) {
    V<WordPtr> address =
        MemoryAddressToUintPtrOrOOBTrap(memory.memory->address_type, addr.op);
    V<Smi> mem_index = __ SmiConstant(Smi::FromInt(memory.index));
    V<Smi> utf8 = __ SmiConstant(Smi::FromInt(static_cast<int32_t>(variant)));
    OpIndex result = CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::WasmStringViewWtf8Encode>(
        decoder, {address, pos.op, bytes.op,
                  V<ByteArray>::Cast(NullCheck(view)), mem_index, utf8});
    next_pos->op = __ Projection(result, 0, RepresentationFor(next_pos->type));
    bytes_written->op =
        __ Projection(result, 1, RepresentationFor(bytes_written->type));
  }
 
  void StringViewWtf8Slice(FullDecoder* decoder, const Value& view,
                           const Value& start, const Value& end,
                           Value* result) {
    V<String> result_value = CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::WasmStringViewWtf8Slice>(
        decoder, {V<ByteArray>::Cast(NullCheck(view)), start.op, end.op});
    result->op = __ AnnotateWasmType(result_value, result->type);
  }
 
  void StringAsWtf16(FullDecoder* decoder, const Value& str, Value* result) {
    result->op = __ StringAsWtf16(V<String>::Cast(NullCheck(str)));
  }
 
  V<Word32> GetCodeUnitImpl(FullDecoder* decoder, V<String> string,
                            V<Word32> offset) {
    auto prepare = __ StringPrepareForGetCodeUnit(string);
    V<Object> base = __ template Projection<0>(prepare);
    V<WordPtr> base_offset = __ template Projection<1>(prepare);
    V<Word32> charwidth_shift = __ template Projection<2>(prepare);
 
    // Bounds check.
    V<Word32> length = LoadStringLength(string);
    __ TrapIfNot(__ Uint32LessThan(offset, length),
                 TrapId::kTrapStringOffsetOutOfBounds);
 
    Label<> onebyte(&asm_);
    Label<> bailout(&asm_);
    Label<Word32> done(&asm_);
    GOTO_IF(UNLIKELY(__ Word32Equal(charwidth_shift,
                                    compiler::kCharWidthBailoutSentinel)),
            bailout);
    GOTO_IF(__ Word32Equal(charwidth_shift, 0), onebyte);
 
    // Two-byte.
    V<WordPtr> object_offset = __ WordPtrAdd(
        __ WordPtrMul(__ ChangeInt32ToIntPtr(offset), 2), base_offset);
    // Bitcast the tagged to a wordptr as the offset already contains the
    // kHeapObjectTag handling. Furthermore, in case of external strings the
    // tagged value is a smi 0, which doesn't really encode a tagged load.
    V<WordPtr> base_ptr = __ BitcastTaggedToWordPtr(base);
    V<Word32> result_value =
        __ Load(base_ptr, object_offset, LoadOp::Kind::RawAligned().Immutable(),
                MemoryRepresentation::Uint16());
    GOTO(done, result_value);
 
    // One-byte.
    BIND(onebyte);
    object_offset = __ WordPtrAdd(__ ChangeInt32ToIntPtr(offset), base_offset);
    // Bitcast the tagged to a wordptr as the offset already contains the
    // kHeapObjectTag handling. Furthermore, in case of external strings the
    // tagged value is a smi 0, which doesn't really encode a tagged load.
    base_ptr = __ BitcastTaggedToWordPtr(base);
    result_value =
        __ Load(base_ptr, object_offset, LoadOp::Kind::RawAligned().Immutable(),
                MemoryRepresentation::Uint8());
    GOTO(done, result_value);
 
    BIND(bailout);
    GOTO(done, CallBuiltinThroughJumptable<
                   BuiltinCallDescriptor::WasmStringViewWtf16GetCodeUnit>(
                   decoder, {string, offset}));
 
    BIND(done, final_result);
    // Make sure the original string is kept alive as long as we're operating
    // on pointers extracted from it (otherwise e.g. external strings' resources
    // might get freed prematurely).
    __ Retain(string);
    return final_result;
  }
 
  void StringViewWtf16GetCodeUnit(FullDecoder* decoder, const Value& view,
                                  const Value& pos, Value* result) {
    result->op =
        GetCodeUnitImpl(decoder, V<String>::Cast(NullCheck(view)), pos.op);
  }
 
  V<Word32> StringCodePointAt(FullDecoder* decoder, V<String> string,
                              V<Word32> offset) {
    auto prepare = __ StringPrepareForGetCodeUnit(string);
    V<Object> base = __ template Projection<0>(prepare);
    V<WordPtr> base_offset = __ template Projection<1>(prepare);
    V<Word32> charwidth_shift = __ template Projection<2>(prepare);
 
    // Bounds check.
    V<Word32> length = LoadStringLength(string);
    __ TrapIfNot(__ Uint32LessThan(offset, length),
                 TrapId::kTrapStringOffsetOutOfBounds);
 
    Label<> onebyte(&asm_);
    Label<> bailout(&asm_);
    Label<Word32> done(&asm_);
    GOTO_IF(
        __ Word32Equal(charwidth_shift, compiler::kCharWidthBailoutSentinel),
        bailout);
    GOTO_IF(__ Word32Equal(charwidth_shift, 0), onebyte);
 
    // Two-byte.
    V<WordPtr> object_offset = __ WordPtrAdd(
        __ WordPtrMul(__ ChangeInt32ToIntPtr(offset), 2), base_offset);
    // Bitcast the tagged to a wordptr as the offset already contains the
    // kHeapObjectTag handling. Furthermore, in case of external strings the
    // tagged value is a smi 0, which doesn't really encode a tagged load.
    V<WordPtr> base_ptr = __ BitcastTaggedToWordPtr(base);
    V<Word32> lead =
        __ Load(base_ptr, object_offset, LoadOp::Kind::RawAligned().Immutable(),
                MemoryRepresentation::Uint16());
    V<Word32> is_lead_surrogate =
        __ Word32Equal(__ Word32BitwiseAnd(lead, 0xFC00), 0xD800);
    GOTO_IF_NOT(is_lead_surrogate, done, lead);
    V<Word32> trail_offset = __ Word32Add(offset, 1);
    GOTO_IF_NOT(__ Uint32LessThan(trail_offset, length), done, lead);
    V<Word32> trail = __ Load(
        base_ptr, __ WordPtrAdd(object_offset, __ IntPtrConstant(2)),
        LoadOp::Kind::RawAligned().Immutable(), MemoryRepresentation::Uint16());
    V<Word32> is_trail_surrogate =
        __ Word32Equal(__ Word32BitwiseAnd(trail, 0xFC00), 0xDC00);
    GOTO_IF_NOT(is_trail_surrogate, done, lead);
    V<Word32> surrogate_bias =
        __ Word32Constant(0x10000 - (0xD800 << 10) - 0xDC00);
    V<Word32> result = __ Word32Add(__ Word32ShiftLeft(lead, 10),
                                    __ Word32Add(trail, surrogate_bias));
    GOTO(done, result);
 
    // One-byte.
    BIND(onebyte);
    object_offset = __ WordPtrAdd(__ ChangeInt32ToIntPtr(offset), base_offset);
    // Bitcast the tagged to a wordptr as the offset already contains the
    // kHeapObjectTag handling. Furthermore, in case of external strings the
    // tagged value is a smi 0, which doesn't really encode a tagged load.
    base_ptr = __ BitcastTaggedToWordPtr(base);
    result =
        __ Load(base_ptr, object_offset, LoadOp::Kind::RawAligned().Immutable(),
                MemoryRepresentation::Uint8());
    GOTO(done, result);
 
    BIND(bailout);
    GOTO(done, CallBuiltinThroughJumptable<
                   BuiltinCallDescriptor::WasmStringCodePointAt>(
                   decoder, {string, offset}));
 
    BIND(done, final_result);
    // Make sure the original string is kept alive as long as we're operating
    // on pointers extracted from it (otherwise e.g. external strings' resources
    // might get freed prematurely).
    __ Retain(string);
    return final_result;
  }
 
  void StringViewWtf16Encode(FullDecoder* decoder,
                             const MemoryIndexImmediate& imm, const Value& view,
                             const Value& offset, const Value& pos,
                             const Value& codeunits, Value* result) {
    V<String> string = V<String>::Cast(NullCheck(view));
    V<WordPtr> address =
        MemoryAddressToUintPtrOrOOBTrap(imm.memory->address_type, offset.op);
    V<Smi> mem_index = __ SmiConstant(Smi::FromInt(imm.index));
    result->op = CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::WasmStringViewWtf16Encode>(
        decoder, {address, pos.op, codeunits.op, string, mem_index});
  }
 
  void StringViewWtf16Slice(FullDecoder* decoder, const Value& view,
                            const Value& start, const Value& end,
                            Value* result) {
    V<String> string = V<String>::Cast(NullCheck(view));
    V<String> result_value = CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::WasmStringViewWtf16Slice>(
        decoder, {string, start.op, end.op});
    result->op = __ AnnotateWasmType(result_value, result->type);
  }
 
  void StringAsIter(FullDecoder* decoder, const Value& str, Value* result) {
    V<String> string = V<String>::Cast(NullCheck(str));
    V<WasmStringViewIter> result_value =
        CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmStringAsIter>(
            decoder, {string});
    result->op = __ AnnotateWasmType(result_value, result->type);
  }
 
  void StringViewIterNext(FullDecoder* decoder, const Value& view,
                          Value* result) {
    V<WasmStringViewIter> iter = V<WasmStringViewIter>::Cast(NullCheck(view));
    result->op = CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::WasmStringViewIterNext>(decoder, {iter});
  }
 
  void StringViewIterAdvance(FullDecoder* decoder, const Value& view,
                             const Value& codepoints, Value* result) {
    V<WasmStringViewIter> iter = V<WasmStringViewIter>::Cast(NullCheck(view));
    result->op = CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::WasmStringViewIterAdvance>(
        decoder, {iter, codepoints.op});
  }
 
  void StringViewIterRewind(FullDecoder* decoder, const Value& view,
                            const Value& codepoints, Value* result) {
    V<WasmStringViewIter> iter = V<WasmStringViewIter>::Cast(NullCheck(view));
    result->op = CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::WasmStringViewIterRewind>(decoder,
                                                         {iter, codepoints.op});
  }
 
  void StringViewIterSlice(FullDecoder* decoder, const Value& view,
                           const Value& codepoints, Value* result) {
    V<WasmStringViewIter> iter = V<WasmStringViewIter>::Cast(NullCheck(view));
    V<String> result_value = CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::WasmStringViewIterSlice>(decoder,
                                                        {iter, codepoints.op});
    result->op = __ AnnotateWasmType(result_value, result->type);
  }
 
  void StringCompare(FullDecoder* decoder, const Value& lhs, const Value& rhs,
                     Value* result) {
    V<String> lhs_val = V<String>::Cast(NullCheck(lhs));
    V<String> rhs_val = V<String>::Cast(NullCheck(rhs));
    result->op = __ UntagSmi(
        CallBuiltinThroughJumptable<BuiltinCallDescriptor::StringCompare>(
            decoder, {lhs_val, rhs_val}));
  }
 
  void StringFromCodePoint(FullDecoder* decoder, const Value& code_point,
                           Value* result) {
    V<String> result_value = CallBuiltinThroughJumptable<
        BuiltinCallDescriptor::WasmStringFromCodePoint>(decoder,
                                                        {code_point.op});
    result->op = __ AnnotateWasmType(result_value, result->type);
  }
 
  void StringHash(FullDecoder* decoder, const Value& string, Value* result) {
    V<String> string_val = V<String>::Cast(NullCheck(string));
 
    Label<> runtime_label(&Asm());
    Label<Word32> end_label(&Asm());
 
    V<Word32> raw_hash = __ template LoadField<Word32>(
        string_val, compiler::AccessBuilder::ForNameRawHashField());
    V<Word32> hash_not_computed_mask =
        __ Word32Constant(static_cast<int32_t>(Name::kHashNotComputedMask));
    static_assert(Name::HashFieldTypeBits::kShift == 0);
    V<Word32> hash_not_computed =
        __ Word32BitwiseAnd(raw_hash, hash_not_computed_mask);
    GOTO_IF(hash_not_computed, runtime_label);
 
    // Fast path if hash is already computed: Decode raw hash value.
    static_assert(Name::HashBits::kLastUsedBit == kBitsPerInt - 1);
    V<Word32> hash = __ Word32ShiftRightLogical(
        raw_hash, static_cast<int32_t>(Name::HashBits::kShift));
    GOTO(end_label, hash);
 
    BIND(runtime_label);
    V<Word32> hash_runtime =
        CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmStringHash>(
            decoder, {string_val});
    GOTO(end_label, hash_runtime);
 
    BIND(end_label, hash_val);
    result->op = hash_val;
  }
 
  void Forward(FullDecoder* decoder, const Value& from, Value* to) {
    to->op = from.op;
  }
 
 private:
  // The InstanceCache caches commonly used fields of the
  // WasmTrustedInstanceData.
  // We can extend the set of cached fields as needed.
  // This caching serves two purposes:
  // (1) It makes sure that the respective fields are loaded early on, as
  //     opposed to within conditional branches, so the values are easily
  //     reusable.
  // (2) It makes sure that the loaded values are actually reused.
  // It achieves these effects more reliably and more cheaply than general-
  // purpose optimizations could (loop peeling isn't always used; load
  // elimination struggles with arbitrary side effects of indexed stores;
  // we don't currently have a generic mechanism for hoisting loads out of
  // conditional branches).
  class InstanceCache {
   public:
    explicit InstanceCache(Assembler& assembler)
        : mem_start_(assembler), mem_size_(assembler), asm_(assembler) {}
 
    void Initialize(V<WasmTrustedInstanceData> trusted_instance_data,
                    const WasmModule* mod) {
      DCHECK(!trusted_data_.valid());  // Only call {Initialize()} once.
      trusted_data_ = trusted_instance_data;
      managed_object_maps_ =
          __ Load(trusted_instance_data, LoadOp::Kind::TaggedBase().Immutable(),
                  MemoryRepresentation::TaggedPointer(),
                  WasmTrustedInstanceData::kManagedObjectMapsOffset);
      native_context_ =
          __ Load(trusted_instance_data, LoadOp::Kind::TaggedBase().Immutable(),
                  MemoryRepresentation::TaggedPointer(),
                  WasmTrustedInstanceData::kNativeContextOffset);
 
      if (!mod->memories.empty()) {
#if DEBUG
        has_memory_ = true;
#endif
        const WasmMemory& mem = mod->memories[0];
        memory_can_grow_ = mem.initial_pages != mem.maximum_pages;
        // For now, we don't cache the size of shared growable memories.
        // If we wanted to support this case, we would have to reload the
        // memory size when loop stack checks detect an interrupt request.
        // Since memory size caching is particularly important for asm.js,
        // which never uses growable or shared memories, this limitation is
        // considered acceptable for now.
        memory_size_cached_ = !mem.is_shared || !memory_can_grow_;
        // Trap handler enabled memories never move.
        // Memories that can't grow have no reason to move.
        // Shared memories can only be grown in-place.
        memory_can_move_ = mem.bounds_checks != kTrapHandler &&
                           memory_can_grow_ && !mem.is_shared;
        memory_is_shared_ = mem.is_shared;
        if (memory_size_cached_) {
          mem_size_ = LoadMemSize();
        }
        mem_start_ = LoadMemStart();
      }
    }
 
    // TODO(14108): Port the dynamic "cached_memory_index" infrastructure
    // from Turbofan.
    void ReloadCachedMemory() {
      if (memory_can_move()) mem_start_ = LoadMemStart();
      if (memory_can_grow_ && memory_size_cached_) mem_size_ = LoadMemSize();
    }
 
    V<WasmTrustedInstanceData> trusted_instance_data() { return trusted_data_; }
    V<FixedArray> managed_object_maps() { return managed_object_maps_; }
    V<NativeContext> native_context() { return native_context_; }
    V<WordPtr> memory0_start() {
      DCHECK(has_memory_);
      return mem_start_;
    }
    V<WordPtr> memory0_size() {
      DCHECK(has_memory_);
      if (!memory_size_cached_) return LoadMemSize();
      return mem_size_;
    }
 
   private:
    static constexpr uint8_t kUnused = ~uint8_t{0};
 
    V<WordPtr> LoadMemStart() {
      DCHECK(has_memory_);
      // In contrast to memory size loads, we can mark memory start loads as
      // eliminable: shared memories never move, and non-shared memories can't
      // have their start modified by other threads.
      LoadOp::Kind kind = LoadOp::Kind::TaggedBase();
      if (!memory_can_move()) kind = kind.Immutable();
      return __ Load(trusted_data_, kind, MemoryRepresentation::UintPtr(),
                     WasmTrustedInstanceData::kMemory0StartOffset);
    }
 
    V<WordPtr> LoadMemSize() {
      DCHECK(has_memory_);
      LoadOp::Kind kind = LoadOp::Kind::TaggedBase();
      if (memory_is_shared_ && memory_can_grow_) {
        // Memory size loads should not be load-eliminated as the memory size
        // can be modified by another thread.
        kind = kind.NotLoadEliminable();
      }
      if (!memory_can_grow_) kind = kind.Immutable();
      return __ Load(trusted_data_, kind, MemoryRepresentation::UintPtr(),
                     WasmTrustedInstanceData::kMemory0SizeOffset);
    }
 
    bool memory_can_move() { return memory_can_move_; }
 
    // For compatibility with `__` macro.
    Assembler& Asm() { return asm_; }
 
    // Cached immutable fields (need no Phi nodes):
    V<WasmTrustedInstanceData> trusted_data_;
    V<FixedArray> managed_object_maps_;
    V<NativeContext> native_context_;
 
    // Cached mutable fields:
    ScopedVar<WordPtr> mem_start_;
    ScopedVar<WordPtr> mem_size_;
 
    // Other fields for internal usage.
    Assembler& asm_;
    bool memory_is_shared_{false};
    bool memory_can_grow_{false};
    bool memory_can_move_{false};
    bool memory_size_cached_{false};
#if DEBUG
    bool has_memory_{false};
#endif
  };
 
  enum class CheckForException { kNo, kCatchInThisFrame, kCatchInParentFrame };
 
 private:
  // Holds phi inputs for a specific block. These include SSA values, stack
  // merge values, and cached fields from the instance..
  // Conceptually, this is a two-dimensional, rectangular array of size
  // `phi_count * inputs_per_phi`, since each phi has the same number of inputs,
  // namely the number of incoming edges for this block.
  class BlockPhis {
   public:
    // Ctor for regular blocks.
    V8_INLINE BlockPhis(FullDecoder* decoder, Merge<Value>* merge)
        : incoming_exceptions_(decoder -> zone()) {
      // Allocate space and initialize the types of all phis.
      uint32_t num_locals = decoder->num_locals();
      uint32_t merge_arity = merge != nullptr ? merge->arity : 0;
 
      phi_count_ = num_locals + merge_arity;
      phi_types_ = decoder->zone()->AllocateArray<ValueType>(phi_count_);
 
      base::Vector<ValueType> locals = decoder->local_types();
      std::uninitialized_copy(locals.begin(), locals.end(), phi_types_);
      for (uint32_t i = 0; i < merge_arity; i++) {
        new (&phi_types_[num_locals + i]) ValueType((*merge)[i].type);
      }
      AllocatePhiInputs(decoder->zone());
    }
 
    // Consider this "private"; it's next to the constructors (where it's
    // called) for context.
    void AllocatePhiInputs(Zone* zone) {
      // Only reserve some space for the inputs to be added later.
      phi_inputs_capacity_total_ = phi_count_ * input_capacity_per_phi_;
      phi_inputs_ = zone->AllocateArray<OpIndex>(phi_inputs_capacity_total_);
 
#ifdef DEBUG
      constexpr uint32_t kNoInputs = 0;
      input_count_per_phi_ = std::vector(phi_count_, kNoInputs);
#endif
    }
 
    // Default ctor and later initialization for function returns.
    explicit BlockPhis(Zone* zone) : incoming_exceptions_(zone) {}
    void InitReturnPhis(base::Vector<const ValueType> return_types) {
      // For `return_phis_`, nobody should have inserted into `this` before
      // calling `InitReturnPhis`.
      DCHECK_EQ(phi_count_, 0);
      DCHECK_EQ(inputs_per_phi_, 0);
 
      uint32_t return_count = static_cast<uint32_t>(return_types.size());
      phi_count_ = return_count;
      phi_types_ = zone()->AllocateArray<ValueType>(phi_count_);
 
      std::uninitialized_copy(return_types.begin(), return_types.end(),
                              phi_types_);
      AllocatePhiInputs(zone());
    }
 
    void AddInputForPhi(size_t phi_i, OpIndex input) {
      if (V8_UNLIKELY(phi_inputs_total_ >= phi_inputs_capacity_total_)) {
        GrowInputsVector();
      }
 
#ifdef DEBUG
      // We rely on adding inputs in the order of phis, i.e.,
      // `AddInputForPhi(0, ...); AddInputForPhi(1, ...); ...`.
      size_t phi_inputs_start = phi_i * input_capacity_per_phi_;
      size_t phi_input_offset_from_start = inputs_per_phi_;
      CHECK_EQ(input_count_per_phi_[phi_i]++, phi_input_offset_from_start);
      size_t phi_input_offset = phi_inputs_start + phi_input_offset_from_start;
      CHECK_EQ(next_phi_input_add_offset_, phi_input_offset);
#endif
      new (&phi_inputs_[next_phi_input_add_offset_]) OpIndex(input);
 
      phi_inputs_total_++;
      next_phi_input_add_offset_ += input_capacity_per_phi_;
      if (next_phi_input_add_offset_ >= phi_inputs_capacity_total_) {
        // We have finished adding the last input for all phis.
        inputs_per_phi_++;
        next_phi_input_add_offset_ = inputs_per_phi_;
#ifdef DEBUG
        EnsureAllPhisHaveSameInputCount();
#endif
      }
    }
 
    uint32_t phi_count() const { return phi_count_; }
 
    ValueType phi_type(size_t phi_i) const { return phi_types_[phi_i]; }
 
    base::Vector<const OpIndex> phi_inputs(size_t phi_i) const {
      size_t phi_inputs_start = phi_i * input_capacity_per_phi_;
      return base::VectorOf(&phi_inputs_[phi_inputs_start], inputs_per_phi_);
    }
 
    void AddIncomingException(OpIndex exception) {
      incoming_exceptions_.push_back(exception);
    }
 
    base::Vector<const OpIndex> incoming_exceptions() const {
      return base::VectorOf(incoming_exceptions_);
    }
 
#if DEBUG
    void DcheckConsistency() { EnsureAllPhisHaveSameInputCount(); }
#endif
 
   private:
    // Invariants:
    // The number of phis for a given block (e.g., locals, merged stack values,
    // and cached instance fields) is known when constructing the `BlockPhis`
    // and doesn't grow afterwards.
    // The number of _inputs_ for each phi is however _not_ yet known when
    // constructing this, but grows over time as new incoming edges for a given
    // block are created.
    // After such an edge is created, each phi has the same number of inputs.
    // When eventually creating a phi, we also need all inputs layed out
    // contiguously.
    // Due to those requirements, we write our own little container, see below.
 
    // First the backing storage:
    // Of size `phi_count_`, one type per phi.
    ValueType* phi_types_ = nullptr;
    // Of size `phi_inputs_capacity_total_ == phi_count_ *
    // input_capacity_per_phi_`, of which `phi_inputs_total_ == phi_count_ *
    // inputs_per_phi_` are set/initialized. All inputs for a given phi are
    // stored contiguously, but between them are uninitialized elements for
    // adding new inputs without reallocating.
    OpIndex* phi_inputs_ = nullptr;
 
    // Stored explicitly to save multiplications in the hot `AddInputForPhi()`.
    // Also pulled up to be in the same cache-line as `phi_inputs_`.
    uint32_t phi_inputs_capacity_total_ = 0;  // Updated with `phi_inputs_`.
    uint32_t phi_inputs_total_ = 0;
    uint32_t next_phi_input_add_offset_ = 0;
 
    // The dimensions.
    uint32_t phi_count_ = 0;
    uint32_t inputs_per_phi_ = 0;
    static constexpr uint32_t kInitialInputCapacityPerPhi = 2;
    uint32_t input_capacity_per_phi_ = kInitialInputCapacityPerPhi;
 
#ifdef DEBUG
    std::vector<uint32_t> input_count_per_phi_;
    void EnsureAllPhisHaveSameInputCount() const {
      CHECK_EQ(phi_inputs_total_, phi_count() * inputs_per_phi_);
      CHECK_EQ(phi_count(), input_count_per_phi_.size());
      CHECK(std::all_of(input_count_per_phi_.begin(),
                        input_count_per_phi_.end(),
                        [=, this](uint32_t input_count) {
                          return input_count == inputs_per_phi_;
                        }));
    }
#endif
 
    // The number of `incoming_exceptions` is also not known when constructing
    // the block, but at least it is only one-dimensional, so we can use a
    // simple `ZoneVector`.
    ZoneVector<OpIndex> incoming_exceptions_;
 
    Zone* zone() { return incoming_exceptions_.zone(); }
 
    V8_NOINLINE V8_PRESERVE_MOST void GrowInputsVector() {
      // We should have always initialized some storage, see
      // `kInitialInputCapacityPerPhi`.
      DCHECK_NOT_NULL(phi_inputs_);
      DCHECK_NE(phi_inputs_capacity_total_, 0);
 
      OpIndex* old_phi_inputs = phi_inputs_;
      uint32_t old_input_capacity_per_phi = input_capacity_per_phi_;
      uint32_t old_phi_inputs_capacity_total = phi_inputs_capacity_total_;
 
      input_capacity_per_phi_ *= 2;
      phi_inputs_capacity_total_ *= 2;
      phi_inputs_ = zone()->AllocateArray<OpIndex>(phi_inputs_capacity_total_);
 
      // This is essentially a strided copy, where we expand the storage by
      // "inserting" unitialized elements in between contiguous stretches of
      // inputs belonging to the same phi.
#ifdef DEBUG
      EnsureAllPhisHaveSameInputCount();
#endif
      for (size_t phi_i = 0; phi_i < phi_count(); ++phi_i) {
        const OpIndex* old_begin =
            &old_phi_inputs[phi_i * old_input_capacity_per_phi];
        const OpIndex* old_end = old_begin + inputs_per_phi_;
        OpIndex* begin = &phi_inputs_[phi_i * input_capacity_per_phi_];
        std::uninitialized_copy(old_begin, old_end, begin);
      }
 
      zone()->DeleteArray(old_phi_inputs, old_phi_inputs_capacity_total);
    }
  };
 
  // Perform a null check if the input type is nullable.
  V<Object> NullCheck(const Value& value,
                      TrapId trap_id = TrapId::kTrapNullDereference) {
    V<Object> not_null_value = V<Object>::Cast(value.op);
    if (value.type.is_nullable()) {
      not_null_value = __ AssertNotNull(value.op, value.type, trap_id);
    }
    return not_null_value;
  }
 
  // Creates a new block, initializes a {BlockPhis} for it, and registers it
  // with block_phis_. We pass a {merge} only if we later need to recover values
  // for that merge.
  TSBlock* NewBlockWithPhis(FullDecoder* decoder, Merge<Value>* merge) {
    TSBlock* block = __ NewBlock();
    block_phis_.emplace(block, BlockPhis(decoder, merge));
    return block;
  }
 
  // Sets up a control flow edge from the current SSA environment and a stack to
  // {block}. The stack is {stack_values} if present, otherwise the current
  // decoder stack.
  void SetupControlFlowEdge(FullDecoder* decoder, TSBlock* block,
                            uint32_t drop_values = 0,
                            V<Object> exception = OpIndex::Invalid(),
                            Merge<Value>* stack_values = nullptr) {
    if (__ current_block() == nullptr) return;
    // It is guaranteed that this element exists.
    DCHECK_NE(block_phis_.find(block), block_phis_.end());
    BlockPhis& phis_for_block = block_phis_.find(block)->second;
    uint32_t merge_arity = static_cast<uint32_t>(phis_for_block.phi_count()) -
                           decoder->num_locals();
 
    for (size_t i = 0; i < ssa_env_.size(); i++) {
      phis_for_block.AddInputForPhi(i, ssa_env_[i]);
    }
    // We never drop values from an explicit merge.
    DCHECK_IMPLIES(stack_values != nullptr, drop_values == 0);
    Value* stack_base = merge_arity == 0 ? nullptr
                        : stack_values != nullptr
                            ? &(*stack_values)[0]
                            : decoder->stack_value(merge_arity + drop_values);
    for (size_t i = 0; i < merge_arity; i++) {
      DCHECK(stack_base[i].op.valid());
      phis_for_block.AddInputForPhi(decoder->num_locals() + i,
                                    stack_base[i].op);
    }
    if (exception.valid()) {
      phis_for_block.AddIncomingException(exception);
    }
  }
 
  OpIndex MaybePhi(base::Vector<const OpIndex> elements, ValueType type) {
    if (elements.empty()) return OpIndex::Invalid();
    for (size_t i = 1; i < elements.size(); i++) {
      if (elements[i] != elements[0]) {
        return __ Phi(elements, RepresentationFor(type));
      }
    }
    return elements[0];
  }
 
  // Binds a block, initializes phis for its SSA environment from its entry in
  // {block_phis_}, and sets values to its {merge} (if available) from the
  // its entry in {block_phis_}.
  void BindBlockAndGeneratePhis(FullDecoder* decoder, TSBlock* tsblock,
                                Merge<Value>* merge,
                                OpIndex* exception = nullptr) {
    __ Bind(tsblock);
    auto block_phis_it = block_phis_.find(tsblock);
    DCHECK_NE(block_phis_it, block_phis_.end());
    BlockPhis& block_phis = block_phis_it->second;
 
    uint32_t merge_arity = merge != nullptr ? merge->arity : 0;
    DCHECK_EQ(decoder->num_locals() + merge_arity, block_phis.phi_count());
 
#ifdef DEBUG
    // Check consistency of Phi storage. We do this here rather than inside
    // {block_phis.phi_inputs()} to avoid overall O(n²) complexity.
    block_phis.DcheckConsistency();
#endif
 
    for (uint32_t i = 0; i < decoder->num_locals(); i++) {
      ssa_env_[i] = MaybePhi(block_phis.phi_inputs(i), block_phis.phi_type(i));
    }
    for (uint32_t i = 0; i < merge_arity; i++) {
      uint32_t phi_index = decoder->num_locals() + i;
      (*merge)[i].op = MaybePhi(block_phis.phi_inputs(phi_index),
                                block_phis.phi_type(phi_index));
    }
    DCHECK_IMPLIES(exception == nullptr,
                   block_phis.incoming_exceptions().empty());
    if (exception != nullptr && !exception->valid()) {
      *exception = MaybePhi(block_phis.incoming_exceptions(), kWasmExternRef);
    }
    block_phis_.erase(block_phis_it);
  }
 
  V<Any> DefaultValue(ValueType type) {
    switch (type.kind()) {
      case kI8:
      case kI16:
      case kI32:
        return __ Word32Constant(int32_t{0});
      case kI64:
        return __ Word64Constant(int64_t{0});
      case kF16:
      case kF32:
        return __ Float32Constant(0.0f);
      case kF64:
        return __ Float64Constant(0.0);
      case kRefNull:
        return __ Null(type);
      case kS128: {
        uint8_t value[kSimd128Size] = {};
        return __ Simd128Constant(value);
      }
      case kVoid:
      case kRef:
      case kTop:
      case kBottom:
        UNREACHABLE();
    }
  }
 
 private:
  V<FrameState> CreateFrameState(FullDecoder* decoder,
                                 const FunctionSig* callee_sig,
                                 const Value* func_ref_or_index,
                                 const Value args[]) {
    compiler::turboshaft::FrameStateData::Builder builder;
    if (parent_frame_state_.valid()) {
      builder.AddParentFrameState(parent_frame_state_.value());
    }
    // The first input is the closure for JS. (The instruction selector will
    // just skip this input as the liftoff frame doesn't have a closure.)
    V<Object> dummy_tagged = __ SmiZeroConstant();
    builder.AddInput(MachineType::AnyTagged(), dummy_tagged);
    // Add the parameters.
    size_t param_count = decoder->sig_->parameter_count();
    for (size_t i = 0; i < param_count; ++i) {
      builder.AddInput(decoder->sig_->GetParam(i).machine_type(), ssa_env_[i]);
    }
    // Add the context. Wasm doesn't have a JS context, so this is another
    // value skipped by the instruction selector.
    builder.AddInput(MachineType::AnyTagged(), dummy_tagged);
 
    // Add the wasm locals.
    for (size_t i = param_count; i < ssa_env_.size(); ++i) {
      builder.AddInput(
          decoder->local_type(static_cast<uint32_t>(i)).machine_type(),
          ssa_env_[i]);
    }
    // Add the wasm stack values.
    // Note that the decoder stack is already in the state after the call, i.e.
    // the callee and the arguments were already popped from the stack and the
    // returns are pushed. Therefore skip the results and manually add the
    // call_ref stack values.
    for (int32_t i = decoder->stack_size();
         i > static_cast<int32_t>(callee_sig->return_count()); --i) {
      Value* val = decoder->stack_value(i);
      builder.AddInput(val->type.machine_type(), val->op);
    }
    // Add the call_ref or call_indirect stack values.
    if (args != nullptr) {
      for (const Value& arg :
           base::VectorOf(args, callee_sig->parameter_count())) {
        builder.AddInput(arg.type.machine_type(), arg.op);
      }
    }
    if (func_ref_or_index) {
      builder.AddInput(func_ref_or_index->type.machine_type(),
                       func_ref_or_index->op);
    }
    // The call_ref (callee) or the table index.
    const size_t kExtraLocals = func_ref_or_index != nullptr ? 1 : 0;
    size_t wasm_local_count = ssa_env_.size() - param_count;
    size_t local_count = kExtraLocals + decoder->stack_size() +
                         wasm_local_count - callee_sig->return_count();
    local_count += args != nullptr ? callee_sig->parameter_count() : 0;
    Zone* zone = Asm().data()->compilation_zone();
    auto* function_info = zone->New<compiler::FrameStateFunctionInfo>(
        compiler::FrameStateType::kLiftoffFunction,
        static_cast<uint16_t>(param_count), 0, static_cast<int>(local_count),
        IndirectHandle<SharedFunctionInfo>(), kNullMaybeHandle,
        GetLiftoffFrameSize(decoder), func_index_);
    auto* frame_state_info = zone->New<compiler::FrameStateInfo>(
        BytecodeOffset(decoder->pc_offset()),
        compiler::OutputFrameStateCombine::Ignore(), function_info);
 
    // Limit the maximum amount of inputs to a deopt node to a reasonably low
    // value. For each of these values extra metadata needs to be stored for the
    // deoptimizer while the performance upside of a deoptimization node doesn't
    // scale with the amount of inputs, so there are diminishing returns.
    constexpr size_t max_deopt_input_count = 500;
    // We don't want to include the instruction selector just for this limit.
    constexpr size_t max_isel_input_count =
        std::numeric_limits<uint16_t>::max();
    // Note that the instruction selector has limits on the maximum amount of
    // inputs for an instruction while nested FrameStates are unfolded, so the
    // total number of inputs after unfolding needs to be lower than
    // uint16_t::max (adding a random 42 here for additional "lower-level"
    // inputs that might not be accounted for in the FrameState inputs.)
    // The number of inputs might be doubled by the Int64Lowering on 32 bit
    // platforms.
    static_assert((max_deopt_input_count * 2 + 42) *
                      InliningTree::kMaxInliningNestingDepth <
                  max_isel_input_count);
    if (builder.Inputs().size() >= max_deopt_input_count) {
      if (v8_flags.trace_wasm_inlining) {
        PrintF(
            "[function %d%s: Disabling deoptimizations for speculative "
            "inlining as the deoptimization FrameState takes too many inputs "
            "(%zu vs. %zu)]\n",
            func_index_, mode_ == kRegular ? "" : " (inlined)",
            builder.Inputs().size(), max_deopt_input_count);
      }
      // If there are too many inputs, disable deopts completely for the rest of
      // the function as the FrameState could be needed by other deoptimization
      // points in case of nested inlining.
      disable_deopts();
      return OpIndex::Invalid();
    }
 
    return __ FrameState(
        builder.Inputs(), builder.inlined(),
        builder.AllocateFrameStateData(*frame_state_info, zone));
  }
 
  void DeoptIfNot(FullDecoder* decoder, OpIndex deopt_condition,
                  V<FrameState> frame_state) {
    CHECK(deopts_enabled());
    DCHECK(frame_state.valid());
    __ DeoptimizeIfNot(deopt_condition, frame_state,
                       DeoptimizeReason::kWrongCallTarget,
                       compiler::FeedbackSource());
  }
 
  void Deopt(FullDecoder* decoder, V<FrameState> frame_state) {
    CHECK(deopts_enabled());
    DCHECK(frame_state.valid());
    __ Deoptimize(frame_state, DeoptimizeReason::kWrongCallTarget,
                  compiler::FeedbackSource());
  }
 
  uint32_t GetLiftoffFrameSize(const FullDecoder* decoder) {
    if (liftoff_frame_size_ !=
        FunctionTypeFeedback::kUninitializedLiftoffFrameSize) {
      return liftoff_frame_size_;
    }
    const TypeFeedbackStorage& feedback = decoder->module_->type_feedback;
    base::MutexGuard mutex_guard(&feedback.mutex);
    auto function_feedback = feedback.feedback_for_function.find(func_index_);
    CHECK_NE(function_feedback, feedback.feedback_for_function.end());
    liftoff_frame_size_ = function_feedback->second.liftoff_frame_size;
    // The liftoff frame size is strictly required. If it is not properly set,
    // calling the function embedding the deopt node will always fail on the
    // stack check.
    CHECK_NE(liftoff_frame_size_,
             FunctionTypeFeedback::kUninitializedLiftoffFrameSize);
    return liftoff_frame_size_;
  }
 
  V<Word64> ExtractTruncationProjections(V<Tuple<Word64, Word32>> truncated) {
    V<Word64> result = __ template Projection<0>(truncated);
    V<Word32> check = __ template Projection<1>(truncated);
    __ TrapIf(__ Word32Equal(check, 0), TrapId::kTrapFloatUnrepresentable);
    return result;
  }
 
  std::pair<OpIndex, V<Word32>> BuildCCallForFloatConversion(
      OpIndex arg, MemoryRepresentation float_type,
      ExternalReference ccall_ref) {
    uint8_t slot_size = MemoryRepresentation::Int64().SizeInBytes();
    V<WordPtr> stack_slot = __ StackSlot(slot_size, slot_size);
    __ Store(stack_slot, arg, StoreOp::Kind::RawAligned(), float_type,
             compiler::WriteBarrierKind::kNoWriteBarrier);
    MachineType reps[]{MachineType::Int32(), MachineType::Pointer()};
    MachineSignature sig(1, 1, reps);
    V<Word32> overflow = CallC(&sig, ccall_ref, stack_slot);
    return {stack_slot, overflow};
  }
 
  OpIndex BuildCcallConvertFloat(OpIndex arg, MemoryRepresentation float_type,
                                 ExternalReference ccall_ref) {
    auto [stack_slot, overflow] =
        BuildCCallForFloatConversion(arg, float_type, ccall_ref);
    __ TrapIf(__ Word32Equal(overflow, 0),
              compiler::TrapId::kTrapFloatUnrepresentable);
    MemoryRepresentation int64 = MemoryRepresentation::Int64();
    return __ Load(stack_slot, LoadOp::Kind::RawAligned(), int64);
  }
 
  OpIndex BuildCcallConvertFloatSat(OpIndex arg,
                                    MemoryRepresentation float_type,
                                    ExternalReference ccall_ref,
                                    bool is_signed) {
    MemoryRepresentation int64 = MemoryRepresentation::Int64();
    uint8_t slot_size = int64.SizeInBytes();
    V<WordPtr> stack_slot = __ StackSlot(slot_size, slot_size);
    __ Store(stack_slot, arg, StoreOp::Kind::RawAligned(), float_type,
             compiler::WriteBarrierKind::kNoWriteBarrier);
    MachineType reps[]{MachineType::Pointer()};
    MachineSignature sig(0, 1, reps);
    CallC(&sig, ccall_ref, stack_slot);
    return __ Load(stack_slot, LoadOp::Kind::RawAligned(), int64);
  }
 
  OpIndex BuildIntToFloatConversionInstruction(
      OpIndex input, ExternalReference ccall_ref,
      MemoryRepresentation input_representation,
      MemoryRepresentation result_representation) {
    uint8_t slot_size = std::max(input_representation.SizeInBytes(),
                                 result_representation.SizeInBytes());
    V<WordPtr> stack_slot = __ StackSlot(slot_size, slot_size);
    __ Store(stack_slot, input, StoreOp::Kind::RawAligned(),
             input_representation, compiler::WriteBarrierKind::kNoWriteBarrier);
    MachineType reps[]{MachineType::Pointer()};
    MachineSignature sig(0, 1, reps);
    CallC(&sig, ccall_ref, stack_slot);
    return __ Load(stack_slot, LoadOp::Kind::RawAligned(),
                   result_representation);
  }
 
  OpIndex BuildDiv64Call(OpIndex lhs, OpIndex rhs, ExternalReference ccall_ref,
                         wasm::TrapId trap_zero) {
    MemoryRepresentation int64_rep = MemoryRepresentation::Int64();
    V<WordPtr> stack_slot =
        __ StackSlot(2 * int64_rep.SizeInBytes(), int64_rep.SizeInBytes());
    __ Store(stack_slot, lhs, StoreOp::Kind::RawAligned(), int64_rep,
             compiler::WriteBarrierKind::kNoWriteBarrier);
    __ Store(stack_slot, rhs, StoreOp::Kind::RawAligned(), int64_rep,
             compiler::WriteBarrierKind::kNoWriteBarrier,
             int64_rep.SizeInBytes());
 
    MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
    MachineSignature sig(1, 1, sig_types);
    OpIndex rc = CallC(&sig, ccall_ref, stack_slot);
    __ TrapIf(__ Word32Equal(rc, 0), trap_zero);
    __ TrapIf(__ Word32Equal(rc, -1), TrapId::kTrapDivUnrepresentable);
    return __ Load(stack_slot, LoadOp::Kind::RawAligned(), int64_rep);
  }
 
  OpIndex UnOpImpl(WasmOpcode opcode, OpIndex arg,
                   ValueType input_type /* for ref.is_null only*/) {
    switch (opcode) {
      case kExprI32Eqz:
        return __ Word32Equal(arg, 0);
      case kExprF32Abs:
        return __ Float32Abs(arg);
      case kExprF32Neg:
        return __ Float32Negate(arg);
      case kExprF32Sqrt:
        return __ Float32Sqrt(arg);
      case kExprF64Abs:
        return __ Float64Abs(arg);
      case kExprF64Neg:
        return __ Float64Negate(arg);
      case kExprF64Sqrt:
        return __ Float64Sqrt(arg);
      case kExprI32SConvertF32: {
        V<Float32> truncated = UnOpImpl(kExprF32Trunc, arg, kWasmF32);
        V<Word32> result = __ TruncateFloat32ToInt32OverflowToMin(truncated);
        V<Float32> converted_back = __ ChangeInt32ToFloat32(result);
        __ TrapIf(__ Word32Equal(__ Float32Equal(converted_back, truncated), 0),
                  TrapId::kTrapFloatUnrepresentable);
        return result;
      }
      case kExprI32UConvertF32: {
        V<Float32> truncated = UnOpImpl(kExprF32Trunc, arg, kWasmF32);
        V<Word32> result = __ TruncateFloat32ToUint32OverflowToMin(truncated);
        V<Float32> converted_back = __ ChangeUint32ToFloat32(result);
        __ TrapIf(__ Word32Equal(__ Float32Equal(converted_back, truncated), 0),
                  TrapId::kTrapFloatUnrepresentable);
        return result;
      }
      case kExprI32SConvertF64: {
        V<Float64> truncated = UnOpImpl(kExprF64Trunc, arg, kWasmF64);
        V<Word32> result =
            __ TruncateFloat64ToInt32OverflowUndefined(truncated);
        V<Float64> converted_back = __ ChangeInt32ToFloat64(result);
        __ TrapIf(__ Word32Equal(__ Float64Equal(converted_back, truncated), 0),
                  TrapId::kTrapFloatUnrepresentable);
        return result;
      }
      case kExprI32UConvertF64: {
        V<Float64> truncated = UnOpImpl(kExprF64Trunc, arg, kWasmF64);
        V<Word32> result = __ TruncateFloat64ToUint32OverflowToMin(truncated);
        V<Float64> converted_back = __ ChangeUint32ToFloat64(result);
        __ TrapIf(__ Word32Equal(__ Float64Equal(converted_back, truncated), 0),
                  TrapId::kTrapFloatUnrepresentable);
        return result;
      }
      case kExprI64SConvertF32:
        return Is64() ? ExtractTruncationProjections(
                            __ TryTruncateFloat32ToInt64(arg))
                      : BuildCcallConvertFloat(
                            arg, MemoryRepresentation::Float32(),
                            ExternalReference::wasm_float32_to_int64());
      case kExprI64UConvertF32:
        return Is64() ? ExtractTruncationProjections(
                            __ TryTruncateFloat32ToUint64(arg))
                      : BuildCcallConvertFloat(
                            arg, MemoryRepresentation::Float32(),
                            ExternalReference::wasm_float32_to_uint64());
      case kExprI64SConvertF64:
        return Is64() ? ExtractTruncationProjections(
                            __ TryTruncateFloat64ToInt64(arg))
                      : BuildCcallConvertFloat(
                            arg, MemoryRepresentation::Float64(),
                            ExternalReference::wasm_float64_to_int64());
      case kExprI64UConvertF64:
        return Is64() ? ExtractTruncationProjections(
                            __ TryTruncateFloat64ToUint64(arg))
                      : BuildCcallConvertFloat(
                            arg, MemoryRepresentation::Float64(),
                            ExternalReference::wasm_float64_to_uint64());
      case kExprF64SConvertI32:
        return __ ChangeInt32ToFloat64(arg);
      case kExprF64UConvertI32:
        return __ ChangeUint32ToFloat64(arg);
      case kExprF32SConvertI32:
        return __ ChangeInt32ToFloat32(arg);
      case kExprF32UConvertI32:
        return __ ChangeUint32ToFloat32(arg);
      case kExprI32SConvertSatF32: {
        V<Float32> truncated = UnOpImpl(kExprF32Trunc, arg, kWasmF32);
        V<Word32> converted =
            __ TruncateFloat32ToInt32OverflowUndefined(truncated);
        V<Float32> converted_back = __ ChangeInt32ToFloat32(converted);
 
        Label<Word32> done(&asm_);
 
        IF (LIKELY(__ Float32Equal(truncated, converted_back))) {
          GOTO(done, converted);
        } ELSE {
          // Overflow.
          IF (__ Float32Equal(arg, arg)) {
            // Not NaN.
            IF (__ Float32LessThan(arg, 0)) {
              // Negative arg.
              GOTO(done,
                   __ Word32Constant(std::numeric_limits<int32_t>::min()));
            } ELSE {
              // Positive arg.
              GOTO(done,
                   __ Word32Constant(std::numeric_limits<int32_t>::max()));
            }
          } ELSE {
            // NaN.
            GOTO(done, __ Word32Constant(0));
          }
        }
        BIND(done, result);
 
        return result;
      }
      case kExprI32UConvertSatF32: {
        V<Float32> truncated = UnOpImpl(kExprF32Trunc, arg, kWasmF32);
        V<Word32> converted =
            __ TruncateFloat32ToUint32OverflowUndefined(truncated);
        V<Float32> converted_back = __ ChangeUint32ToFloat32(converted);
 
        Label<Word32> done(&asm_);
 
        IF (LIKELY(__ Float32Equal(truncated, converted_back))) {
          GOTO(done, converted);
        } ELSE {
          // Overflow.
          IF (__ Float32Equal(arg, arg)) {
            // Not NaN.
            IF (__ Float32LessThan(arg, 0)) {
              // Negative arg.
              GOTO(done, __ Word32Constant(0));
            } ELSE {
              // Positive arg.
              GOTO(done,
                   __ Word32Constant(std::numeric_limits<uint32_t>::max()));
            }
          } ELSE {
            // NaN.
            GOTO(done, __ Word32Constant(0));
          }
        }
        BIND(done, result);
 
        return result;
      }
      case kExprI32SConvertSatF64: {
        V<Float64> truncated = UnOpImpl(kExprF64Trunc, arg, kWasmF64);
        V<Word32> converted =
            __ TruncateFloat64ToInt32OverflowUndefined(truncated);
        V<Float64> converted_back = __ ChangeInt32ToFloat64(converted);
 
        Label<Word32> done(&asm_);
 
        IF (LIKELY(__ Float64Equal(truncated, converted_back))) {
          GOTO(done, converted);
        } ELSE {
          // Overflow.
          IF (__ Float64Equal(arg, arg)) {
            // Not NaN.
            IF (__ Float64LessThan(arg, 0)) {
              // Negative arg.
              GOTO(done,
                   __ Word32Constant(std::numeric_limits<int32_t>::min()));
            } ELSE {
              // Positive arg.
              GOTO(done,
                   __ Word32Constant(std::numeric_limits<int32_t>::max()));
            }
          } ELSE {
            // NaN.
            GOTO(done, __ Word32Constant(0));
          }
        }
        BIND(done, result);
 
        return result;
      }
      case kExprI32UConvertSatF64: {
        V<Float64> truncated = UnOpImpl(kExprF64Trunc, arg, kWasmF64);
        V<Word32> converted =
            __ TruncateFloat64ToUint32OverflowUndefined(truncated);
        V<Float64> converted_back = __ ChangeUint32ToFloat64(converted);
 
        Label<Word32> done(&asm_);
 
        IF (LIKELY(__ Float64Equal(truncated, converted_back))) {
          GOTO(done, converted);
        } ELSE {
          // Overflow.
          IF (__ Float64Equal(arg, arg)) {
            // Not NaN.
            IF (__ Float64LessThan(arg, 0)) {
              // Negative arg.
              GOTO(done, __ Word32Constant(0));
            } ELSE {
              // Positive arg.
              GOTO(done,
                   __ Word32Constant(std::numeric_limits<uint32_t>::max()));
            }
          } ELSE {
            // NaN.
            GOTO(done, __ Word32Constant(0));
          }
        }
        BIND(done, result);
 
        return result;
      }
      case kExprI64SConvertSatF32: {
        if constexpr (!Is64()) {
          bool is_signed = true;
          return BuildCcallConvertFloatSat(
              arg, MemoryRepresentation::Float32(),
              ExternalReference::wasm_float32_to_int64_sat(), is_signed);
        }
        V<Tuple<Word64, Word32>> converted = __ TryTruncateFloat32ToInt64(arg);
        Label<compiler::turboshaft::Word64> done(&asm_);
 
        if (SupportedOperations::sat_conversion_is_safe()) {
          return __ Projection<0>(converted);
        }
        IF (LIKELY(__ Projection<1>(converted))) {
          GOTO(done, __ Projection<0>(converted));
        } ELSE {
          // Overflow.
          IF (__ Float32Equal(arg, arg)) {
            // Not NaN.
            IF (__ Float32LessThan(arg, 0)) {
              // Negative arg.
              GOTO(done,
                   __ Word64Constant(std::numeric_limits<int64_t>::min()));
            } ELSE {
              // Positive arg.
              GOTO(done,
                   __ Word64Constant(std::numeric_limits<int64_t>::max()));
            }
          } ELSE {
            // NaN.
            GOTO(done, __ Word64Constant(int64_t{0}));
          }
        }
        BIND(done, result);
 
        return result;
      }
      case kExprI64UConvertSatF32: {
        if constexpr (!Is64()) {
          bool is_signed = false;
          return BuildCcallConvertFloatSat(
              arg, MemoryRepresentation::Float32(),
              ExternalReference::wasm_float32_to_uint64_sat(), is_signed);
        }
        V<Tuple<Word64, Word32>> converted = __ TryTruncateFloat32ToUint64(arg);
        Label<compiler::turboshaft::Word64> done(&asm_);
 
        if (SupportedOperations::sat_conversion_is_safe()) {
          return __ template Projection<0>(converted);
        }
 
        IF (LIKELY(__ template Projection<1>(converted))) {
          GOTO(done, __ template Projection<0>(converted));
        } ELSE {
          // Overflow.
          IF (__ Float32Equal(arg, arg)) {
            // Not NaN.
            IF (__ Float32LessThan(arg, 0)) {
              // Negative arg.
              GOTO(done, __ Word64Constant(int64_t{0}));
            } ELSE {
              // Positive arg.
              GOTO(done,
                   __ Word64Constant(std::numeric_limits<uint64_t>::max()));
            }
          } ELSE {
            // NaN.
            GOTO(done, __ Word64Constant(int64_t{0}));
          }
        }
        BIND(done, result);
 
        return result;
      }
      case kExprI64SConvertSatF64: {
        if constexpr (!Is64()) {
          bool is_signed = true;
          return BuildCcallConvertFloatSat(
              arg, MemoryRepresentation::Float64(),
              ExternalReference::wasm_float64_to_int64_sat(), is_signed);
        }
        V<Tuple<Word64, Word32>> converted = __ TryTruncateFloat64ToInt64(arg);
        Label<compiler::turboshaft::Word64> done(&asm_);
 
        if (SupportedOperations::sat_conversion_is_safe()) {
          return __ template Projection<0>(converted);
        }
 
        IF (LIKELY(__ template Projection<1>(converted))) {
          GOTO(done, __ template Projection<0>(converted));
        } ELSE {
          // Overflow.
          IF (__ Float64Equal(arg, arg)) {
            // Not NaN.
            IF (__ Float64LessThan(arg, 0)) {
              // Negative arg.
              GOTO(done,
                   __ Word64Constant(std::numeric_limits<int64_t>::min()));
            } ELSE {
              // Positive arg.
              GOTO(done,
                   __ Word64Constant(std::numeric_limits<int64_t>::max()));
            }
          } ELSE {
            // NaN.
            GOTO(done, __ Word64Constant(int64_t{0}));
          }
        }
        BIND(done, result);
 
        return result;
      }
      case kExprI64UConvertSatF64: {
        if constexpr (!Is64()) {
          bool is_signed = false;
          return BuildCcallConvertFloatSat(
              arg, MemoryRepresentation::Float64(),
              ExternalReference::wasm_float64_to_uint64_sat(), is_signed);
        }
        V<Tuple<Word64, Word32>> converted = __ TryTruncateFloat64ToUint64(arg);
        Label<compiler::turboshaft::Word64> done(&asm_);
 
        if (SupportedOperations::sat_conversion_is_safe()) {
          return __ template Projection<0>(converted);
        }
 
        IF (LIKELY(__ template Projection<1>(converted))) {
          GOTO(done, __ template Projection<0>(converted));
        } ELSE {
          // Overflow.
          IF (__ Float64Equal(arg, arg)) {
            // Not NaN.
            IF (__ Float64LessThan(arg, 0)) {
              // Negative arg.
              GOTO(done, __ Word64Constant(int64_t{0}));
            } ELSE {
              // Positive arg.
              GOTO(done,
                   __ Word64Constant(std::numeric_limits<uint64_t>::max()));
            }
          } ELSE {
            // NaN.
            GOTO(done, __ Word64Constant(int64_t{0}));
          }
        }
        BIND(done, result);
 
        return result;
      }
      case kExprF32ConvertF64:
        return __ TruncateFloat64ToFloat32(arg);
      case kExprF64ConvertF32:
        return __ ChangeFloat32ToFloat64(arg);
      case kExprF32ReinterpretI32:
        return __ BitcastWord32ToFloat32(arg);
      case kExprI32ReinterpretF32:
        return __ BitcastFloat32ToWord32(arg);
      case kExprI32Clz:
        return __ Word32CountLeadingZeros(arg);
      case kExprI32Ctz:
        if (SupportedOperations::word32_ctz()) {
          return __ Word32CountTrailingZeros(arg);
        } else {
          // TODO(14108): Use reverse_bits if supported.
          auto sig =
              FixedSizeSignature<MachineType>::Returns(MachineType::Uint32())
                  .Params(MachineType::Uint32());
          return CallC(&sig, ExternalReference::wasm_word32_ctz(), arg);
        }
      case kExprI32Popcnt:
        if (SupportedOperations::word32_popcnt()) {
          return __ Word32PopCount(arg);
        } else {
          auto sig =
              FixedSizeSignature<MachineType>::Returns(MachineType::Uint32())
                  .Params(MachineType::Uint32());
          return CallC(&sig, ExternalReference::wasm_word32_popcnt(), arg);
        }
      case kExprF32Floor:
        if (SupportedOperations::float32_round_down()) {
          return __ Float32RoundDown(arg);
        } else {
          return CallCStackSlotToStackSlot(arg,
                                           ExternalReference::wasm_f32_floor(),
                                           MemoryRepresentation::Float32());
        }
      case kExprF32Ceil:
        if (SupportedOperations::float32_round_up()) {
          return __ Float32RoundUp(arg);
        } else {
          return CallCStackSlotToStackSlot(arg,
                                           ExternalReference::wasm_f32_ceil(),
                                           MemoryRepresentation::Float32());
        }
      case kExprF32Trunc:
        if (SupportedOperations::float32_round_to_zero()) {
          return __ Float32RoundToZero(arg);
        } else {
          return CallCStackSlotToStackSlot(arg,
                                           ExternalReference::wasm_f32_trunc(),
                                           MemoryRepresentation::Float32());
        }
      case kExprF32NearestInt:
        if (SupportedOperations::float32_round_ties_even()) {
          return __ Float32RoundTiesEven(arg);
        } else {
          return CallCStackSlotToStackSlot(
              arg, ExternalReference::wasm_f32_nearest_int(),
              MemoryRepresentation::Float32());
        }
      case kExprF64Floor:
        if (SupportedOperations::float64_round_down()) {
          return __ Float64RoundDown(arg);
        } else {
          return CallCStackSlotToStackSlot(arg,
                                           ExternalReference::wasm_f64_floor(),
                                           MemoryRepresentation::Float64());
        }
      case kExprF64Ceil:
        if (SupportedOperations::float64_round_up()) {
          return __ Float64RoundUp(arg);
        } else {
          return CallCStackSlotToStackSlot(arg,
                                           ExternalReference::wasm_f64_ceil(),
                                           MemoryRepresentation::Float64());
        }
      case kExprF64Trunc:
        if (SupportedOperations::float64_round_to_zero()) {
          return __ Float64RoundToZero(arg);
        } else {
          return CallCStackSlotToStackSlot(arg,
                                           ExternalReference::wasm_f64_trunc(),
                                           MemoryRepresentation::Float64());
        }
      case kExprF64NearestInt:
        if (SupportedOperations::float64_round_ties_even()) {
          return __ Float64RoundTiesEven(arg);
        } else {
          return CallCStackSlotToStackSlot(
              arg, ExternalReference::wasm_f64_nearest_int(),
              MemoryRepresentation::Float64());
        }
      case kExprF64Acos:
        return CallCStackSlotToStackSlot(
            arg, ExternalReference::f64_acos_wrapper_function(),
            MemoryRepresentation::Float64());
      case kExprF64Asin:
        return CallCStackSlotToStackSlot(
            arg, ExternalReference::f64_asin_wrapper_function(),
            MemoryRepresentation::Float64());
      case kExprF64Atan:
        return __ Float64Atan(arg);
      case kExprF64Cos:
        return __ Float64Cos(arg);
      case kExprF64Sin:
        return __ Float64Sin(arg);
      case kExprF64Tan:
        return __ Float64Tan(arg);
      case kExprF64Exp:
        return __ Float64Exp(arg);
      case kExprF64Log:
        return __ Float64Log(arg);
      case kExprI32ConvertI64:
        return __ TruncateWord64ToWord32(arg);
      case kExprI64SConvertI32:
        return __ ChangeInt32ToInt64(arg);
      case kExprI64UConvertI32:
        return __ ChangeUint32ToUint64(arg);
      case kExprF64ReinterpretI64:
        return __ BitcastWord64ToFloat64(arg);
      case kExprI64ReinterpretF64:
        return __ BitcastFloat64ToWord64(arg);
      case kExprI64Clz:
        return __ Word64CountLeadingZeros(arg);
      case kExprI64Ctz:
        if (SupportedOperations::word64_ctz() ||
            (!Is64() && SupportedOperations::word32_ctz())) {
          return __ Word64CountTrailingZeros(arg);
        } else if (Is64()) {
          // TODO(14108): Use reverse_bits if supported.
          auto sig =
              FixedSizeSignature<MachineType>::Returns(MachineType::Uint32())
                  .Params(MachineType::Uint64());
          return __ ChangeUint32ToUint64(
              CallC(&sig, ExternalReference::wasm_word64_ctz(), arg));
        } else {
          // lower_word == 0 ? 32 + CTZ32(upper_word) : CTZ32(lower_word);
          OpIndex upper_word =
              __ TruncateWord64ToWord32(__ Word64ShiftRightLogical(arg, 32));
          OpIndex lower_word = __ TruncateWord64ToWord32(arg);
          auto sig =
              FixedSizeSignature<MachineType>::Returns(MachineType::Uint32())
                  .Params(MachineType::Uint32());
          Label<Word32> done(&asm_);
          IF (__ Word32Equal(lower_word, 0)) {
            GOTO(done,
                 __ Word32Add(CallC(&sig, ExternalReference::wasm_word32_ctz(),
                                    upper_word),
                              32));
          } ELSE {
            GOTO(done,
                 CallC(&sig, ExternalReference::wasm_word32_ctz(), lower_word));
          }
          BIND(done, result);
          return __ ChangeUint32ToUint64(result);
        }
      case kExprI64Popcnt:
        if (SupportedOperations::word64_popcnt() ||
            (!Is64() && SupportedOperations::word32_popcnt())) {
          return __ Word64PopCount(arg);
        } else if (Is64()) {
          // Call wasm_word64_popcnt.
          auto sig =
              FixedSizeSignature<MachineType>::Returns(MachineType::Uint32())
                  .Params(MachineType::Uint64());
          return __ ChangeUint32ToUint64(
              CallC(&sig, ExternalReference::wasm_word64_popcnt(), arg));
        } else {
          // Emit two calls to wasm_word32_popcnt.
          OpIndex upper_word =
              __ TruncateWord64ToWord32(__ Word64ShiftRightLogical(arg, 32));
          OpIndex lower_word = __ TruncateWord64ToWord32(arg);
          auto sig =
              FixedSizeSignature<MachineType>::Returns(MachineType::Uint32())
                  .Params(MachineType::Uint32());
          return __ ChangeUint32ToUint64(__ Word32Add(
              CallC(&sig, ExternalReference::wasm_word32_popcnt(), lower_word),
              CallC(&sig, ExternalReference::wasm_word32_popcnt(),
                    upper_word)));
        }
      case kExprI64Eqz:
        return __ Word64Equal(arg, 0);
      case kExprF32SConvertI64:
        if constexpr (!Is64()) {
          return BuildIntToFloatConversionInstruction(
              arg, ExternalReference::wasm_int64_to_float32(),
              MemoryRepresentation::Int64(), MemoryRepresentation::Float32());
        }
        return __ ChangeInt64ToFloat32(arg);
      case kExprF32UConvertI64:
        if constexpr (!Is64()) {
          return BuildIntToFloatConversionInstruction(
              arg, ExternalReference::wasm_uint64_to_float32(),
              MemoryRepresentation::Uint64(), MemoryRepresentation::Float32());
        }
        return __ ChangeUint64ToFloat32(arg);
      case kExprF64SConvertI64:
        if constexpr (!Is64()) {
          return BuildIntToFloatConversionInstruction(
              arg, ExternalReference::wasm_int64_to_float64(),
              MemoryRepresentation::Int64(), MemoryRepresentation::Float64());
        }
        return __ ChangeInt64ToFloat64(arg);
      case kExprF64UConvertI64:
        if constexpr (!Is64()) {
          return BuildIntToFloatConversionInstruction(
              arg, ExternalReference::wasm_uint64_to_float64(),
              MemoryRepresentation::Uint64(), MemoryRepresentation::Float64());
        }
        return __ ChangeUint64ToFloat64(arg);
      case kExprI32SExtendI8:
        return __ Word32SignExtend8(arg);
      case kExprI32SExtendI16:
        return __ Word32SignExtend16(arg);
      case kExprI64SExtendI8:
        return __ Word64SignExtend8(arg);
      case kExprI64SExtendI16:
        return __ Word64SignExtend16(arg);
      case kExprI64SExtendI32:
        return __ ChangeInt32ToInt64(__ TruncateWord64ToWord32(arg));
      case kExprRefIsNull:
        return __ IsNull(arg, input_type);
      case kExprI32AsmjsLoadMem8S:
        return AsmjsLoadMem(arg, MemoryRepresentation::Int8());
      case kExprI32AsmjsLoadMem8U:
        return AsmjsLoadMem(arg, MemoryRepresentation::Uint8());
      case kExprI32AsmjsLoadMem16S:
        return AsmjsLoadMem(arg, MemoryRepresentation::Int16());
      case kExprI32AsmjsLoadMem16U:
        return AsmjsLoadMem(arg, MemoryRepresentation::Uint16());
      case kExprI32AsmjsLoadMem:
        return AsmjsLoadMem(arg, MemoryRepresentation::Int32());
      case kExprF32AsmjsLoadMem:
        return AsmjsLoadMem(arg, MemoryRepresentation::Float32());
      case kExprF64AsmjsLoadMem:
        return AsmjsLoadMem(arg, MemoryRepresentation::Float64());
      case kExprI32AsmjsSConvertF32:
      case kExprI32AsmjsUConvertF32:
        return __ JSTruncateFloat64ToWord32(__ ChangeFloat32ToFloat64(arg));
      case kExprI32AsmjsSConvertF64:
      case kExprI32AsmjsUConvertF64:
        return __ JSTruncateFloat64ToWord32(arg);
      case kExprRefAsNonNull:
        // We abuse ref.as_non_null, which isn't otherwise used in this switch,
        // as a sentinel for the negation of ref.is_null.
        return __ Word32Equal(__ IsNull(arg, input_type), 0);
      case kExprAnyConvertExtern:
        return __ AnyConvertExtern(arg);
      case kExprExternConvertAny:
        return __ ExternConvertAny(arg);
      default:
        UNREACHABLE();
    }
  }
 
  OpIndex BinOpImpl(WasmOpcode opcode, OpIndex lhs, OpIndex rhs) {
    switch (opcode) {
      case kExprI32Add:
        return __ Word32Add(lhs, rhs);
      case kExprI32Sub:
        return __ Word32Sub(lhs, rhs);
      case kExprI32Mul:
        return __ Word32Mul(lhs, rhs);
      case kExprI32DivS: {
        __ TrapIf(__ Word32Equal(rhs, 0), TrapId::kTrapDivByZero);
        V<Word32> unrepresentable_condition = __ Word32BitwiseAnd(
            __ Word32Equal(rhs, -1), __ Word32Equal(lhs, kMinInt));
        __ TrapIf(unrepresentable_condition, TrapId::kTrapDivUnrepresentable);
        return __ Int32Div(lhs, rhs);
      }
      case kExprI32DivU:
        __ TrapIf(__ Word32Equal(rhs, 0), TrapId::kTrapDivByZero);
        return __ Uint32Div(lhs, rhs);
      case kExprI32RemS: {
        __ TrapIf(__ Word32Equal(rhs, 0), TrapId::kTrapRemByZero);
        Label<Word32> done(&asm_);
        IF (UNLIKELY(__ Word32Equal(rhs, -1))) {
          GOTO(done, __ Word32Constant(0));
        } ELSE {
          GOTO(done, __ Int32Mod(lhs, rhs));
        };
 
        BIND(done, result);
        return result;
      }
      case kExprI32RemU:
        __ TrapIf(__ Word32Equal(rhs, 0), TrapId::kTrapRemByZero);
        return __ Uint32Mod(lhs, rhs);
      case kExprI32And:
        return __ Word32BitwiseAnd(lhs, rhs);
      case kExprI32Ior:
        return __ Word32BitwiseOr(lhs, rhs);
      case kExprI32Xor:
        return __ Word32BitwiseXor(lhs, rhs);
      case kExprI32Shl:
        // If possible, the bitwise-and gets optimized away later.
        return __ Word32ShiftLeft(lhs, __ Word32BitwiseAnd(rhs, 0x1f));
      case kExprI32ShrS:
        return __ Word32ShiftRightArithmetic(lhs,
                                             __ Word32BitwiseAnd(rhs, 0x1f));
      case kExprI32ShrU:
        return __ Word32ShiftRightLogical(lhs, __ Word32BitwiseAnd(rhs, 0x1f));
      case kExprI32Ror:
        return __ Word32RotateRight(lhs, __ Word32BitwiseAnd(rhs, 0x1f));
      case kExprI32Rol:
        if (SupportedOperations::word32_rol()) {
          return __ Word32RotateLeft(lhs, __ Word32BitwiseAnd(rhs, 0x1f));
        } else {
          return __ Word32RotateRight(
              lhs, __ Word32Sub(32, __ Word32BitwiseAnd(rhs, 0x1f)));
        }
      case kExprI32Eq:
        return __ Word32Equal(lhs, rhs);
      case kExprI32Ne:
        return __ Word32Equal(__ Word32Equal(lhs, rhs), 0);
      case kExprI32LtS:
        return __ Int32LessThan(lhs, rhs);
      case kExprI32LeS:
        return __ Int32LessThanOrEqual(lhs, rhs);
      case kExprI32LtU:
        return __ Uint32LessThan(lhs, rhs);
      case kExprI32LeU:
        return __ Uint32LessThanOrEqual(lhs, rhs);
      case kExprI32GtS:
        return __ Int32LessThan(rhs, lhs);
      case kExprI32GeS:
        return __ Int32LessThanOrEqual(rhs, lhs);
      case kExprI32GtU:
        return __ Uint32LessThan(rhs, lhs);
      case kExprI32GeU:
        return __ Uint32LessThanOrEqual(rhs, lhs);
      case kExprI64Add:
        return __ Word64Add(lhs, rhs);
      case kExprI64Sub:
        return __ Word64Sub(lhs, rhs);
      case kExprI64Mul:
        return __ Word64Mul(lhs, rhs);
      case kExprI64DivS: {
        if constexpr (!Is64()) {
          return BuildDiv64Call(lhs, rhs, ExternalReference::wasm_int64_div(),
                                wasm::TrapId::kTrapDivByZero);
        }
        __ TrapIf(__ Word64Equal(rhs, 0), TrapId::kTrapDivByZero);
        V<Word32> unrepresentable_condition = __ Word32BitwiseAnd(
            __ Word64Equal(rhs, -1),
            __ Word64Equal(lhs, std::numeric_limits<int64_t>::min()));
        __ TrapIf(unrepresentable_condition, TrapId::kTrapDivUnrepresentable);
        return __ Int64Div(lhs, rhs);
      }
      case kExprI64DivU:
        if constexpr (!Is64()) {
          return BuildDiv64Call(lhs, rhs, ExternalReference::wasm_uint64_div(),
                                wasm::TrapId::kTrapDivByZero);
        }
        __ TrapIf(__ Word64Equal(rhs, 0), TrapId::kTrapDivByZero);
        return __ Uint64Div(lhs, rhs);
      case kExprI64RemS: {
        if constexpr (!Is64()) {
          return BuildDiv64Call(lhs, rhs, ExternalReference::wasm_int64_mod(),
                                wasm::TrapId::kTrapRemByZero);
        }
        __ TrapIf(__ Word64Equal(rhs, 0), TrapId::kTrapRemByZero);
        Label<Word64> done(&asm_);
        IF (UNLIKELY(__ Word64Equal(rhs, -1))) {
          GOTO(done, __ Word64Constant(int64_t{0}));
        } ELSE {
          GOTO(done, __ Int64Mod(lhs, rhs));
        };
 
        BIND(done, result);
        return result;
      }
      case kExprI64RemU:
        if constexpr (!Is64()) {
          return BuildDiv64Call(lhs, rhs, ExternalReference::wasm_uint64_mod(),
                                wasm::TrapId::kTrapRemByZero);
        }
        __ TrapIf(__ Word64Equal(rhs, 0), TrapId::kTrapRemByZero);
        return __ Uint64Mod(lhs, rhs);
      case kExprI64And:
        return __ Word64BitwiseAnd(lhs, rhs);
      case kExprI64Ior:
        return __ Word64BitwiseOr(lhs, rhs);
      case kExprI64Xor:
        return __ Word64BitwiseXor(lhs, rhs);
      case kExprI64Shl:
        // If possible, the bitwise-and gets optimized away later.
        return __ Word64ShiftLeft(
            lhs, __ Word32BitwiseAnd(__ TruncateWord64ToWord32(rhs), 0x3f));
      case kExprI64ShrS:
        return __ Word64ShiftRightArithmetic(
            lhs, __ Word32BitwiseAnd(__ TruncateWord64ToWord32(rhs), 0x3f));
      case kExprI64ShrU:
        return __ Word64ShiftRightLogical(
            lhs, __ Word32BitwiseAnd(__ TruncateWord64ToWord32(rhs), 0x3f));
      case kExprI64Ror:
        return __ Word64RotateRight(
            lhs, __ Word32BitwiseAnd(__ TruncateWord64ToWord32(rhs), 0x3f));
      case kExprI64Rol:
        if (SupportedOperations::word64_rol()) {
          return __ Word64RotateLeft(
              lhs, __ Word32BitwiseAnd(__ TruncateWord64ToWord32(rhs), 0x3f));
        } else {
          return __ Word64RotateRight(
              lhs, __ Word32BitwiseAnd(
                       __ Word32Sub(64, __ TruncateWord64ToWord32(rhs)), 0x3f));
        }
      case kExprI64Eq:
        return __ Word64Equal(lhs, rhs);
      case kExprI64Ne:
        return __ Word32Equal(__ Word64Equal(lhs, rhs), 0);
      case kExprI64LtS:
        return __ Int64LessThan(lhs, rhs);
      case kExprI64LeS:
        return __ Int64LessThanOrEqual(lhs, rhs);
      case kExprI64LtU:
        return __ Uint64LessThan(lhs, rhs);
      case kExprI64LeU:
        return __ Uint64LessThanOrEqual(lhs, rhs);
      case kExprI64GtS:
        return __ Int64LessThan(rhs, lhs);
      case kExprI64GeS:
        return __ Int64LessThanOrEqual(rhs, lhs);
      case kExprI64GtU:
        return __ Uint64LessThan(rhs, lhs);
      case kExprI64GeU:
        return __ Uint64LessThanOrEqual(rhs, lhs);
      case kExprF32CopySign: {
        V<Word32> lhs_without_sign =
            __ Word32BitwiseAnd(__ BitcastFloat32ToWord32(lhs), 0x7fffffff);
        V<Word32> rhs_sign =
            __ Word32BitwiseAnd(__ BitcastFloat32ToWord32(rhs), 0x80000000);
        return __ BitcastWord32ToFloat32(
            __ Word32BitwiseOr(lhs_without_sign, rhs_sign));
      }
      case kExprF32Add:
        return __ Float32Add(lhs, rhs);
      case kExprF32Sub:
        return __ Float32Sub(lhs, rhs);
      case kExprF32Mul:
        return __ Float32Mul(lhs, rhs);
      case kExprF32Div:
        return __ Float32Div(lhs, rhs);
      case kExprF32Eq:
        return __ Float32Equal(lhs, rhs);
      case kExprF32Ne:
        return __ Word32Equal(__ Float32Equal(lhs, rhs), 0);
      case kExprF32Lt:
        return __ Float32LessThan(lhs, rhs);
      case kExprF32Le:
        return __ Float32LessThanOrEqual(lhs, rhs);
      case kExprF32Gt:
        return __ Float32LessThan(rhs, lhs);
      case kExprF32Ge:
        return __ Float32LessThanOrEqual(rhs, lhs);
      case kExprF32Min:
        return __ Float32Min(rhs, lhs);
      case kExprF32Max:
        return __ Float32Max(rhs, lhs);
      case kExprF64CopySign: {
        V<Word64> lhs_without_sign = __ Word64BitwiseAnd(
            __ BitcastFloat64ToWord64(lhs), 0x7fffffffffffffff);
        V<Word64> rhs_sign = __ Word64BitwiseAnd(__ BitcastFloat64ToWord64(rhs),
                                                 0x8000000000000000);
        return __ BitcastWord64ToFloat64(
            __ Word64BitwiseOr(lhs_without_sign, rhs_sign));
      }
      case kExprF64Add:
        return __ Float64Add(lhs, rhs);
      case kExprF64Sub:
        return __ Float64Sub(lhs, rhs);
      case kExprF64Mul:
        return __ Float64Mul(lhs, rhs);
      case kExprF64Div:
        return __ Float64Div(lhs, rhs);
      case kExprF64Eq:
        return __ Float64Equal(lhs, rhs);
      case kExprF64Ne:
        return __ Word32Equal(__ Float64Equal(lhs, rhs), 0);
      case kExprF64Lt:
        return __ Float64LessThan(lhs, rhs);
      case kExprF64Le:
        return __ Float64LessThanOrEqual(lhs, rhs);
      case kExprF64Gt:
        return __ Float64LessThan(rhs, lhs);
      case kExprF64Ge:
        return __ Float64LessThanOrEqual(rhs, lhs);
      case kExprF64Min:
        return __ Float64Min(lhs, rhs);
      case kExprF64Max:
        return __ Float64Max(lhs, rhs);
      case kExprF64Pow:
        return __ Float64Power(lhs, rhs);
      case kExprF64Atan2:
        return __ Float64Atan2(lhs, rhs);
      case kExprF64Mod:
        return CallCStackSlotToStackSlot(
            lhs, rhs, ExternalReference::f64_mod_wrapper_function(),
            MemoryRepresentation::Float64());
      case kExprRefEq:
        return __ TaggedEqual(lhs, rhs);
      case kExprI32AsmjsDivS: {
        // asmjs semantics return 0 when dividing by 0.
        if (SupportedOperations::int32_div_is_safe()) {
          return __ Int32Div(lhs, rhs);
        }
        Label<Word32> done(&asm_);
        IF (UNLIKELY(__ Word32Equal(rhs, 0))) {
          GOTO(done, __ Word32Constant(0));
        } ELSE {
          IF (UNLIKELY(__ Word32Equal(rhs, -1))) {
            GOTO(done, __ Word32Sub(0, lhs));
          } ELSE {
            GOTO(done, __ Int32Div(lhs, rhs));
          }
        }
        BIND(done, result);
        return result;
      }
      case kExprI32AsmjsDivU: {
        // asmjs semantics return 0 when dividing by 0.
        if (SupportedOperations::uint32_div_is_safe()) {
          return __ Uint32Div(lhs, rhs);
        }
        Label<Word32> done(&asm_);
        IF (UNLIKELY(__ Word32Equal(rhs, 0))) {
          GOTO(done, __ Word32Constant(0));
        } ELSE {
          GOTO(done, __ Uint32Div(lhs, rhs));
        }
        BIND(done, result);
        return result;
      }
      case kExprI32AsmjsRemS: {
        // General case for signed integer modulus, with optimization for
        // (unknown) power of 2 right hand side.
        //
        //   if 0 < rhs then
        //     mask = rhs - 1
        //     if rhs & mask != 0 then
        //       lhs % rhs
        //     else
        //       if lhs < 0 then
        //         -(-lhs & mask)
        //       else
        //         lhs & mask
        //   else
        //     if rhs < -1 then
        //       lhs % rhs
        //     else
        //       zero
        Label<Word32> done(&asm_);
        IF (__ Int32LessThan(0, rhs)) {
          V<Word32> mask = __ Word32Sub(rhs, 1);
          IF (__ Word32Equal(__ Word32BitwiseAnd(rhs, mask), 0)) {
            IF (UNLIKELY(__ Int32LessThan(lhs, 0))) {
              V<Word32> neg_lhs = __ Word32Sub(0, lhs);
              V<Word32> combined = __ Word32BitwiseAnd(neg_lhs, mask);
              GOTO(done, __ Word32Sub(0, combined));
            } ELSE {
              GOTO(done, __ Word32BitwiseAnd(lhs, mask));
            }
          } ELSE {
            GOTO(done, __ Int32Mod(lhs, rhs));
          }
        } ELSE {
          IF (__ Int32LessThan(rhs, -1)) {
            GOTO(done, __ Int32Mod(lhs, rhs));
          } ELSE {
            GOTO(done, __ Word32Constant(0));
          }
        }
        BIND(done, result);
        return result;
      }
      case kExprI32AsmjsRemU: {
        // asmjs semantics return 0 for mod with 0.
        Label<Word32> done(&asm_);
        IF (UNLIKELY(__ Word32Equal(rhs, 0))) {
          GOTO(done, __ Word32Constant(0));
        } ELSE {
          GOTO(done, __ Uint32Mod(lhs, rhs));
        }
        BIND(done, result);
        return result;
      }
      case kExprI32AsmjsStoreMem8:
        AsmjsStoreMem(lhs, rhs, MemoryRepresentation::Int8());
        return rhs;
      case kExprI32AsmjsStoreMem16:
        AsmjsStoreMem(lhs, rhs, MemoryRepresentation::Int16());
        return rhs;
      case kExprI32AsmjsStoreMem:
        AsmjsStoreMem(lhs, rhs, MemoryRepresentation::Int32());
        return rhs;
      case kExprF32AsmjsStoreMem:
        AsmjsStoreMem(lhs, rhs, MemoryRepresentation::Float32());
        return rhs;
      case kExprF64AsmjsStoreMem:
        AsmjsStoreMem(lhs, rhs, MemoryRepresentation::Float64());
        return rhs;
      default:
        UNREACHABLE();
    }
  }
 
  std::pair<V<WordPtr>, compiler::BoundsCheckResult> BoundsCheckMem(
      const wasm::WasmMemory* memory, MemoryRepresentation repr, OpIndex index,
      uintptr_t offset, compiler::EnforceBoundsCheck enforce_bounds_check,
      compiler::AlignmentCheck alignment_check) {
    // The function body decoder already validated that the access is not
    // statically OOB.
    DCHECK(base::IsInBounds(offset, static_cast<uintptr_t>(repr.SizeInBytes()),
                            memory->max_memory_size));
 
    wasm::BoundsCheckStrategy bounds_checks = memory->bounds_checks;
    // Convert the index to uintptr.
    // TODO(jkummerow): This should reuse MemoryAddressToUintPtrOrOOBTrap.
    V<WordPtr> converted_index = index;
    if (!memory->is_memory64()) {
      // Note: this doesn't just satisfy the compiler's internal consistency
      // checks, it's also load-bearing to prevent escaping from a compromised
      // sandbox (where in-sandbox corruption can cause the high word of
      // what's supposed to be an i32 to be non-zero).
      converted_index = __ ChangeUint32ToUintPtr(index);
    } else if (kSystemPointerSize == kInt32Size) {
      // Truncate index to 32-bit.
      converted_index = V<WordPtr>::Cast(__ TruncateWord64ToWord32(index));
    }
 
    const uintptr_t align_mask = repr.SizeInBytes() - 1;
    // Do alignment checks only for > 1 byte accesses (otherwise they trivially
    // pass).
    if (static_cast<bool>(alignment_check) && align_mask != 0) {
      // TODO(14108): Optimize constant index as per wasm-compiler.cc.
 
      // Unlike regular memory accesses, atomic memory accesses should trap if
      // the effective offset is misaligned.
      // TODO(wasm): this addition is redundant with one inserted by
      // {MemBuffer}.
      OpIndex effective_offset =
          __ WordPtrAdd(MemBuffer(memory->index, offset), converted_index);
 
      V<Word32> cond = __ TruncateWordPtrToWord32(__ WordPtrBitwiseAnd(
          effective_offset, __ IntPtrConstant(align_mask)));
      __ TrapIfNot(__ Word32Equal(cond, __ Word32Constant(0)),
                   TrapId::kTrapUnalignedAccess);
    }
 
    // If no bounds checks should be performed (for testing), just return the
    // converted index and assume it to be in-bounds.
    if (bounds_checks == wasm::kNoBoundsChecks) {
      return {converted_index, compiler::BoundsCheckResult::kInBounds};
    }
 
    if (memory->is_memory64() && kSystemPointerSize == kInt32Size) {
      // In memory64 mode on 32-bit systems, the upper 32 bits need to be zero
      // to succeed the bounds check.
      DCHECK_EQ(kExplicitBoundsChecks, bounds_checks);
      V<Word32> high_word =
          __ TruncateWord64ToWord32(__ Word64ShiftRightLogical(index, 32));
      __ TrapIf(high_word, TrapId::kTrapMemOutOfBounds);
    }
 
    uintptr_t end_offset = offset + repr.SizeInBytes() - 1u;
    DCHECK_LT(end_offset, memory->max_memory_size);
 
    // The index can be invalid if we are generating unreachable operations.
    if (end_offset <= memory->min_memory_size && index.valid() &&
        __ output_graph().Get(index).Is<ConstantOp>()) {
      ConstantOp& constant_index_op =
          __ output_graph().Get(index).Cast<ConstantOp>();
      uintptr_t constant_index = memory->is_memory64()
                                     ? constant_index_op.word64()
                                     : constant_index_op.word32();
      if (constant_index < memory->min_memory_size - end_offset) {
        return {converted_index, compiler::BoundsCheckResult::kInBounds};
      }
    }
 
#if V8_TRAP_HANDLER_SUPPORTED
    if (bounds_checks == kTrapHandler &&
        enforce_bounds_check ==
            compiler::EnforceBoundsCheck::kCanOmitBoundsCheck) {
      if (memory->is_memory64()) {
        // Bounds check `index` against `kMaxMemory64Size - end_offset`, such
        // that at runtime `index + end_offset` will be within
        // `kMaxMemory64Size`, where the trap handler can handle out-of-bound
        // accesses.
        V<Word32> cond = __ Uint64LessThan(
            V<Word64>::Cast(converted_index),
            __ Word64Constant(uint64_t{wasm::kMaxMemory64Size - end_offset}));
        __ TrapIfNot(cond, TrapId::kTrapMemOutOfBounds);
      }
      return {converted_index, compiler::BoundsCheckResult::kTrapHandler};
    }
#else
    CHECK_NE(bounds_checks, kTrapHandler);
#endif  // V8_TRAP_HANDLER_SUPPORTED
 
    V<WordPtr> memory_size = MemSize(memory->index);
    if (end_offset > memory->min_memory_size) {
      // The end offset is larger than the smallest memory.
      // Dynamically check the end offset against the dynamic memory size.
      __ TrapIfNot(
          __ UintPtrLessThan(__ UintPtrConstant(end_offset), memory_size),
          TrapId::kTrapMemOutOfBounds);
    }
 
    // This produces a positive number since {end_offset <= min_size <=
    // mem_size}.
    V<WordPtr> effective_size = __ WordPtrSub(memory_size, end_offset);
    __ TrapIfNot(__ UintPtrLessThan(converted_index, effective_size),
                 TrapId::kTrapMemOutOfBounds);
    return {converted_index, compiler::BoundsCheckResult::kDynamicallyChecked};
  }
 
  V<WordPtr> MemStart(uint32_t index) {
    if (index == 0) {
      // TODO(14108): Port TF's dynamic "cached_memory_index" infrastructure.
      return instance_cache_.memory0_start();
    } else {
      // TODO(14616): Fix sharedness.
      V<TrustedFixedAddressArray> instance_memories =
          LOAD_IMMUTABLE_PROTECTED_INSTANCE_FIELD(trusted_instance_data(false),
                                                  MemoryBasesAndSizes,
                                                  TrustedFixedAddressArray);
      return __ Load(instance_memories, LoadOp::Kind::TaggedBase(),
                     MemoryRepresentation::UintPtr(),
                     TrustedFixedAddressArray::OffsetOfElementAt(2 * index));
    }
  }
 
  V<WordPtr> MemBuffer(uint32_t mem_index, uintptr_t offset) {
    V<WordPtr> mem_start = MemStart(mem_index);
    if (offset == 0) return mem_start;
    return __ WordPtrAdd(mem_start, offset);
  }
 
  V<WordPtr> MemSize(uint32_t index) {
    if (index == 0) {
      // TODO(14108): Port TF's dynamic "cached_memory_index" infrastructure.
      return instance_cache_.memory0_size();
    } else {
      // TODO(14616): Fix sharedness.
      V<TrustedByteArray> instance_memories =
          LOAD_IMMUTABLE_PROTECTED_INSTANCE_FIELD(trusted_instance_data(false),
                                                  MemoryBasesAndSizes,
                                                  TrustedByteArray);
      return __ Load(
          instance_memories, LoadOp::Kind::TaggedBase().NotLoadEliminable(),
          MemoryRepresentation::UintPtr(),
          TrustedFixedAddressArray::OffsetOfElementAt(2 * index + 1));
    }
  }
 
  LoadOp::Kind GetMemoryAccessKind(
      MemoryRepresentation repr,
      compiler::BoundsCheckResult bounds_check_result) {
    LoadOp::Kind result;
    if (bounds_check_result == compiler::BoundsCheckResult::kTrapHandler) {
      DCHECK(repr == MemoryRepresentation::Int8() ||
             repr == MemoryRepresentation::Uint8() ||
             SupportedOperations::IsUnalignedLoadSupported(repr));
      result = LoadOp::Kind::Protected();
    } else if (repr != MemoryRepresentation::Int8() &&
               repr != MemoryRepresentation::Uint8() &&
               !SupportedOperations::IsUnalignedLoadSupported(repr)) {
      result = LoadOp::Kind::RawUnaligned();
    } else {
      result = LoadOp::Kind::RawAligned();
    }
    return result.NotLoadEliminable();
  }
 
  void TraceMemoryOperation(FullDecoder* decoder, bool is_store,
                            MemoryRepresentation repr, V<WordPtr> index,
                            uintptr_t offset) {
    int kAlign = 4;  // Ensure that the LSB is 0, like a Smi.
    V<WordPtr> info = __ StackSlot(sizeof(MemoryTracingInfo), kAlign);
    V<WordPtr> effective_offset = __ WordPtrAdd(index, offset);
    __ Store(info, effective_offset, StoreOp::Kind::RawAligned(),
             MemoryRepresentation::UintPtr(), compiler::kNoWriteBarrier,
             offsetof(MemoryTracingInfo, offset));
    __ Store(info, __ Word32Constant(is_store ? 1 : 0),
             StoreOp::Kind::RawAligned(), MemoryRepresentation::Uint8(),
             compiler::kNoWriteBarrier, offsetof(MemoryTracingInfo, is_store));
    V<Word32> rep_as_int = __ Word32Constant(
        static_cast<int>(repr.ToMachineType().representation()));
    __ Store(info, rep_as_int, StoreOp::Kind::RawAligned(),
             MemoryRepresentation::Uint8(), compiler::kNoWriteBarrier,
             offsetof(MemoryTracingInfo, mem_rep));
    CallRuntime(decoder->zone(), Runtime::kWasmTraceMemory, {info},
                __ NoContextConstant());
  }
 
  void StackCheck(WasmStackCheckOp::Kind kind, FullDecoder* decoder) {
    if (V8_UNLIKELY(!v8_flags.wasm_stack_checks)) return;
    __ WasmStackCheck(kind);
  }
 
 private:
  std::pair<V<Word32>, V<HeapObject>> BuildImportedFunctionTargetAndImplicitArg(
      FullDecoder* decoder, uint32_t function_index) {
    ModuleTypeIndex sig_index =
        decoder->module_->functions[function_index].sig_index;
    bool shared = decoder->module_->type(sig_index).is_shared;
    return WasmGraphBuilderBase::BuildImportedFunctionTargetAndImplicitArg(
        function_index, trusted_instance_data(shared));
  }
 
  // Returns the call target and the implicit argument (WasmTrustedInstanceData
  // or WasmImportData) for an indirect call.
  std::pair<V<Word32>, V<ExposedTrustedObject>>
  BuildIndirectCallTargetAndImplicitArg(FullDecoder* decoder,
                                        V<WordPtr> index_wordptr,
                                        CallIndirectImmediate imm,
                                        bool needs_type_or_null_check = true) {
    static_assert(kV8MaxWasmTableSize < size_t{kMaxInt});
    const WasmTable* table = imm.table_imm.table;
 
    /* Step 1: Load the indirect function tables for this table. */
    V<WasmDispatchTable> dispatch_table;
    if (imm.table_imm.index == 0) {
      dispatch_table =
          LOAD_PROTECTED_INSTANCE_FIELD(trusted_instance_data(table->shared),
                                        DispatchTable0, WasmDispatchTable);
    } else {
      V<ProtectedFixedArray> dispatch_tables =
          LOAD_IMMUTABLE_PROTECTED_INSTANCE_FIELD(
              trusted_instance_data(table->shared), DispatchTables,
              ProtectedFixedArray);
      dispatch_table =
          V<WasmDispatchTable>::Cast(__ LoadProtectedFixedArrayElement(
              dispatch_tables, imm.table_imm.index));
    }
 
    /* Step 2: Bounds check against the table size. */
    V<Word32> table_length;
    bool needs_dynamic_size =
        !table->has_maximum_size || table->maximum_size != table->initial_size;
    if (needs_dynamic_size) {
      table_length = __ LoadField<Word32>(
          dispatch_table, AccessBuilder::ForWasmDispatchTableLength());
    } else {
      table_length = __ Word32Constant(table->initial_size);
    }
    V<Word32> in_bounds = __ UintPtrLessThan(
        index_wordptr, __ ChangeUint32ToUintPtr(table_length));
    __ TrapIfNot(in_bounds, TrapId::kTrapTableOutOfBounds);
 
    /* Step 3: Check the canonical real signature against the canonical declared
     * signature. */
    ModuleTypeIndex sig_index = imm.sig_imm.index;
    bool needs_type_check =
        needs_type_or_null_check &&
        !EquivalentTypes(table->type.AsNonNull(),
                         ValueType::Ref(imm.sig_imm.heap_type()),
                         decoder->module_, decoder->module_);
    bool needs_null_check =
        needs_type_or_null_check && table->type.is_nullable();
 
    V<WordPtr> dispatch_table_entry_offset = __ WordPtrAdd(
        __ WordPtrMul(index_wordptr, WasmDispatchTable::kEntrySize),
        WasmDispatchTable::kEntriesOffset);
 
    if (needs_type_check) {
      CanonicalTypeIndex sig_id = env_->module->canonical_sig_id(sig_index);
      V<Word32> expected_canonical_sig =
          __ RelocatableWasmCanonicalSignatureId(sig_id.index);
 
      V<Word32> loaded_sig =
          __ Load(dispatch_table, dispatch_table_entry_offset,
                  LoadOp::Kind::TaggedBase(), MemoryRepresentation::Uint32(),
                  WasmDispatchTable::kSigBias);
      V<Word32> sigs_match = __ Word32Equal(expected_canonical_sig, loaded_sig);
      if (!decoder->module_->type(sig_index).is_final) {
        // In this case, a full type check is needed.
        Label<> end(&asm_);
 
        // First, check if signatures happen to match exactly.
        GOTO_IF(sigs_match, end);
 
        if (needs_null_check) {
          // Trap on null element.
          __ TrapIf(__ Word32Equal(loaded_sig, -1),
                    TrapId::kTrapFuncSigMismatch);
        }
        bool shared = decoder->module_->type(sig_index).is_shared;
        V<Map> formal_rtt = __ RttCanon(managed_object_maps(shared), sig_index);
        int rtt_depth = GetSubtypingDepth(decoder->module_, sig_index);
        DCHECK_GE(rtt_depth, 0);
 
        // Since we have the canonical index of the real rtt, we have to load it
        // from the isolate rtt-array (which is canonically indexed). Since this
        // reference is weak, we have to promote it to a strong reference.
        // Note: The reference cannot have been cleared: Since the loaded_sig
        // corresponds to a function of the same canonical type, that function
        // will have kept the type alive.
        V<WeakFixedArray> rtts = LOAD_ROOT(WasmCanonicalRtts);
        V<Object> weak_rtt = __ Load(
            rtts, __ ChangeInt32ToIntPtr(loaded_sig),
            LoadOp::Kind::TaggedBase(), MemoryRepresentation::TaggedPointer(),
            OFFSET_OF_DATA_START(WeakFixedArray), kTaggedSizeLog2);
        V<Map> real_rtt =
            V<Map>::Cast(__ BitcastWordPtrToTagged(__ WordPtrBitwiseAnd(
                __ BitcastHeapObjectToWordPtr(V<HeapObject>::Cast(weak_rtt)),
                ~kWeakHeapObjectMask)));
        V<WasmTypeInfo> type_info =
            __ Load(real_rtt, LoadOp::Kind::TaggedBase(),
                    MemoryRepresentation::TaggedPointer(),
                    Map::kConstructorOrBackPointerOrNativeContextOffset);
        // If the depth of the rtt is known to be less than the minimum
        // supertype array length, we can access the supertype without
        // bounds-checking the supertype array.
        if (static_cast<uint32_t>(rtt_depth) >=
            wasm::kMinimumSupertypeArraySize) {
          V<Word32> supertypes_length =
              __ UntagSmi(__ Load(type_info, LoadOp::Kind::TaggedBase(),
                                  MemoryRepresentation::TaggedSigned(),
                                  WasmTypeInfo::kSupertypesLengthOffset));
          __ TrapIfNot(__ Uint32LessThan(rtt_depth, supertypes_length),
                       OpIndex::Invalid(), TrapId::kTrapFuncSigMismatch);
        }
        V<Map> maybe_match =
            __ Load(type_info, LoadOp::Kind::TaggedBase(),
                    MemoryRepresentation::TaggedPointer(),
                    WasmTypeInfo::kSupertypesOffset + kTaggedSize * rtt_depth);
        __ TrapIfNot(__ TaggedEqual(maybe_match, formal_rtt),
                     OpIndex::Invalid(), TrapId::kTrapFuncSigMismatch);
        GOTO(end);
        BIND(end);
      } else {
        // In this case, signatures must match exactly.
        __ TrapIfNot(sigs_match, TrapId::kTrapFuncSigMismatch);
      }
    } else if (needs_null_check) {
      V<Word32> loaded_sig =
          __ Load(dispatch_table, dispatch_table_entry_offset,
                  LoadOp::Kind::TaggedBase(), MemoryRepresentation::Uint32(),
                  WasmDispatchTable::kSigBias);
      __ TrapIf(__ Word32Equal(-1, loaded_sig), TrapId::kTrapFuncSigMismatch);
    }
 
    /* Step 4: Extract ref and target. */
    V<Word32> target = __ Load(
        dispatch_table, dispatch_table_entry_offset, LoadOp::Kind::TaggedBase(),
        MemoryRepresentation::Uint32(), WasmDispatchTable::kTargetBias);
    V<ExposedTrustedObject> implicit_arg =
        V<ExposedTrustedObject>::Cast(__ LoadProtectedPointerField(
            dispatch_table, dispatch_table_entry_offset,
            LoadOp::Kind::TaggedBase(), WasmDispatchTable::kImplicitArgBias,
            0));
 
    return {target, implicit_arg};
  }
 
  // Load the call target and implicit arg (WasmTrustedInstanceData or
  // WasmImportData) from a function reference.
  std::pair<V<Word32>, V<ExposedTrustedObject>>
  BuildFunctionReferenceTargetAndImplicitArg(V<WasmFuncRef> func_ref,
                                             ValueType type) {
    if (type.is_nullable() &&
        null_check_strategy_ == compiler::NullCheckStrategy::kExplicit) {
      func_ref = V<WasmFuncRef>::Cast(
          __ AssertNotNull(func_ref, type, TrapId::kTrapNullDereference));
    }
 
    LoadOp::Kind load_kind =
        type.is_nullable() && null_check_strategy_ ==
                                  compiler::NullCheckStrategy::kTrapHandler
            ? LoadOp::Kind::TrapOnNull().Immutable()
            : LoadOp::Kind::TaggedBase().Immutable();
 
    V<WasmInternalFunction> internal_function =
        V<WasmInternalFunction>::Cast(__ LoadTrustedPointerField(
            func_ref, load_kind, kWasmInternalFunctionIndirectPointerTag,
            WasmFuncRef::kTrustedInternalOffset));
 
    return BuildFunctionTargetAndImplicitArg(internal_function);
  }
 
  OpIndex AnnotateResultIfReference(OpIndex result, wasm::ValueType type) {
    return type.is_object_reference()
               ? __ AnnotateWasmType(V<Object>::Cast(result), type)
               : result;
  }
 
  void BuildWasmCall(
      FullDecoder* decoder, const FunctionSig* sig, V<CallTarget> callee,
      V<HeapObject> ref, const Value args[], Value returns[],
      compiler::WasmCallKind call_kind = compiler::WasmCallKind::kWasmFunction,
      CheckForException check_for_exception =
          CheckForException::kCatchInThisFrame) {
    const TSCallDescriptor* descriptor = TSCallDescriptor::Create(
        compiler::GetWasmCallDescriptor(__ graph_zone(), sig, call_kind),
        compiler::CanThrow::kYes, compiler::LazyDeoptOnThrow::kNo,
        __ graph_zone());
 
    SmallZoneVector<OpIndex, 16> arg_indices(sig->parameter_count() + 1,
                                             decoder->zone());
    arg_indices[0] = ref;
    for (uint32_t i = 0; i < sig->parameter_count(); i++) {
      arg_indices[i + 1] = args[i].op;
    }
 
    OpIndex call = CallAndMaybeCatchException(
        decoder, callee, base::VectorOf(arg_indices), descriptor,
        check_for_exception, OpEffects().CanCallAnything());
 
    if (sig->return_count() == 1) {
      returns[0].op = AnnotateResultIfReference(call, sig->GetReturn(0));
    } else if (sig->return_count() > 1) {
      for (uint32_t i = 0; i < sig->return_count(); i++) {
        wasm::ValueType type = sig->GetReturn(i);
        returns[i].op = AnnotateResultIfReference(
            __ Projection(call, i, RepresentationFor(type)), type);
      }
    }
    // Calls might mutate cached instance fields.
    instance_cache_.ReloadCachedMemory();
  }
 
 private:
  void BuildWasmMaybeReturnCall(
      FullDecoder* decoder, const FunctionSig* sig, V<CallTarget> callee,
      V<HeapObject> ref, const Value args[],
      compiler::WasmCallKind call_kind = compiler::kWasmFunction) {
    if (mode_ == kRegular || mode_ == kInlinedTailCall) {
      const TSCallDescriptor* descriptor = TSCallDescriptor::Create(
          compiler::GetWasmCallDescriptor(__ graph_zone(), sig, call_kind),
          compiler::CanThrow::kYes, compiler::LazyDeoptOnThrow::kNo,
          __ graph_zone());
 
      SmallZoneVector<OpIndex, 16> arg_indices(sig->parameter_count() + 1,
                                               decoder->zone_);
      arg_indices[0] = ref;
      for (uint32_t i = 0; i < sig->parameter_count(); i++) {
        arg_indices[i + 1] = args[i].op;
      }
      __ TailCall(callee, base::VectorOf(arg_indices), descriptor);
    } else {
      if (__ generating_unreachable_operations()) return;
      // This is a tail call in the inlinee, which in turn was a regular call.
      // Transform the tail call into a regular call, and return the return
      // values to the caller.
      size_t return_count = sig->return_count();
      SmallZoneVector<Value, 16> returns(return_count, decoder->zone_);
      // Since an exception in a tail call cannot be caught in this frame, we
      // should only catch exceptions in the generated call if this is a
      // recursively inlined function, and the parent frame provides a handler.
      BuildWasmCall(decoder, sig, callee, ref, args, returns.data(), call_kind,
                    CheckForException::kCatchInParentFrame);
      for (size_t i = 0; i < return_count; i++) {
        return_phis_->AddInputForPhi(i, returns[i].op);
      }
      __ Goto(return_block_);
    }
  }
 
  template <typename Descriptor>
  compiler::turboshaft::detail::index_type_for_t<typename Descriptor::results_t>
  CallBuiltinThroughJumptable(
      FullDecoder* decoder, const typename Descriptor::arguments_t& args,
      CheckForException check_for_exception = CheckForException::kNo)
    requires(!Descriptor::kNeedsContext)
  {
    DCHECK_NE(check_for_exception, CheckForException::kCatchInParentFrame);
 
    V<WordPtr> callee =
        __ RelocatableWasmBuiltinCallTarget(Descriptor::kFunction);
    auto arguments = std::apply(
        [](auto&&... as) {
          return base::SmallVector<
              OpIndex, std::tuple_size_v<typename Descriptor::arguments_t> + 1>{
              std::forward<decltype(as)>(as)...};
        },
        args);
 
    return CallAndMaybeCatchException(
        decoder, callee, base::VectorOf(arguments),
        Descriptor::Create(StubCallMode::kCallWasmRuntimeStub,
                           __ output_graph().graph_zone()),
        check_for_exception, Descriptor::kEffects);
  }
 
  template <typename Descriptor>
  compiler::turboshaft::detail::index_type_for_t<typename Descriptor::results_t>
  CallBuiltinThroughJumptable(
      FullDecoder* decoder, V<Context> context,
      const typename Descriptor::arguments_t& args,
      CheckForException check_for_exception = CheckForException::kNo)
    requires Descriptor::kNeedsContext
  {
    DCHECK_NE(check_for_exception, CheckForException::kCatchInParentFrame);
 
    V<WordPtr> callee =
        __ RelocatableWasmBuiltinCallTarget(Descriptor::kFunction);
    auto arguments = std::apply(
        [context](auto&&... as) {
          return base::SmallVector<
              OpIndex, std::tuple_size_v<typename Descriptor::arguments_t> + 1>{
              std::forward<decltype(as)>(as)..., context};
        },
        args);
 
    return CallAndMaybeCatchException(
        decoder, callee, base::VectorOf(arguments),
        Descriptor::Create(StubCallMode::kCallWasmRuntimeStub,
                           __ output_graph().graph_zone()),
        check_for_exception, Descriptor::kEffects);
  }
 
  template <typename Descriptor>
  compiler::turboshaft::detail::index_type_for_t<typename Descriptor::results_t>
  CallBuiltinByPointer(
      FullDecoder* decoder, const typename Descriptor::arguments_t& args,
      CheckForException check_for_exception = CheckForException::kNo)
    requires(!Descriptor::kNeedsContext)
  {
    DCHECK_NE(check_for_exception, CheckForException::kCatchInParentFrame);
 
    V<WordPtr> callee = GetBuiltinPointerTarget(Descriptor::kFunction);
    auto arguments = std::apply(
        [](auto&&... as) {
          return base::SmallVector<
              OpIndex, std::tuple_size_v<typename Descriptor::arguments_t> + 1>{
              std::forward<decltype(as)>(as)...};
        },
        args);
 
    return CallAndMaybeCatchException(
        decoder, callee, base::VectorOf(arguments),
        Descriptor::Create(StubCallMode::kCallBuiltinPointer,
                           __ output_graph().graph_zone()),
        check_for_exception, Descriptor::kEffects);
  }
 
 private:
  void MaybeSetPositionToParent(OpIndex call,
                                CheckForException check_for_exception) {
    // For tail calls that we transform to regular calls, we need to set the
    // call's position to that of the inlined call node to get correct stack
    // traces.
    if (check_for_exception == CheckForException::kCatchInParentFrame) {
      __ output_graph().operation_origins()[call] = WasmPositionToOpIndex(
          parent_position_.ScriptOffset(), parent_position_.InliningId() == -1
                                               ? kNoInliningId
                                               : parent_position_.InliningId());
    }
  }
 
  OpIndex CallAndMaybeCatchException(FullDecoder* decoder, V<CallTarget> callee,
                                     base::Vector<const OpIndex> args,
                                     const TSCallDescriptor* descriptor,
                                     CheckForException check_for_exception,
                                     OpEffects effects) {
    if (check_for_exception == CheckForException::kNo) {
      return __ Call(callee, OpIndex::Invalid(), args, descriptor, effects);
    }
    bool handled_in_this_frame =
        decoder && decoder->current_catch() != -1 &&
        check_for_exception == CheckForException::kCatchInThisFrame;
    if (!handled_in_this_frame && mode_ != kInlinedWithCatch) {
      OpIndex call =
          __ Call(callee, OpIndex::Invalid(), args, descriptor, effects);
      MaybeSetPositionToParent(call, check_for_exception);
      return call;
    }
 
    TSBlock* catch_block;
    if (handled_in_this_frame) {
      Control* current_catch =
          decoder->control_at(decoder->control_depth_of_current_catch());
      catch_block = current_catch->false_or_loop_or_catch_block;
    } else {
      DCHECK_EQ(mode_, kInlinedWithCatch);
      catch_block = return_catch_block_;
    }
    TSBlock* success_block = __ NewBlock();
    TSBlock* exception_block = __ NewBlock();
    OpIndex call;
    {
      Assembler::CatchScope scope(asm_, exception_block);
 
      call = __ Call(callee, OpIndex::Invalid(), args, descriptor, effects);
      __ Goto(success_block);
    }
 
    __ Bind(exception_block);
    OpIndex exception = __ CatchBlockBegin();
    if (handled_in_this_frame) {
      // The exceptional operation could have modified memory size; we need
      // to reload the memory context into the exceptional control path.
      instance_cache_.ReloadCachedMemory();
      SetupControlFlowEdge(decoder, catch_block, 0, exception);
    } else {
      DCHECK_EQ(mode_, kInlinedWithCatch);
      if (exception.valid()) return_phis_->AddIncomingException(exception);
      // Reloading the InstanceCache will happen when {return_exception_phis_}
      // are retrieved.
    }
    __ Goto(catch_block);
 
    __ Bind(success_block);
 
    MaybeSetPositionToParent(call, check_for_exception);
 
    return call;
  }
 
  OpIndex CallCStackSlotToInt32(OpIndex arg, ExternalReference ref,
                                MemoryRepresentation arg_type) {
    OpIndex stack_slot_param =
        __ StackSlot(arg_type.SizeInBytes(), arg_type.SizeInBytes());
    __ Store(stack_slot_param, arg, StoreOp::Kind::RawAligned(), arg_type,
             compiler::WriteBarrierKind::kNoWriteBarrier);
    MachineType reps[]{MachineType::Int32(), MachineType::Pointer()};
    MachineSignature sig(1, 1, reps);
    return CallC(&sig, ref, stack_slot_param);
  }
 
  V<Word32> CallCStackSlotToInt32(
      ExternalReference ref,
      std::initializer_list<std::pair<OpIndex, MemoryRepresentation>> args) {
    int slot_size = 0;
    for (auto arg : args) slot_size += arg.second.SizeInBytes();
    // Since we are storing the arguments unaligned anyway, we do not need
    // alignment > 0.
    V<WordPtr> stack_slot_param = __ StackSlot(slot_size, 0);
    int offset = 0;
    for (auto arg : args) {
      __ Store(stack_slot_param, arg.first,
               StoreOp::Kind::MaybeUnaligned(arg.second), arg.second,
               compiler::WriteBarrierKind::kNoWriteBarrier, offset);
      offset += arg.second.SizeInBytes();
    }
    MachineType reps[]{MachineType::Int32(), MachineType::Pointer()};
    MachineSignature sig(1, 1, reps);
    return CallC(&sig, ref, stack_slot_param);
  }
 
  OpIndex CallCStackSlotToStackSlot(
      ExternalReference ref, MemoryRepresentation res_type,
      std::initializer_list<std::pair<OpIndex, MemoryRepresentation>> args) {
    int slot_size = 0;
    for (auto arg : args) slot_size += arg.second.SizeInBytes();
    // Since we are storing the arguments unaligned anyway, we do not need
    // alignment > 0.
    slot_size = std::max<int>(slot_size, res_type.SizeInBytes());
    V<WordPtr> stack_slot_param = __ StackSlot(slot_size, 0);
    int offset = 0;
    for (auto arg : args) {
      __ Store(stack_slot_param, arg.first,
               StoreOp::Kind::MaybeUnaligned(arg.second), arg.second,
               compiler::WriteBarrierKind::kNoWriteBarrier, offset);
      offset += arg.second.SizeInBytes();
    }
    MachineType reps[]{MachineType::Pointer()};
    MachineSignature sig(0, 1, reps);
    CallC(&sig, ref, stack_slot_param);
    return __ Load(stack_slot_param, LoadOp::Kind::RawAligned(), res_type);
  }
 
  OpIndex CallCStackSlotToStackSlot(OpIndex arg, ExternalReference ref,
                                    MemoryRepresentation arg_type) {
    return CallCStackSlotToStackSlot(arg, ref, arg_type, arg_type);
  }
 
  OpIndex CallCStackSlotToStackSlot(OpIndex arg, ExternalReference ref,
                                    MemoryRepresentation arg_type,
                                    MemoryRepresentation res_type) {
    return CallCStackSlotToStackSlot(ref, res_type, {{arg, arg_type}});
  }
 
  OpIndex CallCStackSlotToStackSlot(OpIndex arg0, OpIndex arg1,
                                    ExternalReference ref,
                                    MemoryRepresentation arg_type) {
    return CallCStackSlotToStackSlot(ref, arg_type,
                                     {{arg0, arg_type}, {arg1, arg_type}});
  }
 
  V<WordPtr> MemOrTableAddressToUintPtrOrOOBTrap(AddressType address_type,
                                                 V<Word> index,
                                                 TrapId trap_reason) {
    // Note: this {ChangeUint32ToUintPtr} doesn't just satisfy the compiler's
    // consistency checks, it's also load-bearing to prevent escaping from a
    // compromised sandbox (where in-sandbox corruption can cause the high
    // word of what's supposed to be an i32 to be non-zero).
    if (address_type == AddressType::kI32) {
      return __ ChangeUint32ToUintPtr(V<Word32>::Cast(index));
    }
    if constexpr (Is64()) {
      return V<WordPtr>::Cast(index);
    }
    __ TrapIf(__ TruncateWord64ToWord32(
                  __ Word64ShiftRightLogical(V<Word64>::Cast(index), 32)),
              OpIndex::Invalid(), trap_reason);
    return V<WordPtr>::Cast(__ TruncateWord64ToWord32(V<Word64>::Cast(index)));
  }
 
  V<WordPtr> MemoryAddressToUintPtrOrOOBTrap(AddressType address_type,
                                             V<Word> index) {
    return MemOrTableAddressToUintPtrOrOOBTrap(address_type, index,
                                               TrapId::kTrapMemOutOfBounds);
  }
 
  V<WordPtr> TableAddressToUintPtrOrOOBTrap(AddressType address_type,
                                            V<Word> index) {
    return MemOrTableAddressToUintPtrOrOOBTrap(address_type, index,
                                               TrapId::kTrapTableOutOfBounds);
  }
 
  V<Smi> ChangeUint31ToSmi(V<Word32> value) {
    if constexpr (COMPRESS_POINTERS_BOOL) {
      return V<Smi>::Cast(
          __ Word32ShiftLeft(value, kSmiShiftSize + kSmiTagSize));
    } else {
      return V<Smi>::Cast(__ WordPtrShiftLeft(__ ChangeUint32ToUintPtr(value),
                                              kSmiShiftSize + kSmiTagSize));
    }
  }
 
  V<Word32> ChangeSmiToUint32(V<Smi> value) {
    if constexpr (COMPRESS_POINTERS_BOOL) {
      return __ Word32ShiftRightLogical(V<Word32>::Cast(value),
                                        kSmiShiftSize + kSmiTagSize);
    } else {
      return __ TruncateWordPtrToWord32(__ WordPtrShiftRightLogical(
          V<WordPtr>::Cast(value), kSmiShiftSize + kSmiTagSize));
    }
  }
 
  void BuildEncodeException32BitValue(V<FixedArray> values_array,
                                      uint32_t index, V<Word32> value) {
    V<Smi> upper_half =
        ChangeUint31ToSmi(__ Word32ShiftRightLogical(value, 16));
    __ StoreFixedArrayElement(values_array, index, upper_half,
                              compiler::kNoWriteBarrier);
    V<Smi> lower_half = ChangeUint31ToSmi(__ Word32BitwiseAnd(value, 0xffffu));
    __ StoreFixedArrayElement(values_array, index + 1, lower_half,
                              compiler::kNoWriteBarrier);
  }
 
  V<Word32> BuildDecodeException32BitValue(V<FixedArray> exception_values_array,
                                           int index) {
    V<Word32> upper_half = __ Word32ShiftLeft(
        ChangeSmiToUint32(V<Smi>::Cast(
            __ LoadFixedArrayElement(exception_values_array, index))),
        16);
    V<Word32> lower_half = ChangeSmiToUint32(V<Smi>::Cast(
        __ LoadFixedArrayElement(exception_values_array, index + 1)));
    return __ Word32BitwiseOr(upper_half, lower_half);
  }
 
  V<Word64> BuildDecodeException64BitValue(V<FixedArray> exception_values_array,
                                           int index) {
    V<Word64> upper_half = __ Word64ShiftLeft(
        __ ChangeUint32ToUint64(
            BuildDecodeException32BitValue(exception_values_array, index)),
        32);
    V<Word64> lower_half = __ ChangeUint32ToUint64(
        BuildDecodeException32BitValue(exception_values_array, index + 2));
    return __ Word64BitwiseOr(upper_half, lower_half);
  }
 
  void UnpackWasmException(FullDecoder* decoder, V<Object> exception,
                           base::Vector<Value> values) {
    V<FixedArray> exception_values_array = V<FixedArray>::Cast(
        CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmGetOwnProperty>(
            decoder, instance_cache_.native_context(),
            {exception, LOAD_ROOT(wasm_exception_values_symbol)}));
 
    int index = 0;
    for (Value& value : values) {
      switch (value.type.kind()) {
        case kI32:
          value.op =
              BuildDecodeException32BitValue(exception_values_array, index);
          index += 2;
          break;
        case kI64:
          value.op =
              BuildDecodeException64BitValue(exception_values_array, index);
          index += 4;
          break;
        case kF32:
          value.op = __ BitcastWord32ToFloat32(
              BuildDecodeException32BitValue(exception_values_array, index));
          index += 2;
          break;
        case kF64:
          value.op = __ BitcastWord64ToFloat64(
              BuildDecodeException64BitValue(exception_values_array, index));
          index += 4;
          break;
        case kS128: {
          V<compiler::turboshaft::Simd128> value_s128;
          value_s128 = __ Simd128Splat(
              BuildDecodeException32BitValue(exception_values_array, index),
              compiler::turboshaft::Simd128SplatOp::Kind::kI32x4);
          index += 2;
          using Kind = compiler::turboshaft::Simd128ReplaceLaneOp::Kind;
          value_s128 = __ Simd128ReplaceLane(
              value_s128,
              BuildDecodeException32BitValue(exception_values_array, index),
              Kind::kI32x4, 1);
          index += 2;
          value_s128 = __ Simd128ReplaceLane(
              value_s128,
              BuildDecodeException32BitValue(exception_values_array, index),
              Kind::kI32x4, 2);
          index += 2;
          value.op = __ Simd128ReplaceLane(
              value_s128,
              BuildDecodeException32BitValue(exception_values_array, index),
              Kind::kI32x4, 3);
          index += 2;
          break;
        }
        case kRef:
        case kRefNull:
          value.op = __ LoadFixedArrayElement(exception_values_array, index);
          index++;
          break;
        case kI8:
        case kI16:
        case kF16:
        case kVoid:
        case kTop:
        case kBottom:
          UNREACHABLE();
      }
    }
  }
 
  void ThrowRef(FullDecoder* decoder, OpIndex exn) {
    CallBuiltinThroughJumptable<BuiltinCallDescriptor::WasmThrowRef>(
        decoder, {exn}, CheckForException::kCatchInThisFrame);
    __ Unreachable();
  }
 
  void AsmjsStoreMem(V<Word32> index, OpIndex value,
                     MemoryRepresentation repr) {
    // Since asmjs does not support unaligned accesses, we can bounds-check
    // ignoring the access size.
    // Technically, we should do a signed 32-to-ptr extension here. However,
    // that is an explicit instruction, whereas unsigned extension is implicit.
    // Since the difference is only observable for memories larger than 2 GiB,
    // and since we disallow such memories, we can use unsigned extension.
    V<WordPtr> index_ptr = __ ChangeUint32ToUintPtr(index);
    IF (LIKELY(__ UintPtrLessThan(index_ptr, MemSize(0)))) {
      __ Store(MemStart(0), index_ptr, value, StoreOp::Kind::RawAligned(), repr,
               compiler::kNoWriteBarrier, 0);
    }
  }
 
  OpIndex AsmjsLoadMem(V<Word32> index, MemoryRepresentation repr) {
    // Since asmjs does not support unaligned accesses, we can bounds-check
    // ignoring the access size.
    Variable result = __ NewVariable(repr.ToRegisterRepresentation());
 
    // Technically, we should do a signed 32-to-ptr extension here. However,
    // that is an explicit instruction, whereas unsigned extension is implicit.
    // Since the difference is only observable for memories larger than 2 GiB,
    // and since we disallow such memories, we can use unsigned extension.
    V<WordPtr> index_ptr = __ ChangeUint32ToUintPtr(index);
    IF (LIKELY(__ UintPtrLessThan(index_ptr, MemSize(0)))) {
      __ SetVariable(result, __ Load(MemStart(0), index_ptr,
                                     LoadOp::Kind::RawAligned(), repr));
    } ELSE {
      switch (repr) {
        case MemoryRepresentation::Int8():
        case MemoryRepresentation::Int16():
        case MemoryRepresentation::Int32():
        case MemoryRepresentation::Uint8():
        case MemoryRepresentation::Uint16():
        case MemoryRepresentation::Uint32():
          __ SetVariable(result, __ Word32Constant(0));
          break;
        case MemoryRepresentation::Float32():
          __ SetVariable(result, __ Float32Constant(
                                     std::numeric_limits<float>::quiet_NaN()));
          break;
        case MemoryRepresentation::Float64():
          __ SetVariable(result, __ Float64Constant(
                                     std::numeric_limits<double>::quiet_NaN()));
          break;
        default:
          UNREACHABLE();
      }
    }
 
    OpIndex result_op = __ GetVariable(result);
    __ SetVariable(result, OpIndex::Invalid());
    return result_op;
  }
 
  void BoundsCheckArray(V<WasmArrayNullable> array, V<Word32> index,
                        ValueType array_type) {
    if (V8_UNLIKELY(v8_flags.experimental_wasm_skip_bounds_checks)) {
      if (array_type.is_nullable()) {
        __ AssertNotNull(array, array_type, TrapId::kTrapNullDereference);
      }
    } else {
      OpIndex length = __ ArrayLength(array, array_type.is_nullable()
                                                 ? compiler::kWithNullCheck
                                                 : compiler::kWithoutNullCheck);
      __ TrapIfNot(__ Uint32LessThan(index, length),
                   TrapId::kTrapArrayOutOfBounds);
    }
  }
 
  V<WasmArray> BoundsCheckArrayWithLength(V<WasmArrayNullable> array,
                                          V<Word32> index, V<Word32> length,
                                          compiler::CheckForNull null_check) {
    if (V8_UNLIKELY(v8_flags.experimental_wasm_skip_bounds_checks)) {
      return V<WasmArray>::Cast(array);
    }
    V<Word32> array_length = __ ArrayLength(array, null_check);
    V<Word32> range_end = __ Word32Add(index, length);
    V<Word32> range_valid = __ Word32BitwiseAnd(
        // OOB if (index + length > array.len).
        __ Uint32LessThanOrEqual(range_end, array_length),
        // OOB if (index + length) overflows.
        __ Uint32LessThanOrEqual(index, range_end));
    __ TrapIfNot(range_valid, TrapId::kTrapArrayOutOfBounds);
    // The array is now guaranteed to be non-null.
    return V<WasmArray>::Cast(array);
  }
 
  void BrOnCastImpl(FullDecoder* decoder, V<Map> rtt,
                    compiler::WasmTypeCheckConfig config, const Value& object,
                    Value* value_on_branch, uint32_t br_depth,
                    bool null_succeeds) {
    OpIndex cast_succeeds = __ WasmTypeCheck(object.op, rtt, config);
    IF (cast_succeeds) {
      // Narrow type for the successful cast target branch.
      Forward(decoder, object, value_on_branch);
      BrOrRet(decoder, br_depth);
    }
    // Note: Differently to below for br_on_cast_fail, we do not Forward
    // the value here to perform a TypeGuard. It can't be done here due to
    // asymmetric decoder code. A Forward here would be popped from the stack
    // and ignored by the decoder. Therefore the decoder has to call Forward
    // itself.
  }
 
  void BrOnCastFailImpl(FullDecoder* decoder, V<Map> rtt,
                        compiler::WasmTypeCheckConfig config,
                        const Value& object, Value* value_on_fallthrough,
                        uint32_t br_depth, bool null_succeeds) {
    OpIndex cast_succeeds = __ WasmTypeCheck(object.op, rtt, config);
    IF (__ Word32Equal(cast_succeeds, 0)) {
      // It is necessary in case of {null_succeeds} to forward the value.
      // This will add a TypeGuard to the non-null type (as in this case the
      // object is non-nullable).
      Forward(decoder, object, decoder->stack_value(1));
      BrOrRet(decoder, br_depth);
    }
    // Narrow type for the successful cast fallthrough branch.
    value_on_fallthrough->op =
        __ AnnotateWasmType(V<Object>::Cast(object.op), config.to);
  }
 
  V<HeapObject> ArrayNewImpl(FullDecoder* decoder, ModuleTypeIndex index,
                             const ArrayType* array_type, V<Word32> length,
                             V<Any> initial_value) {
    // Initialize the array header.
    bool shared = decoder->module_->type(index).is_shared;
    V<Map> rtt = __ RttCanon(managed_object_maps(shared), index);
    V<WasmArray> array = __ WasmAllocateArray(rtt, length, array_type);
    // Initialize the elements.
    ArrayFillImpl(array, __ Word32Constant(0), initial_value, length,
                  array_type, false);
    return array;
  }
 
  V<Map> GetRttFromDescriptor(const Value& descriptor) {
    V<WasmStructNullable> descriptor_struct =
        V<WasmStructNullable>::Cast(descriptor.op);
    // TODO(jkummerow): Refactor this to share the logic with
    // wasm-lowering-reducer.h.
    bool force_explicit_null_check =
        !trap_handler::IsTrapHandlerEnabled() || !V8_STATIC_ROOTS_BOOL;
    bool explicit_null_check =
        descriptor.type.is_nullable() && force_explicit_null_check;
    bool implicit_null_check =
        descriptor.type.is_nullable() && !force_explicit_null_check;
 
    if (explicit_null_check) {
      __ TrapIf(__ IsNull(descriptor_struct, wasm::kWasmAnyRef),
                TrapId::kTrapNullDereference);
    }
    LoadOp::Kind load_kind = implicit_null_check ? LoadOp::Kind::TrapOnNull()
                                                 : LoadOp::Kind::TaggedBase();
    load_kind = load_kind.Immutable();
    return __ Load(descriptor_struct, load_kind,
                   MemoryRepresentation::TaggedPointer(),
                   WasmStruct::kHeaderSize);
  }
 
  V<WasmStruct> StructNewImpl(FullDecoder* decoder,
                              const StructIndexImmediate& imm,
                              const Value& descriptor, OpIndex args[]) {
    const TypeDefinition& type = decoder->module_->type(imm.index);
    DCHECK_EQ(type.has_descriptor(), descriptor.op.valid());
    V<Map> rtt;
    if (type.has_descriptor()) {
      rtt = GetRttFromDescriptor(descriptor);
    } else {
      rtt = __ RttCanon(managed_object_maps(type.is_shared), imm.index);
    }
 
    V<WasmStruct> struct_value;
    if (type.is_descriptor()) {
      struct_value = CallBuiltinThroughJumptable<
          BuiltinCallDescriptor::WasmAllocateDescriptorStruct>(
          decoder, {rtt, __ Word32Constant(imm.index.index)});
    } else {
      struct_value = __ WasmAllocateStruct(rtt, imm.struct_type);
    }
 
    for (uint32_t i = 0; i < imm.struct_type->field_count(); ++i) {
      __ StructSet(struct_value, args[i], imm.struct_type, imm.index, i,
                   compiler::kWithoutNullCheck);
    }
    // If this assert fails then initialization of padding field might be
    // necessary.
    static_assert(Heap::kMinObjectSizeInTaggedWords == 2 &&
                      WasmStruct::kHeaderSize == 2 * kTaggedSize,
                  "empty struct might require initialization of padding field");
    return struct_value;
  }
 
  bool IsSimd128ZeroConstant(OpIndex op) {
    DCHECK_IMPLIES(!op.valid(), __ generating_unreachable_operations());
    if (__ generating_unreachable_operations()) return false;
    const Simd128ConstantOp* s128_op =
        __ output_graph().Get(op).TryCast<Simd128ConstantOp>();
    return s128_op && s128_op->IsZero();
  }
 
  void ArrayFillImpl(V<WasmArray> array, V<Word32> index, V<Any> value,
                     OpIndex length, const wasm::ArrayType* type,
                     bool emit_write_barrier) {
    wasm::ValueType element_type = type->element_type();
 
    // Initialize the array. Use an external function for large arrays with
    // null/number initializer. Use a loop for small arrays and reference arrays
    // with a non-null initial value.
    Label<> done(&asm_);
 
    // The builtin cannot handle s128 values other than 0.
    if (!(element_type == wasm::kWasmS128 && !IsSimd128ZeroConstant(value))) {
      constexpr uint32_t kArrayNewMinimumSizeForMemSet = 16;
      IF_NOT (__ Uint32LessThan(
                  length, __ Word32Constant(kArrayNewMinimumSizeForMemSet))) {
        OpIndex stack_slot = StoreInStackSlot(value, element_type);
        MachineType arg_types[]{
            MachineType::TaggedPointer(), MachineType::Uint32(),
            MachineType::Uint32(),        MachineType::Uint32(),
            MachineType::Uint32(),        MachineType::Pointer()};
        MachineSignature sig(0, 6, arg_types);
        CallC(&sig, ExternalReference::wasm_array_fill(),
              {array, index, length,
               __ Word32Constant(emit_write_barrier ? 1 : 0),
               __ Word32Constant(element_type.raw_bit_field()), stack_slot});
        GOTO(done);
      }
    }
 
    ScopedVar<Word32> current_index(this, index);
 
    WHILE(__ Uint32LessThan(current_index, __ Word32Add(index, length))) {
      __ ArraySet(array, current_index, value, type->element_type());
      current_index = __ Word32Add(current_index, 1);
    }
 
    GOTO(done);
 
    BIND(done);
  }
 
  V<WordPtr> StoreInStackSlot(OpIndex value, wasm::ValueType type) {
    // The input is unpacked.
    ValueType input_type = type.Unpacked();
 
    switch (type.kind()) {
      case wasm::kI32:
      case wasm::kI8:
      case wasm::kI16:
      case wasm::kI64:
      case wasm::kF32:
      case wasm::kF64:
      case wasm::kRefNull:
      case wasm::kRef:
        break;
      case wasm::kS128:
        // We can only get here if {value} is the constant 0.
        DCHECK(__ output_graph().Get(value).Cast<Simd128ConstantOp>().IsZero());
        value = __ Word64Constant(uint64_t{0});
        input_type = kWasmI64;
        break;
      case wasm::kF16:
        UNIMPLEMENTED();
      case wasm::kVoid:
      case kTop:
      case wasm::kBottom:
        UNREACHABLE();
    }
 
    // Differently to values on the heap, stack slots are always uncompressed.
    MemoryRepresentation memory_rep =
        type.is_reference() ? MemoryRepresentation::UncompressedTaggedPointer()
                            : MemoryRepresentation::FromMachineRepresentation(
                                  input_type.machine_representation());
    V<WordPtr> stack_slot =
        __ StackSlot(memory_rep.SizeInBytes(), memory_rep.SizeInBytes(),
                     type.is_reference());
    __ Store(stack_slot, value, StoreOp::Kind::RawAligned(), memory_rep,
             compiler::WriteBarrierKind::kNoWriteBarrier);
    return stack_slot;
  }
 
  bool InlineTargetIsTypeCompatible(const WasmModule* module,
                                    const FunctionSig* sig,
                                    const FunctionSig* inlinee) {
    if (sig->parameter_count() != inlinee->parameter_count()) return false;
    if (sig->return_count() != inlinee->return_count()) return false;
    for (size_t i = 0; i < sig->return_count(); ++i) {
      if (!IsSubtypeOf(inlinee->GetReturn(i), sig->GetReturn(i), module))
        return false;
    }
    for (size_t i = 0; i < sig->parameter_count(); ++i) {
      if (!IsSubtypeOf(sig->GetParam(i), inlinee->GetParam(i), module))
        return false;
    }
    return true;
  }
 
  void InlineWasmCall(FullDecoder* decoder, uint32_t func_index,
                      const FunctionSig* sig, uint32_t feedback_case,
                      bool is_tail_call, const Value args[], Value returns[]) {
    DCHECK_IMPLIES(is_tail_call, returns == nullptr);
    const WasmFunction& inlinee = decoder->module_->functions[func_index];
    // In a corrupted sandbox, we can't trust the collected feedback.
    SBXCHECK(InlineTargetIsTypeCompatible(decoder->module_, sig, inlinee.sig));
 
    SmallZoneVector<OpIndex, 16> inlinee_args(
        inlinee.sig->parameter_count() + 1, decoder->zone_);
    bool inlinee_is_shared = decoder->module_->function_is_shared(func_index);
    inlinee_args[0] = trusted_instance_data(inlinee_is_shared);
    for (size_t i = 0; i < inlinee.sig->parameter_count(); i++) {
      inlinee_args[i + 1] = args[i].op;
    }
 
    base::Vector<const uint8_t> function_bytes =
        wire_bytes_->GetCode(inlinee.code);
 
    const wasm::FunctionBody inlinee_body{
        inlinee.sig, inlinee.code.offset(), function_bytes.begin(),
        function_bytes.end(), inlinee_is_shared};
 
    // If the inlinee was not validated before, do that now.
    if (V8_UNLIKELY(!decoder->module_->function_was_validated(func_index))) {
      if (ValidateFunctionBody(decoder->zone_, decoder->enabled_,
                               decoder->module_, decoder->detected_,
                               inlinee_body)
              .failed()) {
        // At this point we cannot easily raise a compilation error any more.
        // Since this situation is highly unlikely though, we just ignore this
        // inlinee, emit a regular call, and move on. The same validation error
        // will be triggered again when actually compiling the invalid function.
        V<WordPtr> callee =
            __ RelocatableConstant(func_index, RelocInfo::WASM_CALL);
        if (is_tail_call) {
          BuildWasmMaybeReturnCall(
              decoder, sig, callee,
              trusted_instance_data(
                  decoder->module_->function_is_shared(func_index)),
              args);
        } else {
          BuildWasmCall(decoder, sig, callee,
                        trusted_instance_data(
                            decoder->module_->function_is_shared(func_index)),
                        args, returns);
        }
        return;
      }
      decoder->module_->set_function_validated(func_index);
    }
 
    BlockPhis fresh_return_phis(decoder->zone_);
 
    Mode inlinee_mode;
    TSBlock* callee_catch_block = nullptr;
    TSBlock* callee_return_block;
    BlockPhis* inlinee_return_phis;
 
    if (is_tail_call) {
      if (mode_ == kInlinedTailCall || mode_ == kRegular) {
        inlinee_mode = kInlinedTailCall;
        callee_return_block = nullptr;
        inlinee_return_phis = nullptr;
      } else {
        // A tail call inlined inside a regular call inherits its settings,
        // as any `return` statement returns to the nearest non-tail caller.
        inlinee_mode = mode_;
        callee_return_block = return_block_;
        inlinee_return_phis = return_phis_;
        if (mode_ == kInlinedWithCatch) {
          callee_catch_block = return_catch_block_;
        }
      }
    } else {
      // Regular call (i.e. not a tail call).
      if (mode_ == kInlinedWithCatch || decoder->current_catch() != -1) {
        inlinee_mode = kInlinedWithCatch;
        // TODO(14108): If this is a nested inlining, can we forward the
        // caller's catch block instead?
        callee_catch_block = __ NewBlock();
      } else {
        inlinee_mode = kInlinedUnhandled;
      }
      callee_return_block = __ NewBlock();
      inlinee_return_phis = &fresh_return_phis;
    }
 
    OptionalV<FrameState> frame_state;
    if (deopts_enabled()) {
      frame_state = is_tail_call
                        ? parent_frame_state_
                        : CreateFrameState(decoder, sig, /*funcref*/ nullptr,
                                           /*args*/ nullptr);
    }
 
    WasmFullDecoder<TurboshaftGraphBuildingInterface::ValidationTag,
                    TurboshaftGraphBuildingInterface>
        inlinee_decoder(decoder->zone_, decoder->module_, decoder->enabled_,
                        decoder->detected_, inlinee_body, decoder->zone_, env_,
                        asm_, inlinee_mode, instance_cache_, assumptions_,
                        inlining_positions_, func_index, inlinee_is_shared,
                        wire_bytes_, base::VectorOf(inlinee_args),
                        callee_return_block, inlinee_return_phis,
                        callee_catch_block, is_tail_call, frame_state);
    SourcePosition call_position =
        SourcePosition(decoder->position(), inlining_id_ == kNoInliningId
                                                ? SourcePosition::kNotInlined
                                                : inlining_id_);
    inlining_positions_->push_back(
        {static_cast<int>(func_index), is_tail_call, call_position});
    inlinee_decoder.interface().set_inlining_id(
        static_cast<uint8_t>(inlining_positions_->size() - 1));
    inlinee_decoder.interface().set_parent_position(call_position);
    // Explicitly disable deopts if it has already been disabled for this
    // function.
    if (!deopts_enabled()) {
      inlinee_decoder.interface().disable_deopts();
    }
    if (v8_flags.liftoff) {
      if (inlining_decisions_ && inlining_decisions_->feedback_found()) {
        inlinee_decoder.interface().set_inlining_decisions(
            inlining_decisions_
                ->function_calls()[feedback_slot_][feedback_case]);
      }
    } else {
      no_liftoff_inlining_budget_ -= inlinee.code.length();
      inlinee_decoder.interface().set_no_liftoff_inlining_budget(
          no_liftoff_inlining_budget_);
    }
    inlinee_decoder.Decode();
    // The function was already validated above.
    DCHECK(inlinee_decoder.ok());
 
    DCHECK_IMPLIES(!is_tail_call && inlinee_mode == kInlinedWithCatch,
                   inlinee_return_phis != nullptr);
 
    if (!is_tail_call && inlinee_mode == kInlinedWithCatch &&
        !inlinee_return_phis->incoming_exceptions().empty()) {
      // We need to handle exceptions in the inlined call.
      __ Bind(callee_catch_block);
      OpIndex exception =
          MaybePhi(inlinee_return_phis->incoming_exceptions(), kWasmExternRef);
      bool handled_in_this_frame = decoder->current_catch() != -1;
      TSBlock* catch_block;
      if (handled_in_this_frame) {
        Control* current_catch =
            decoder->control_at(decoder->control_depth_of_current_catch());
        catch_block = current_catch->false_or_loop_or_catch_block;
        // The exceptional operation could have modified memory size; we need
        // to reload the memory context into the exceptional control path.
        instance_cache_.ReloadCachedMemory();
        SetupControlFlowEdge(decoder, catch_block, 0, exception);
      } else {
        DCHECK_EQ(mode_, kInlinedWithCatch);
        catch_block = return_catch_block_;
        if (exception.valid()) return_phis_->AddIncomingException(exception);
        // Reloading the InstanceCache will happen when {return_exception_phis_}
        // are retrieved.
      }
      __ Goto(catch_block);
    }
 
    if (!is_tail_call) {
      __ Bind(callee_return_block);
      BlockPhis* return_phis = inlinee_decoder.interface().return_phis();
      size_t return_count = inlinee.sig->return_count();
      for (size_t i = 0; i < return_count; i++) {
        returns[i].op =
            MaybePhi(return_phis->phi_inputs(i), return_phis->phi_type(i));
      }
    }
 
    if (!v8_flags.liftoff) {
      set_no_liftoff_inlining_budget(
          inlinee_decoder.interface().no_liftoff_inlining_budget());
    }
  }
 
  TrapId GetTrapIdForTrap(wasm::TrapReason reason) {
    switch (reason) {
#define TRAPREASON_TO_TRAPID(name)                                 \
  case wasm::k##name:                                              \
    static_assert(static_cast<int>(TrapId::k##name) ==             \
                      static_cast<int>(Builtin::kThrowWasm##name), \
                  "trap id mismatch");                             \
    return TrapId::k##name;
      FOREACH_WASM_TRAPREASON(TRAPREASON_TO_TRAPID)
#undef TRAPREASON_TO_TRAPID
      default:
        UNREACHABLE();
    }
  }
 
  // We need this shift so that resulting OpIndex offsets are multiples of
  // `sizeof(OperationStorageSlot)`.
  static constexpr int kPositionFieldShift = 3;
  static_assert(sizeof(compiler::turboshaft::OperationStorageSlot) ==
                1 << kPositionFieldShift);
  static constexpr int kPositionFieldSize = 23;
  static_assert(kV8MaxWasmFunctionSize < (1 << kPositionFieldSize));
  static constexpr int kInliningIdFieldSize = 6;
  static constexpr uint8_t kNoInliningId = 63;
  static_assert((1 << kInliningIdFieldSize) - 1 == kNoInliningId);
  // We need to assign inlining_ids to inlined nodes.
  static_assert(kNoInliningId > InliningTree::kMaxInlinedCount);
 
  // We encode the wasm code position and the inlining index in an OpIndex
  // stored in the output graph's node origins.
  using PositionField =
      base::BitField<WasmCodePosition, kPositionFieldShift, kPositionFieldSize>;
  using InliningIdField = PositionField::Next<uint8_t, kInliningIdFieldSize>;
 
  OpIndex WasmPositionToOpIndex(WasmCodePosition position, int inlining_id) {
    return OpIndex::FromOffset(PositionField::encode(position) |
                               InliningIdField::encode(inlining_id));
  }
 
  SourcePosition OpIndexToSourcePosition(OpIndex index) {
    DCHECK(index.valid());
    uint8_t inlining_id = InliningIdField::decode(index.offset());
    return SourcePosition(PositionField::decode(index.offset()),
                          inlining_id == kNoInliningId
                              ? SourcePosition::kNotInlined
                              : inlining_id);
  }
 
  BranchHint GetBranchHint(FullDecoder* decoder) {
    // Minimize overhead when branch hints aren't being used.
    if (branch_hinting_mode_ == BranchHintingMode::kNone) {
      return BranchHint::kNone;
    }
    if (branch_hinting_mode_ == BranchHintingMode::kModuleProvided) {
      return branch_hints_->GetHintFor(decoder->pc_relative_offset());
    }
    if (branch_hinting_mode_ == BranchHintingMode::kStress) {
      return branch_hinting_stresser_.GetNextHint();
    }
    UNREACHABLE();
  }
 
 private:
  bool should_inline(FullDecoder* decoder, int feedback_slot, int size) {
    if (!v8_flags.wasm_inlining) return false;
    // TODO(42204563,41480394,335082212): Do not inline if the current function
    // is shared (which also implies the target cannot be shared either).
    if (shared_) return false;
 
    // Configuration without Liftoff and feedback, e.g., for testing.
    if (!v8_flags.liftoff) {
      return size < no_liftoff_inlining_budget_ &&
             // In a production configuration, `InliningTree` decides what to
             // (not) inline, e.g., asm.js functions or to not exceed
             // `kMaxInlinedCount`. But without Liftoff, we need to "manually"
             // comply with these constraints here.
             !is_asmjs_module(decoder->module_) &&
             inlining_positions_->size() < InliningTree::kMaxInlinedCount;
    }
 
    // Default, production configuration: Liftoff collects feedback, which
    // decides whether we inline:
    if (inlining_decisions_ && inlining_decisions_->feedback_found()) {
      DCHECK_GT(inlining_decisions_->function_calls().size(), feedback_slot);
      // We should inline if at least one case for this feedback slot needs
      // to be inlined.
      for (InliningTree* tree :
           inlining_decisions_->function_calls()[feedback_slot]) {
        if (tree && tree->is_inlined()) {
          DCHECK(!decoder->module_->function_is_shared(tree->function_index()));
          return true;
        }
      }
      return false;
    }
    return false;
  }
 
  void set_inlining_decisions(InliningTree* inlining_decisions) {
    inlining_decisions_ = inlining_decisions;
  }
 
  BlockPhis* return_phis() { return return_phis_; }
  void set_inlining_id(uint8_t inlining_id) {
    DCHECK_NE(inlining_id, kNoInliningId);
    inlining_id_ = inlining_id;
  }
  void set_parent_position(SourcePosition position) {
    parent_position_ = position;
  }
  int no_liftoff_inlining_budget() { return no_liftoff_inlining_budget_; }
  void set_no_liftoff_inlining_budget(int no_liftoff_inlining_budget) {
    no_liftoff_inlining_budget_ = no_liftoff_inlining_budget;
  }
 
  void disable_deopts() { deopts_enabled_ = false; }
  bool deopts_enabled() {
    if (!deopts_enabled_.has_value()) {
      deopts_enabled_ = v8_flags.wasm_deopt;
      if (v8_flags.wasm_deopt) {
        const wasm::TypeFeedbackStorage& feedback = env_->module->type_feedback;
        base::MutexGuard mutex_guard(&feedback.mutex);
        auto iter = feedback.deopt_count_for_function.find(func_index_);
        if (iter != feedback.deopt_count_for_function.end() &&
            iter->second >= v8_flags.wasm_deopts_per_function_limit) {
          deopts_enabled_ = false;
          if (v8_flags.trace_wasm_inlining) {
            PrintF(
                "[function %d%s: Disabling deoptimizations for speculative "
                "inlining as the deoptimization limit (%u) for this function "
                "is reached or exceeded (%zu)]\n",
                func_index_, mode_ == kRegular ? "" : " (inlined)",
                iter->second, v8_flags.wasm_deopts_per_function_limit.value());
          }
        }
      }
    }
    return deopts_enabled_.value();
  }
 
  V<WasmTrustedInstanceData> trusted_instance_data(bool element_is_shared) {
    DCHECK_IMPLIES(shared_, element_is_shared);
    return (element_is_shared && !shared_)
               ? LOAD_IMMUTABLE_PROTECTED_INSTANCE_FIELD(
                     instance_cache_.trusted_instance_data(), SharedPart,
                     WasmTrustedInstanceData)
               : instance_cache_.trusted_instance_data();
  }
 
  V<FixedArray> managed_object_maps(bool type_is_shared) {
    DCHECK_IMPLIES(shared_, type_is_shared);
    if (type_is_shared && !shared_) {
      V<WasmTrustedInstanceData> shared_instance = trusted_instance_data(true);
      return LOAD_IMMUTABLE_INSTANCE_FIELD(
          shared_instance, ManagedObjectMaps,
          MemoryRepresentation::TaggedPointer());
    } else {
      return instance_cache_.managed_object_maps();
    }
  }
 
 private:
  Mode mode_;
  ZoneAbslFlatHashMap<TSBlock*, BlockPhis> block_phis_;
  CompilationEnv* env_;
  // Only used for "top-level" instantiations, not for inlining.
  std::unique_ptr<InstanceCache> owned_instance_cache_;
 
  // The instance cache to use (may be owned or passed in).
  InstanceCache& instance_cache_;
 
  // Yes, a *pointer* to a `unique_ptr`. The `unique_ptr` is allocated lazily
  // when adding the first assumption.
  std::unique_ptr<AssumptionsJournal>* assumptions_;
  ZoneVector<WasmInliningPosition>* inlining_positions_;
  uint8_t inlining_id_ = kNoInliningId;
  ZoneVector<OpIndex> ssa_env_;
  compiler::NullCheckStrategy null_check_strategy_ =
      trap_handler::IsTrapHandlerEnabled() && V8_STATIC_ROOTS_BOOL
          ? compiler::NullCheckStrategy::kTrapHandler
          : compiler::NullCheckStrategy::kExplicit;
  int func_index_;
  bool shared_;
  const WireBytesStorage* wire_bytes_;
  BranchHintingMode branch_hinting_mode_;
  const BranchHintMap* branch_hints_ = nullptr;
  BranchHintingStresser branch_hinting_stresser_;
  InliningTree* inlining_decisions_ = nullptr;
  int feedback_slot_ = -1;
  // Inlining budget in case of --no-liftoff.
  int no_liftoff_inlining_budget_ = 0;
  uint32_t liftoff_frame_size_ =
      FunctionTypeFeedback::kUninitializedLiftoffFrameSize;
 
  /* Used for inlining modes */
  // Contains real parameters for this inlined function, including the instance.
  // Used only in StartFunction();
  base::Vector<OpIndex> real_parameters_;
  // The block where this function returns its values (passed by the caller).
  TSBlock* return_block_ = nullptr;
  // The return values and exception values for this function.
  // The caller will reconstruct each one with a Phi.
  BlockPhis* return_phis_ = nullptr;
  // The block where exceptions from this function are caught (passed by the
  // caller).
  TSBlock* return_catch_block_ = nullptr;
  // The position of the call that is being inlined.
  SourcePosition parent_position_;
  bool is_inlined_tail_call_ = false;
 
  std::optional<bool> deopts_enabled_;
  OptionalV<FrameState> parent_frame_state_;
};
 
V8_EXPORT_PRIVATE void BuildTSGraph(
    compiler::turboshaft::PipelineData* data, AccountingAllocator* allocator,
    CompilationEnv* env, WasmDetectedFeatures* detected, Graph& graph,
    const FunctionBody& func_body, const WireBytesStorage* wire_bytes,
    std::unique_ptr<AssumptionsJournal>* assumptions,
    ZoneVector<WasmInliningPosition>* inlining_positions, int func_index) {
  DCHECK(env->module->function_was_validated(func_index));
  Zone zone(allocator, ZONE_NAME);
  WasmGraphBuilderBase::Assembler assembler(data, graph, graph, &zone);
  WasmFullDecoder<TurboshaftGraphBuildingInterface::ValidationTag,
                  TurboshaftGraphBuildingInterface>
      decoder(&zone, env->module, env->enabled_features, detected, func_body,
              &zone, env, assembler, assumptions, inlining_positions,
              func_index, func_body.is_shared, wire_bytes);
  decoder.Decode();
  // The function was already validated, so graph building must always succeed.
  DCHECK(decoder.ok());
}
 
#undef LOAD_IMMUTABLE_INSTANCE_FIELD
#undef LOAD_INSTANCE_FIELD
#undef LOAD_ROOT
#include "src/compiler/turboshaft/undef-assembler-macros.inc"
 
}  // namespace v8::internal::wasm