fast-api-call-lowering-reducer.h

Go to the documentation of this file.

// Copyright 2023 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
 
#ifndef V8_COMPILER_TURBOSHAFT_FAST_API_CALL_LOWERING_REDUCER_H_
#define V8_COMPILER_TURBOSHAFT_FAST_API_CALL_LOWERING_REDUCER_H_
 
#include "include/v8-fast-api-calls.h"
#include "src/compiler/fast-api-calls.h"
#include "src/compiler/globals.h"
#include "src/compiler/turboshaft/assembler.h"
#include "src/compiler/turboshaft/copying-phase.h"
#include "src/compiler/turboshaft/index.h"
#include "src/compiler/turboshaft/operations.h"
#include "src/compiler/turboshaft/phase.h"
#include "src/compiler/turboshaft/representations.h"
 
namespace v8::internal::compiler::turboshaft {
 
#include "src/compiler/turboshaft/define-assembler-macros.inc"
 
template <typename Next>
class FastApiCallLoweringReducer : public Next {
 public:
  TURBOSHAFT_REDUCER_BOILERPLATE(FastApiCallLowering)
 
  OpIndex REDUCE(FastApiCall)(
      V<FrameState> frame_state, V<Object> data_argument, V<Context> context,
      base::Vector<const OpIndex> arguments,
      const FastApiCallParameters* parameters,
      base::Vector<const RegisterRepresentation> out_reps) {
    __ data() -> set_graph_has_lowered_fast_api_calls();
 
    FastApiCallFunction c_function = parameters->c_function;
    const auto& c_signature = parameters->c_signature();
    const int c_arg_count = c_signature->ArgumentCount();
    DCHECK_EQ(c_arg_count, arguments.size());
 
    Label<> handle_error(this);
    Label<Word32> done(this);
    Variable result = __ NewVariable(RegisterRepresentation::FromCTypeInfo(
        c_signature->ReturnInfo(), c_signature->GetInt64Representation()));
 
    OpIndex callee = __ ExternalConstant(ExternalReference::Create(
        c_function.address, ExternalReference::FAST_C_CALL));
 
    base::SmallVector<OpIndex, 16> args;
    for (int i = 0; i < c_arg_count; ++i) {
        CTypeInfo type = c_signature->ArgumentInfo(i);
        args.push_back(AdaptFastCallArgument(arguments[i], type, handle_error));
    }
 
    // While adapting the arguments, we might have noticed an inconsistency that
    // lead to unconditionally jumping to {handle_error}. If this happens, then
    // we don't emit the call.
    if (V8_LIKELY(!__ generating_unreachable_operations())) {
      MachineSignature::Builder builder(
          __ graph_zone(), 1,
          c_arg_count + (c_signature->HasOptions() ? 1 : 0));
 
      builder.AddReturn(MachineType::TypeForCType(c_signature->ReturnInfo()));
 
      for (int i = 0; i < c_arg_count; ++i) {
        CTypeInfo type = c_signature->ArgumentInfo(i);
        START_ALLOW_USE_DEPRECATED()
        MachineType machine_type =
            type.GetSequenceType() == CTypeInfo::SequenceType::kScalar
                ? MachineType::TypeForCType(type)
                : MachineType::AnyTagged();
        END_ALLOW_USE_DEPRECATED()
        builder.AddParam(machine_type);
      }
 
      OpIndex stack_slot;
      if (c_signature->HasOptions()) {
        const int kAlign = alignof(v8::FastApiCallbackOptions);
        const int kSize = sizeof(v8::FastApiCallbackOptions);
        // If this check fails, you've probably added new fields to
        // v8::FastApiCallbackOptions, which means you'll need to write code
        // that initializes and reads from them too.
        static_assert(kSize == sizeof(uintptr_t) * 2);
        stack_slot = __ StackSlot(kSize, kAlign);
 
        // isolate
        __ StoreOffHeap(
            stack_slot,
            __ ExternalConstant(ExternalReference::isolate_address()),
            MemoryRepresentation::UintPtr(),
            offsetof(v8::FastApiCallbackOptions, isolate));
        // data = data_argument
        OpIndex data_argument_to_pass = __ AdaptLocalArgument(data_argument);
        __ StoreOffHeap(stack_slot, data_argument_to_pass,
                        MemoryRepresentation::UintPtr(),
                        offsetof(v8::FastApiCallbackOptions, data));
 
        args.push_back(stack_slot);
        builder.AddParam(MachineType::Pointer());
      }
 
      // Build the actual call.
      const TSCallDescriptor* call_descriptor = TSCallDescriptor::Create(
          Linkage::GetSimplifiedCDescriptor(__ graph_zone(), builder.Get(),
                                            CallDescriptor::kNeedsFrameState),
          CanThrow::kNo, LazyDeoptOnThrow::kNo, __ graph_zone());
      OpIndex c_call_result = WrapFastCall(call_descriptor, callee, frame_state,
                                           context, base::VectorOf(args));
 
      Label<> trigger_exception(this);
 
      V<Object> exception =
          __ Load(__ ExternalConstant(ExternalReference::Create(
                      IsolateAddressId::kExceptionAddress, isolate_)),
                  LoadOp::Kind::RawAligned(), MemoryRepresentation::UintPtr());
      GOTO_IF_NOT(LIKELY(__ TaggedEqual(
                      exception,
                      __ HeapConstant(isolate_->factory()->the_hole_value()))),
                  trigger_exception);
 
      V<Any> fast_call_result = ConvertReturnValue(c_signature, c_call_result);
      __ SetVariable(result, fast_call_result);
 
      GOTO(done, FastApiCallOp::kSuccessValue);
      BIND(trigger_exception);
      __ template CallRuntime<
          typename RuntimeCallDescriptor::PropagateException>(
          isolate_, frame_state, __ NoContextConstant(), LazyDeoptOnThrow::kNo,
          {});
 
      __ Unreachable();
    }
 
    if (BIND(handle_error)) {
      __ SetVariable(result, DefaultReturnValue(c_signature));
      // We pass Tagged<Smi>(0) as the value here, although this should never be
      // visible when calling code reacts to `kFailureValue` properly.
      GOTO(done, FastApiCallOp::kFailureValue);
    }
 
    BIND(done, state);
    return __ Tuple(state, __ GetVariable(result));
  }
 
 private:
  template <typename T>
  V<T> Checked(V<Tuple<T, Word32>> result, Label<>& otherwise) {
    V<Word32> result_state = __ template Projection<1>(result);
    GOTO_IF_NOT(__ Word32Equal(result_state, TryChangeOp::kSuccessValue),
                otherwise);
    return __ template Projection<0>(result);
  }
 
  OpIndex AdaptFastCallArgument(OpIndex argument, CTypeInfo arg_type,
                                Label<>& handle_error) {
    START_ALLOW_USE_DEPRECATED()
    switch (arg_type.GetSequenceType()) {
      case CTypeInfo::SequenceType::kScalar: {
        uint8_t flags = static_cast<uint8_t>(arg_type.GetFlags());
        if (flags & static_cast<uint8_t>(CTypeInfo::Flags::kEnforceRangeBit)) {
          switch (arg_type.GetType()) {
            case CTypeInfo::Type::kInt32: {
              auto result = __ TryTruncateFloat64ToInt32(argument);
              return Checked(result, handle_error);
            }
            case CTypeInfo::Type::kUint32: {
              auto result = __ TryTruncateFloat64ToUint32(argument);
              return Checked(result, handle_error);
            }
            case CTypeInfo::Type::kInt64: {
              auto result = __ TryTruncateFloat64ToInt64(argument);
              return Checked(result, handle_error);
            }
            case CTypeInfo::Type::kUint64: {
              auto result = __ TryTruncateFloat64ToUint64(argument);
              return Checked(result, handle_error);
            }
            default: {
              GOTO(handle_error);
              return argument;
            }
          }
        } else if (flags & static_cast<uint8_t>(CTypeInfo::Flags::kClampBit)) {
          return ClampFastCallArgument(argument, arg_type.GetType());
        } else {
          switch (arg_type.GetType()) {
            case CTypeInfo::Type::kV8Value: {
              return __ AdaptLocalArgument(argument);
            }
            case CTypeInfo::Type::kFloat32: {
              return __ TruncateFloat64ToFloat32(argument);
            }
            case CTypeInfo::Type::kPointer: {
              // Check that the value is a HeapObject.
              GOTO_IF(__ ObjectIsSmi(argument), handle_error);
              Label<WordPtr> done(this);
 
              // Check if the value is null.
              GOTO_IF(UNLIKELY(__ TaggedEqual(
                          argument, __ HeapConstant(factory_->null_value()))),
                      done, 0);
 
              // Check that the value is a JSExternalObject.
              GOTO_IF_NOT(
                  __ TaggedEqual(__ LoadMapField(argument),
                                 __ HeapConstant(factory_->external_map())),
                  handle_error);
 
              GOTO(done, __ template LoadField<WordPtr>(
                             V<HeapObject>::Cast(argument),
                             AccessBuilder::ForJSExternalObjectValue()));
 
              BIND(done, result);
              return result;
            }
            case CTypeInfo::Type::kSeqOneByteString: {
              // Check that the value is a HeapObject.
              GOTO_IF(__ ObjectIsSmi(argument), handle_error);
              V<HeapObject> argument_obj = V<HeapObject>::Cast(argument);
 
              V<Map> map = __ LoadMapField(argument_obj);
              V<Word32> instance_type = __ LoadInstanceTypeField(map);
 
              V<Word32> encoding = __ Word32BitwiseAnd(
                  instance_type, kStringRepresentationAndEncodingMask);
              GOTO_IF_NOT(__ Word32Equal(encoding, kSeqOneByteStringTag),
                          handle_error);
 
              V<WordPtr> length_in_bytes = __ template LoadField<WordPtr>(
                  argument_obj, AccessBuilder::ForStringLength());
              V<WordPtr> data_ptr = __ GetElementStartPointer(
                  argument_obj, AccessBuilder::ForSeqOneByteStringCharacter());
 
              constexpr int kAlign = alignof(FastOneByteString);
              constexpr int kSize = sizeof(FastOneByteString);
              static_assert(kSize == sizeof(uintptr_t) + sizeof(size_t),
                            "The size of "
                            "FastOneByteString isn't equal to the sum of its "
                            "expected members.");
              OpIndex stack_slot = __ StackSlot(kSize, kAlign);
              __ StoreOffHeap(stack_slot, data_ptr,
                              MemoryRepresentation::UintPtr());
              __ StoreOffHeap(stack_slot, length_in_bytes,
                              MemoryRepresentation::Uint32(), sizeof(size_t));
              static_assert(sizeof(uintptr_t) == sizeof(size_t),
                            "The string length can't "
                            "fit the PointerRepresentation used to store it.");
              return stack_slot;
            }
            default: {
              return argument;
            }
          }
        }
      }
      case CTypeInfo::SequenceType::kIsSequence: {
        CHECK_EQ(arg_type.GetType(), CTypeInfo::Type::kVoid);
 
        // Check that the value is a HeapObject.
        GOTO_IF(__ ObjectIsSmi(argument), handle_error);
 
        // Check that the value is a JSArray.
        V<Map> map = __ LoadMapField(argument);
        V<Word32> instance_type = __ LoadInstanceTypeField(map);
        GOTO_IF_NOT(__ Word32Equal(instance_type, JS_ARRAY_TYPE), handle_error);
 
        return __ AdaptLocalArgument(argument);
      }
      default: {
        UNREACHABLE();
      }
    }
    END_ALLOW_USE_DEPRECATED()
  }
 
  OpIndex ClampFastCallArgument(V<Float64> argument,
                                CTypeInfo::Type scalar_type) {
    double min, max;
    switch (scalar_type) {
      case CTypeInfo::Type::kInt32:
        min = std::numeric_limits<int32_t>::min();
        max = std::numeric_limits<int32_t>::max();
        break;
      case CTypeInfo::Type::kUint32:
        min = 0;
        max = std::numeric_limits<uint32_t>::max();
        break;
      case CTypeInfo::Type::kInt64:
        min = kMinSafeInteger;
        max = kMaxSafeInteger;
        break;
      case CTypeInfo::Type::kUint64:
        min = 0;
        max = kMaxSafeInteger;
        break;
      default:
        UNREACHABLE();
    }
 
    V<Float64> clamped =
        __ Conditional(__ Float64LessThan(min, argument),
                       __ Conditional(__ Float64LessThan(argument, max),
                                      argument, __ Float64Constant(max)),
                       __ Float64Constant(min));
 
    Label<Float64> done(this);
    V<Float64> rounded = __ Float64RoundTiesEven(clamped);
    GOTO_IF(__ Float64IsNaN(rounded), done, 0.0);
    GOTO(done, rounded);
 
    BIND(done, rounded_result);
    switch (scalar_type) {
      case CTypeInfo::Type::kInt32:
        return __ ReversibleFloat64ToInt32(rounded_result);
      case CTypeInfo::Type::kUint32:
        return __ ReversibleFloat64ToUint32(rounded_result);
      case CTypeInfo::Type::kInt64:
        return __ ReversibleFloat64ToInt64(rounded_result);
      case CTypeInfo::Type::kUint64:
        return __ ReversibleFloat64ToUint64(rounded_result);
      default:
        UNREACHABLE();
    }
  }
 
  V<Any> DefaultReturnValue(const CFunctionInfo* c_signature) {
    switch (c_signature->ReturnInfo().GetType()) {
      case CTypeInfo::Type::kVoid:
        return __ HeapConstant(factory_->undefined_value());
      case CTypeInfo::Type::kBool:
      case CTypeInfo::Type::kInt32:
      case CTypeInfo::Type::kUint32:
        return __ Word32Constant(0);
      case CTypeInfo::Type::kInt64:
      case CTypeInfo::Type::kUint64: {
        CFunctionInfo::Int64Representation repr =
            c_signature->GetInt64Representation();
        if (repr == CFunctionInfo::Int64Representation::kBigInt) {
          return __ Word64Constant(int64_t{0});
        }
        DCHECK_EQ(repr, CFunctionInfo::Int64Representation::kNumber);
        return __ Float64Constant(0);
      }
      case CTypeInfo::Type::kFloat32:
        return __ Float32Constant(0);
      case CTypeInfo::Type::kFloat64:
        return __ Float64Constant(0);
      case CTypeInfo::Type::kPointer:
        return __ HeapConstant(factory_->undefined_value());
      case CTypeInfo::Type::kAny:
      case CTypeInfo::Type::kSeqOneByteString:
      case CTypeInfo::Type::kV8Value:
      case CTypeInfo::Type::kApiObject:
      case CTypeInfo::Type::kUint8:
        UNREACHABLE();
    }
  }
 
  V<Any> ConvertReturnValue(const CFunctionInfo* c_signature, OpIndex result) {
    switch (c_signature->ReturnInfo().GetType()) {
      case CTypeInfo::Type::kVoid:
        return __ HeapConstant(factory_->undefined_value());
      case CTypeInfo::Type::kBool:
        static_assert(sizeof(bool) == 1, "unsupported bool size");
        return __ Word32BitwiseAnd(result, __ Word32Constant(0xFF));
      case CTypeInfo::Type::kInt32:
      case CTypeInfo::Type::kUint32:
      case CTypeInfo::Type::kFloat32:
      case CTypeInfo::Type::kFloat64:
        return result;
      case CTypeInfo::Type::kInt64: {
        CFunctionInfo::Int64Representation repr =
            c_signature->GetInt64Representation();
        if (repr == CFunctionInfo::Int64Representation::kBigInt) {
          return result;
        }
        DCHECK_EQ(repr, CFunctionInfo::Int64Representation::kNumber);
        return __ ChangeInt64ToFloat64(result);
      }
      case CTypeInfo::Type::kUint64: {
        CFunctionInfo::Int64Representation repr =
            c_signature->GetInt64Representation();
        if (repr == CFunctionInfo::Int64Representation::kBigInt) {
          return result;
        }
        DCHECK_EQ(repr, CFunctionInfo::Int64Representation::kNumber);
        return __ ChangeUint64ToFloat64(result);
      }
 
      case CTypeInfo::Type::kPointer:
        return BuildAllocateJSExternalObject(result);
      case CTypeInfo::Type::kAny:
      case CTypeInfo::Type::kSeqOneByteString:
      case CTypeInfo::Type::kV8Value:
      case CTypeInfo::Type::kApiObject:
      case CTypeInfo::Type::kUint8:
        UNREACHABLE();
    }
  }
 
  V<HeapObject> BuildAllocateJSExternalObject(V<WordPtr> pointer) {
    Label<HeapObject> done(this);
 
    // Check if the pointer is a null pointer.
    GOTO_IF(__ WordPtrEqual(pointer, 0), done,
            __ HeapConstant(factory_->null_value()));
 
    Uninitialized<HeapObject> external =
        __ Allocate(JSExternalObject::kHeaderSize, AllocationType::kYoung);
    __ InitializeField(external, AccessBuilder::ForMap(),
                       __ HeapConstant(factory_->external_map()));
    V<FixedArray> empty_fixed_array =
        __ HeapConstant(factory_->empty_fixed_array());
    __ InitializeField(external, AccessBuilder::ForJSObjectPropertiesOrHash(),
                       empty_fixed_array);
    __ InitializeField(external, AccessBuilder::ForJSObjectElements(),
                       empty_fixed_array);
 
#ifdef V8_ENABLE_SANDBOX
    OpIndex isolate_ptr =
        __ ExternalConstant(ExternalReference::isolate_address());
    MachineSignature::Builder builder(__ graph_zone(), 1, 2);
    builder.AddReturn(MachineType::Uint32());
    builder.AddParam(MachineType::Pointer());
    builder.AddParam(MachineType::Pointer());
    OpIndex allocate_and_initialize_young_external_pointer_table_entry =
        __ ExternalConstant(
            ExternalReference::
                allocate_and_initialize_young_external_pointer_table_entry());
    auto call_descriptor =
        Linkage::GetSimplifiedCDescriptor(__ graph_zone(), builder.Get());
    OpIndex handle = __ Call(
        allocate_and_initialize_young_external_pointer_table_entry,
        {isolate_ptr, pointer},
        TSCallDescriptor::Create(call_descriptor, CanThrow::kNo,
                                 LazyDeoptOnThrow::kNo, __ graph_zone()));
    __ InitializeField(
        external, AccessBuilder::ForJSExternalObjectPointerHandle(), handle);
#else
    __ InitializeField(external, AccessBuilder::ForJSExternalObjectValue(),
                       pointer);
#endif  // V8_ENABLE_SANDBOX
    GOTO(done, __ FinishInitialization(std::move(external)));
 
    BIND(done, result);
    return result;
  }
 
  OpIndex WrapFastCall(const TSCallDescriptor* descriptor, OpIndex callee,
                       V<FrameState> frame_state, V<Context> context,
                       base::Vector<const OpIndex> arguments) {
    // CPU profiler support.
    OpIndex target_address =
        __ IsolateField(IsolateFieldId::kFastApiCallTarget);
    __ StoreOffHeap(target_address, __ BitcastHeapObjectToWordPtr(callee),
                    MemoryRepresentation::UintPtr());
 
    OpIndex context_address = __ ExternalConstant(
        ExternalReference::Create(IsolateAddressId::kContextAddress, isolate_));
 
    __ StoreOffHeap(context_address, __ BitcastHeapObjectToWordPtr(context),
                    MemoryRepresentation::UintPtr());
 
    // Create the fast call.
    OpIndex result = __ Call(callee, frame_state, arguments, descriptor);
 
    // Reset the CPU profiler target address.
    __ StoreOffHeap(target_address, __ IntPtrConstant(0),
                    MemoryRepresentation::UintPtr());
 
#if DEBUG
    // Reset the context again after the call, to make sure nobody is using the
    // leftover context in the isolate.
    __ StoreOffHeap(context_address,
                    __ WordPtrConstant(Context::kInvalidContext),
                    MemoryRepresentation::UintPtr());
#endif
 
    return result;
  }
 
  Isolate* isolate_ = __ data() -> isolate();
  Factory* factory_ = isolate_->factory();
};
 
#include "src/compiler/turboshaft/undef-assembler-macros.inc"
 
}  // namespace v8::internal::compiler::turboshaft
 
#endif  // V8_COMPILER_TURBOSHAFT_FAST_API_CALL_LOWERING_REDUCER_H_