v8
V8 is Google’s open source high-performance JavaScript and WebAssembly engine, written in C++.
Loading...
Searching...
No Matches
interpreter-builtins-x64.cc
Go to the documentation of this file.
1// Copyright 2024 the V8 project authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "src/codegen/code-factory.h"
6#include "src/codegen/interface-descriptors-inl.h"
7#include "src/codegen/macro-assembler.h"
8#include "src/codegen/signature.h"
9#include "src/execution/frame-constants.h"
10#include "src/execution/isolate.h"
11
12#if V8_ENABLE_WEBASSEMBLY
13#include "src/wasm/interpreter/wasm-interpreter-runtime.h"
14#include "src/wasm/object-access.h"
15#include "src/wasm/wasm-objects.h"
16#endif // V8_ENABLE_WEBASSEMBLY
17
18namespace v8 {
19namespace internal {
20
21#define __ ACCESS_MASM(masm)
22
23#if V8_ENABLE_WEBASSEMBLY
24
25namespace {
26// Helper functions for the GenericJSToWasmInterpreterWrapper.
27
28void PrepareForJsToWasmConversionBuiltinCall(
29 MacroAssembler* masm, Register array_start, Register param_count,
30 Register current_param_slot, Register valuetypes_array_ptr,
31 Register wasm_instance, Register function_data) {
32 __ movq(
33 MemOperand(
34 rbp, BuiltinWasmInterpreterWrapperConstants::kGCScanSlotCountOffset),
35 Immediate(2));
36
37 // Pushes and puts the values in order onto the stack before builtin calls for
38 // the GenericJSToWasmInterpreterWrapper.
39 __ pushq(array_start);
40 __ pushq(param_count);
41 __ pushq(current_param_slot);
42 __ pushq(valuetypes_array_ptr);
43 // The following two slots contain tagged objects that need to be visited
44 // during GC.
45 __ pushq(wasm_instance);
46 __ pushq(function_data);
47 // We had to prepare the parameters for the Call: we have to put the context
48 // into rsi.
49 Register wasm_trusted_instance = wasm_instance;
50 __ LoadTrustedPointerField(
51 wasm_trusted_instance,
52 FieldMemOperand(wasm_instance, WasmInstanceObject::kTrustedDataOffset),
53 kWasmTrustedInstanceDataIndirectPointerTag, kScratchRegister);
54 __ LoadTaggedField(
55 rsi, MemOperand(wasm_trusted_instance,
56 wasm::ObjectAccess::ToTagged(
57 WasmTrustedInstanceData::kNativeContextOffset)));
58}
59
60void RestoreAfterJsToWasmConversionBuiltinCall(
61 MacroAssembler* masm, Register function_data, Register wasm_instance,
62 Register valuetypes_array_ptr, Register current_param_slot,
63 Register param_count, Register array_start) {
64 // Pop and load values from the stack in order into the registers after
65 // builtin calls for the GenericJSToWasmInterpreterWrapper.
66 __ popq(function_data);
67 __ popq(wasm_instance);
68 __ popq(valuetypes_array_ptr);
69 __ popq(current_param_slot);
70 __ popq(param_count);
71 __ popq(array_start);
72
73 __ movq(
74 MemOperand(
75 rbp, BuiltinWasmInterpreterWrapperConstants::kGCScanSlotCountOffset),
76 Immediate(0));
77}
78
79void PrepareForBuiltinCall(MacroAssembler* masm, Register array_start,
80 Register return_count, Register wasm_instance) {
81 // Pushes and puts the values in order onto the stack before builtin calls for
82 // the GenericJSToWasmInterpreterWrapper.
83 __ movq(
84 MemOperand(
85 rbp, BuiltinWasmInterpreterWrapperConstants::kGCScanSlotCountOffset),
86 Immediate(1));
87
88 __ pushq(array_start);
89 __ pushq(return_count);
90 // The following slot contains a tagged object that need to be visited during
91 // GC.
92 __ pushq(wasm_instance);
93 // We had to prepare the parameters for the Call: we have to put the context
94 // into rsi.
95 Register wasm_trusted_instance = wasm_instance;
96 __ LoadTrustedPointerField(
97 wasm_trusted_instance,
98 FieldMemOperand(wasm_instance, WasmInstanceObject::kTrustedDataOffset),
99 kWasmTrustedInstanceDataIndirectPointerTag, kScratchRegister);
100 __ LoadTaggedField(
101 rsi, MemOperand(wasm_trusted_instance,
102 wasm::ObjectAccess::ToTagged(
103 WasmTrustedInstanceData::kNativeContextOffset)));
104}
105
106void RestoreAfterBuiltinCall(MacroAssembler* masm, Register wasm_instance,
107 Register return_count, Register array_start) {
108 // Pop and load values from the stack in order into the registers after
109 // builtin calls for the GenericJSToWasmInterpreterWrapper.
110 __ popq(wasm_instance);
111 __ popq(return_count);
112 __ popq(array_start);
113}
114
115void PrepareForWasmToJsConversionBuiltinCall(
116 MacroAssembler* masm, Register return_count, Register result_index,
117 Register current_return_slot, Register valuetypes_array_ptr,
118 Register wasm_instance, Register fixed_array, Register jsarray) {
119 __ movq(
120 MemOperand(
121 rbp, BuiltinWasmInterpreterWrapperConstants::kGCScanSlotCountOffset),
122 Immediate(3));
123
124 // Pushes and puts the values in order onto the stack before builtin calls
125 // for the GenericJSToWasmInterpreterWrapper.
126 __ pushq(return_count);
127 __ pushq(result_index);
128 __ pushq(current_return_slot);
129 __ pushq(valuetypes_array_ptr);
130 // The following three slots contain tagged objects that need to be visited
131 // during GC.
132 __ pushq(wasm_instance);
133 __ pushq(fixed_array);
134 __ pushq(jsarray);
135 // Put the context into rsi.
136 Register wasm_trusted_instance = wasm_instance;
137 __ LoadTrustedPointerField(
138 wasm_trusted_instance,
139 FieldMemOperand(wasm_instance, WasmInstanceObject::kTrustedDataOffset),
140 kWasmTrustedInstanceDataIndirectPointerTag, kScratchRegister);
141 __ LoadTaggedField(
142 rsi, MemOperand(wasm_trusted_instance,
143 wasm::ObjectAccess::ToTagged(
144 WasmTrustedInstanceData::kNativeContextOffset)));
145}
146
147void RestoreAfterWasmToJsConversionBuiltinCall(
148 MacroAssembler* masm, Register jsarray, Register fixed_array,
149 Register wasm_instance, Register valuetypes_array_ptr,
150 Register current_return_slot, Register result_index,
151 Register return_count) {
152 // Pop and load values from the stack in order into the registers after
153 // builtin calls for the GenericJSToWasmInterpreterWrapper.
154 __ popq(jsarray);
155 __ popq(fixed_array);
156 __ popq(wasm_instance);
157 __ popq(valuetypes_array_ptr);
158 __ popq(current_return_slot);
159 __ popq(result_index);
160 __ popq(return_count);
161}
162
163} // namespace
164
165void Builtins::Generate_WasmInterpreterEntry(MacroAssembler* masm) {
166 Register wasm_instance = rsi;
167 Register function_index = r12;
168 Register array_start = r15;
169
170 // Set up the stackframe.
171 __ EnterFrame(StackFrame::WASM_INTERPRETER_ENTRY);
172
173 __ pushq(wasm_instance);
174 __ pushq(function_index);
175 __ pushq(array_start);
176 __ Move(wasm_instance, 0);
177 __ CallRuntime(Runtime::kWasmRunInterpreter, 3);
178
179 // Deconstruct the stack frame.
180 __ LeaveFrame(StackFrame::WASM_INTERPRETER_ENTRY);
181 __ ret(0);
182}
183
184void LoadFunctionDataAndWasmInstance(MacroAssembler* masm,
185 Register function_data,
186 Register wasm_instance) {
187 Register closure = function_data;
188 Register shared_function_info = closure;
189 __ LoadTaggedField(
190 shared_function_info,
191 MemOperand(
192 closure,
193 wasm::ObjectAccess::SharedFunctionInfoOffsetInTaggedJSFunction()));
194 closure = no_reg;
195 __ LoadTrustedPointerField(
196 function_data,
197 FieldOperand(shared_function_info,
198 SharedFunctionInfo::kTrustedFunctionDataOffset),
199 kUnknownIndirectPointerTag, kScratchRegister);
200 shared_function_info = no_reg;
201
202 Register trusted_instance_data = wasm_instance;
203#if V8_ENABLE_SANDBOX
204 __ DecompressProtected(
205 trusted_instance_data,
206 MemOperand(function_data,
207 WasmExportedFunctionData::kProtectedInstanceDataOffset -
208 kHeapObjectTag));
209#else
210 __ LoadTaggedField(
211 trusted_instance_data,
212 MemOperand(function_data,
213 WasmExportedFunctionData::kProtectedInstanceDataOffset -
214 kHeapObjectTag));
215#endif
216 __ LoadTaggedField(wasm_instance,
217 MemOperand(trusted_instance_data,
218 WasmTrustedInstanceData::kInstanceObjectOffset -
219 kHeapObjectTag));
220}
221
222void LoadFromSignature(MacroAssembler* masm, Register valuetypes_array_ptr,
223 Register return_count, Register param_count) {
224 Register signature = valuetypes_array_ptr;
225 __ movq(return_count,
226 MemOperand(signature, wasm::FunctionSig::kReturnCountOffset));
227 __ movq(param_count,
228 MemOperand(signature, wasm::FunctionSig::kParameterCountOffset));
229 valuetypes_array_ptr = signature;
230 __ movq(valuetypes_array_ptr,
231 MemOperand(signature, wasm::FunctionSig::kRepsOffset));
232}
233
234void LoadValueTypesArray(MacroAssembler* masm, Register function_data,
235 Register valuetypes_array_ptr, Register return_count,
236 Register param_count, Register signature_data) {
237 __ LoadTaggedField(
238 signature_data,
239 FieldOperand(function_data,
240 WasmExportedFunctionData::kPackedArgsSizeOffset));
241 __ SmiToInt32(signature_data);
242
243 Register signature = valuetypes_array_ptr;
244 __ movq(signature,
245 MemOperand(function_data,
246 WasmExportedFunctionData::kSigOffset - kHeapObjectTag));
247 LoadFromSignature(masm, valuetypes_array_ptr, return_count, param_count);
248}
249
250// TODO(paolosev@microsoft.com): this should be converted into a Torque builtin,
251// like it was done for GenericJSToWasmWrapper.
252void Builtins::Generate_GenericJSToWasmInterpreterWrapper(
253 MacroAssembler* masm) {
254 // Set up the stackframe.
255 __ EnterFrame(StackFrame::JS_TO_WASM);
256
257 // -------------------------------------------
258 // Compute offsets and prepare for GC.
259 // -------------------------------------------
260 // GenericJSToWasmInterpreterWrapperFrame:
261 // rbp-N Args/retvals array for Wasm call
262 // ... ...
263 // rbp-0x58 SignatureData (== rbp-N)
264 // rbp-0x50 CurrentIndex
265 // rbp-0x48 ArgRetsIsArgs
266 // rbp-0x40 ArgRetsAddress
267 // rbp-0x38 ValueTypesArray
268 // rbp-0x30 SigReps
269 // rbp-0x28 ReturnCount
270 // rbp-0x20 ParamCount
271 // rbp-0x18 InParamCount
272 // rbp-0x10 GCScanSlotCount
273 // rbp-0x08 Marker(StackFrame::JS_TO_WASM)
274 // rbp Old RBP
275 // rbp+0x08 return address
276 // rbp+0x10 receiver
277 // rpb+0x18 arg
278
279 constexpr int kMarkerOffset =
280 BuiltinWasmInterpreterWrapperConstants::kGCScanSlotCountOffset +
281 kSystemPointerSize;
282 // The number of parameters passed to this function.
283 constexpr int kInParamCountOffset =
284 BuiltinWasmInterpreterWrapperConstants::kGCScanSlotCountOffset -
285 kSystemPointerSize;
286 // The number of parameters according to the signature.
287 constexpr int kParamCountOffset =
288 BuiltinWasmInterpreterWrapperConstants::kParamCountOffset;
289 constexpr int kReturnCountOffset =
290 BuiltinWasmInterpreterWrapperConstants::kReturnCountOffset;
291 constexpr int kSigRepsOffset =
292 BuiltinWasmInterpreterWrapperConstants::kSigRepsOffset;
293 constexpr int kValueTypesArrayStartOffset =
294 BuiltinWasmInterpreterWrapperConstants::kValueTypesArrayStartOffset;
295 // Array for arguments and return values. They will be scanned by GC.
296 constexpr int kArgRetsAddressOffset =
297 BuiltinWasmInterpreterWrapperConstants::kArgRetsAddressOffset;
298 // Arg/Return arrays use the same stack address. So, we should keep a flag
299 // whether we are using the array for args or returns. (1 = Args, 0 = Rets)
300 constexpr int kArgRetsIsArgsOffset =
301 BuiltinWasmInterpreterWrapperConstants::kArgRetsIsArgsOffset;
302 // The index of the argument being converted.
303 constexpr int kCurrentIndexOffset =
304 BuiltinWasmInterpreterWrapperConstants::kCurrentIndexOffset;
305 // Precomputed signature data, a uint32_t with the format:
306 // bit 0-14: PackedArgsSize
307 // bit 15: HasRefArgs
308 // bit 16: HasRefRets
309 constexpr int kSignatureDataOffset =
310 BuiltinWasmInterpreterWrapperConstants::kSignatureDataOffset;
311 // We set and use this slot only when moving parameters into the parameter
312 // registers (so no GC scan is needed).
313 constexpr int kNumSpillSlots =
314 (kMarkerOffset - kSignatureDataOffset) / kSystemPointerSize;
315 __ subq(rsp, Immediate(kNumSpillSlots * kSystemPointerSize));
316 // Put the in_parameter count on the stack, we only need it at the very end
317 // when we pop the parameters off the stack.
318 Register in_param_count = rax;
319 __ decq(in_param_count);
320 __ movq(MemOperand(rbp, kInParamCountOffset), in_param_count);
321 in_param_count = no_reg;
322
323 // -------------------------------------------
324 // Load the Wasm exported function data and the Wasm instance.
325 // -------------------------------------------
326 Register function_data = rdi;
327 Register wasm_instance = kWasmImplicitArgRegister; // rsi
328 LoadFunctionDataAndWasmInstance(masm, function_data, wasm_instance);
329
330 // -------------------------------------------
331 // Load values from the signature.
332 // -------------------------------------------
333 Register valuetypes_array_ptr = r11;
334 Register return_count = r8;
335 Register param_count = rcx;
336 Register signature_data = r15;
337 LoadValueTypesArray(masm, function_data, valuetypes_array_ptr, return_count,
338 param_count, signature_data);
339 __ movq(MemOperand(rbp, kSignatureDataOffset), signature_data);
340 Register array_size = signature_data;
341 signature_data = no_reg;
342 __ andq(array_size,
343 Immediate(wasm::WasmInterpreterRuntime::PackedArgsSizeField::kMask));
344
345 // -------------------------------------------
346 // Store signature-related values to the stack.
347 // -------------------------------------------
348 // We store values on the stack to restore them after function calls.
349 // We cannot push values onto the stack right before the wasm call. The wasm
350 // function expects the parameters, that didn't fit into the registers, on the
351 // top of the stack.
352 __ movq(MemOperand(rbp, kParamCountOffset), param_count);
353 __ movq(MemOperand(rbp, kReturnCountOffset), return_count);
354 __ movq(MemOperand(rbp, kSigRepsOffset), valuetypes_array_ptr);
355 __ movq(MemOperand(rbp, kValueTypesArrayStartOffset), valuetypes_array_ptr);
356
357 // -------------------------------------------
358 // Allocate array for args and return value.
359 // -------------------------------------------
360 Register array_start = array_size;
361 array_size = no_reg;
362 __ negq(array_start);
363 __ addq(array_start, rsp);
364 __ movq(rsp, array_start);
365 __ movq(MemOperand(rbp, kArgRetsAddressOffset), array_start);
366
367 __ movq(MemOperand(rbp, kArgRetsIsArgsOffset), Immediate(1));
368 __ Move(MemOperand(rbp, kCurrentIndexOffset), 0);
369
370 Label prepare_for_wasm_call;
371 __ Cmp(param_count, 0);
372
373 // If we have 0 params: jump through parameter handling.
374 __ j(equal, &prepare_for_wasm_call);
375
376 // Create a section on the stack to pass the evaluated parameters to the
377 // interpreter and to receive the results. This section represents the array
378 // expected as argument by the Runtime_WasmRunInterpreter.
379 // Arguments are stored one after the other without holes, starting at the
380 // beginning of the array, and the interpreter puts the returned values in the
381 // same array, also starting at the beginning.
382
383 // Set the current_param_slot to point to the start of the section.
384 Register current_param_slot = r10;
385 __ movq(current_param_slot, array_start);
386
387 // Loop through the params starting with the first.
388 // [rbp + 8 * current_index + kArgsOffset] gives us the JS argument we are
389 // processing. We iterate through half-open interval [1st param, rbp + 8 *
390 // param_count + kArgsOffset).
391
392 constexpr int kReceiverOnStackSize = kSystemPointerSize;
393 constexpr int kArgsOffset =
394 kFPOnStackSize + kPCOnStackSize + kReceiverOnStackSize;
395
396 Register param = rax;
397 // We have to check the types of the params. The ValueType array contains
398 // first the return then the param types.
399 constexpr int kValueTypeSize = sizeof(wasm::ValueType);
400 static_assert(kValueTypeSize == 4);
401 const int32_t kValueTypeSizeLog2 = log2(kValueTypeSize);
402 // Set the ValueType array pointer to point to the first parameter.
403 Register returns_size = return_count;
404 return_count = no_reg;
405 __ shlq(returns_size, Immediate(kValueTypeSizeLog2));
406 __ addq(valuetypes_array_ptr, returns_size);
407 returns_size = no_reg;
408
409 Register current_index = rbx;
410 __ Move(current_index, 0);
411
412 // -------------------------------------------
413 // Param evaluation loop.
414 // -------------------------------------------
415 Label loop_through_params;
416 __ bind(&loop_through_params);
417
418 __ movq(MemOperand(
419 rbp, BuiltinWasmInterpreterWrapperConstants::kCurrentIndexOffset),
420 current_index);
421 __ movq(param, MemOperand(rbp, current_index, times_system_pointer_size,
422 kArgsOffset));
423
424 Register valuetype = r12;
425 __ movl(valuetype, Operand(valuetypes_array_ptr, 0));
426
427 // -------------------------------------------
428 // Param conversion.
429 // -------------------------------------------
430 // If param is a Smi we can easily convert it. Otherwise we'll call a builtin
431 // for conversion.
432 Label param_conversion_done;
433 Label check_ref_param;
434 Label convert_param;
435 __ cmpq(valuetype, Immediate(wasm::kWasmI32.raw_bit_field()));
436 __ j(not_equal, &check_ref_param);
437 __ JumpIfNotSmi(param, &convert_param);
438
439 // Change the param from Smi to int32.
440 __ SmiUntag(param);
441 // Zero extend.
442 __ movl(param, param);
443 // Place the param into the proper slot.
444 __ movl(MemOperand(current_param_slot, 0), param);
445 __ addq(current_param_slot, Immediate(sizeof(int32_t)));
446 __ jmp(&param_conversion_done);
447
448 Label handle_ref_param;
449 __ bind(&check_ref_param);
450 __ testl(valuetype, Immediate(1));
451 __ j(equal, &convert_param);
452
453 // Place the reference param into the proper slot.
454 __ bind(&handle_ref_param);
455 // Make sure slot for ref args are 64-bit aligned.
456 __ movq(r9, current_param_slot);
457 __ andq(r9, Immediate(0x04));
458 __ addq(current_param_slot, r9);
459 __ movq(MemOperand(current_param_slot, 0), param);
460 __ addq(current_param_slot, Immediate(kSystemPointerSize));
461
462 // -------------------------------------------
463 // Param conversion done.
464 // -------------------------------------------
465 __ bind(&param_conversion_done);
466
467 __ addq(valuetypes_array_ptr, Immediate(kValueTypeSize));
468
469 __ movq(
470 current_index,
471 MemOperand(rbp,
472 BuiltinWasmInterpreterWrapperConstants::kCurrentIndexOffset));
473 __ incq(current_index);
474 __ cmpq(current_index, param_count);
475 __ j(less, &loop_through_params);
476 __ movq(MemOperand(
477 rbp, BuiltinWasmInterpreterWrapperConstants::kCurrentIndexOffset),
478 current_index);
479
480 // ----------- S t a t e -------------
481 // -- r10 : current_param_slot
482 // -- r11 : valuetypes_array_ptr
483 // -- r15 : array_start
484 // -- rdi : function_data
485 // -- rsi : wasm_trusted_instance
486 // -- GpParamRegisters = rax, rdx, rcx, rbx, r9
487 // -----------------------------------
488
489 __ bind(&prepare_for_wasm_call);
490 // -------------------------------------------
491 // Prepare for the Wasm call.
492 // -------------------------------------------
493 // Set thread_in_wasm_flag.
494 Register thread_in_wasm_flag_addr = r12;
495 __ movq(
496 thread_in_wasm_flag_addr,
497 MemOperand(kRootRegister, Isolate::thread_in_wasm_flag_address_offset()));
498 __ movl(MemOperand(thread_in_wasm_flag_addr, 0), Immediate(1));
499 thread_in_wasm_flag_addr = no_reg;
500
501 Register function_index = r12;
502 __ movl(
503 function_index,
504 MemOperand(function_data, WasmExportedFunctionData::kFunctionIndexOffset -
505 kHeapObjectTag));
506 // We pass function_index as Smi.
507
508 // One tagged object (the wasm_instance) to be visited if there is a GC
509 // during the call.
510 constexpr int kWasmCallGCScanSlotCount = 1;
511 __ Move(
512 MemOperand(
513 rbp, BuiltinWasmInterpreterWrapperConstants::kGCScanSlotCountOffset),
514 kWasmCallGCScanSlotCount);
515
516 // -------------------------------------------
517 // Call the Wasm function.
518 // -------------------------------------------
519 __ pushq(wasm_instance);
520 __ Call(BUILTIN_CODE(masm->isolate(), WasmInterpreterEntry),
521 RelocInfo::CODE_TARGET);
522 __ popq(wasm_instance);
523 __ movq(array_start, MemOperand(rbp, kArgRetsAddressOffset));
524 __ Move(MemOperand(rbp, kArgRetsIsArgsOffset), 0);
525
526 function_index = no_reg;
527
528 // Unset thread_in_wasm_flag.
529 thread_in_wasm_flag_addr = r8;
530 __ movq(
531 thread_in_wasm_flag_addr,
532 MemOperand(kRootRegister, Isolate::thread_in_wasm_flag_address_offset()));
533 __ movl(MemOperand(thread_in_wasm_flag_addr, 0), Immediate(0));
534 thread_in_wasm_flag_addr = no_reg;
535
536 // -------------------------------------------
537 // Return handling.
538 //
539 // ----------- S t a t e -------------
540 // -- r15 : array_start
541 // -- rsi : wasm_instance
542 // -------------------------------------------
543 return_count = r8;
544 __ movq(return_count, MemOperand(rbp, kReturnCountOffset));
545
546 // All return values are already in the packed array.
547 __ movq(MemOperand(
548 rbp, BuiltinWasmInterpreterWrapperConstants::kCurrentIndexOffset),
549 return_count);
550
551 Register return_value = rax;
552 Register fixed_array = rdi;
553 __ Move(fixed_array, 0);
554 Register jsarray = rdx;
555 __ Move(jsarray, 0);
556
557 Label return_undefined;
558 __ cmpl(return_count, Immediate(1));
559 // If no return value, load undefined.
560 __ j(less, &return_undefined);
561
562 Label start_return_conversion;
563 // If we have more than one return value, we need to return a JSArray.
564 __ j(equal, &start_return_conversion);
565
566 PrepareForBuiltinCall(masm, array_start, return_count, wasm_instance);
567 __ movq(rax, return_count);
568 __ SmiTag(rax);
569 // Create JSArray to hold results. Possible GC here.
570 __ Call(BUILTIN_CODE(masm->isolate(), WasmAllocateJSArray),
571 RelocInfo::CODE_TARGET);
572 __ movq(jsarray, rax);
573 RestoreAfterBuiltinCall(masm, wasm_instance, return_count, array_start);
574 __ LoadTaggedField(fixed_array, MemOperand(jsarray, JSArray::kElementsOffset -
575 kHeapObjectTag));
576
577 __ bind(&start_return_conversion);
578 Register current_return_slot = array_start;
579
580 Register result_index = r9;
581 __ xorq(result_index, result_index);
582
583 Label convert_return_value;
584 __ jmp(&convert_return_value);
585
586 __ bind(&return_undefined);
587 __ LoadRoot(return_value, RootIndex::kUndefinedValue);
588 Label all_results_conversion_done;
589 __ jmp(&all_results_conversion_done);
590
591 Label next_return_value;
592 __ bind(&next_return_value);
593 __ incq(result_index);
594 __ cmpq(result_index, return_count);
595 __ j(less, &convert_return_value);
596
597 __ bind(&all_results_conversion_done);
598 __ movq(param_count, MemOperand(rbp, kParamCountOffset));
599
600 Label do_return;
601 __ cmpq(fixed_array, Immediate(0));
602 __ j(equal, &do_return);
603 // The result is jsarray.
604 __ movq(rax, jsarray);
605
606 // Calculate the number of parameters we have to pop off the stack. This
607 // number is max(in_param_count, param_count).
608 __ bind(&do_return);
609 in_param_count = rdx;
610 __ movq(in_param_count, MemOperand(rbp, kInParamCountOffset));
611 __ cmpq(param_count, in_param_count);
612 __ cmovq(less, param_count, in_param_count);
613
614 // -------------------------------------------
615 // Deconstruct the stack frame.
616 // -------------------------------------------
617 __ LeaveFrame(StackFrame::JS_TO_WASM);
618
619 // We have to remove the caller frame slots:
620 // - JS arguments
621 // - the receiver
622 // and transfer the control to the return address (the return address is
623 // expected to be on the top of the stack).
624 // We cannot use just the ret instruction for this, because we cannot pass the
625 // number of slots to remove in a Register as an argument.
626 __ DropArguments(param_count, rbx);
627 __ ret(0);
628
629 // --------------------------------------------------------------------------
630 // Deferred code.
631 // --------------------------------------------------------------------------
632
633 // -------------------------------------------
634 // Param conversion builtins.
635 // -------------------------------------------
636 __ bind(&convert_param);
637 // The order of pushes is important. We want the heap objects, that should be
638 // scanned by GC, to be on the top of the stack.
639 // We have to set the indicating value for the GC to the number of values on
640 // the top of the stack that have to be scanned before calling the builtin
641 // function.
642 // We don't need the JS context for these builtin calls.
643 // The builtin expects the parameter to be in register param = rax.
644
645 PrepareForJsToWasmConversionBuiltinCall(
646 masm, array_start, param_count, current_param_slot, valuetypes_array_ptr,
647 wasm_instance, function_data);
648
649 Label param_kWasmI32_not_smi;
650 Label param_kWasmI64;
651 Label param_kWasmF32;
652 Label param_kWasmF64;
653 Label throw_type_error;
654
655 __ cmpq(valuetype, Immediate(wasm::kWasmI32.raw_bit_field()));
656 __ j(equal, &param_kWasmI32_not_smi);
657 __ cmpq(valuetype, Immediate(wasm::kWasmI64.raw_bit_field()));
658 __ j(equal, &param_kWasmI64);
659 __ cmpq(valuetype, Immediate(wasm::kWasmF32.raw_bit_field()));
660 __ j(equal, &param_kWasmF32);
661 __ cmpq(valuetype, Immediate(wasm::kWasmF64.raw_bit_field()));
662 __ j(equal, &param_kWasmF64);
663
664 __ cmpq(valuetype, Immediate(wasm::kWasmS128.raw_bit_field()));
665 // Simd arguments cannot be passed from JavaScript.
666 __ j(equal, &throw_type_error);
667
668 __ int3();
669
670 __ bind(&param_kWasmI32_not_smi);
671 __ Call(BUILTIN_CODE(masm->isolate(), WasmTaggedNonSmiToInt32),
672 RelocInfo::CODE_TARGET);
673 // Param is the result of the builtin.
674 __ AssertZeroExtended(param);
675 RestoreAfterJsToWasmConversionBuiltinCall(
676 masm, function_data, wasm_instance, valuetypes_array_ptr,
677 current_param_slot, param_count, array_start);
678 __ movl(MemOperand(current_param_slot, 0), param);
679 __ addq(current_param_slot, Immediate(sizeof(int32_t)));
680 __ jmp(&param_conversion_done);
681
682 __ bind(&param_kWasmI64);
683 __ Call(BUILTIN_CODE(masm->isolate(), BigIntToI64), RelocInfo::CODE_TARGET);
684 RestoreAfterJsToWasmConversionBuiltinCall(
685 masm, function_data, wasm_instance, valuetypes_array_ptr,
686 current_param_slot, param_count, array_start);
687 __ movq(MemOperand(current_param_slot, 0), param);
688 __ addq(current_param_slot, Immediate(sizeof(int64_t)));
689 __ jmp(&param_conversion_done);
690
691 __ bind(&param_kWasmF32);
692 __ Call(BUILTIN_CODE(masm->isolate(), WasmTaggedToFloat32),
693 RelocInfo::CODE_TARGET);
694 RestoreAfterJsToWasmConversionBuiltinCall(
695 masm, function_data, wasm_instance, valuetypes_array_ptr,
696 current_param_slot, param_count, array_start);
697 __ Movss(MemOperand(current_param_slot, 0), xmm0);
698 __ addq(current_param_slot, Immediate(sizeof(float)));
699 __ jmp(&param_conversion_done);
700
701 __ bind(&param_kWasmF64);
702 __ Call(BUILTIN_CODE(masm->isolate(), WasmTaggedToFloat64),
703 RelocInfo::CODE_TARGET);
704 RestoreAfterJsToWasmConversionBuiltinCall(
705 masm, function_data, wasm_instance, valuetypes_array_ptr,
706 current_param_slot, param_count, array_start);
707 __ Movsd(MemOperand(current_param_slot, 0), xmm0);
708 __ addq(current_param_slot, Immediate(sizeof(double)));
709 __ jmp(&param_conversion_done);
710
711 __ bind(&throw_type_error);
712 // CallRuntime expects kRootRegister (r13) to contain the root.
713 __ CallRuntime(Runtime::kWasmThrowJSTypeError);
714 __ int3(); // Should not return.
715
716 // -------------------------------------------
717 // Return conversions.
718 // -------------------------------------------
719 __ bind(&convert_return_value);
720 // We have to make sure that the kGCScanSlotCount is set correctly when we
721 // call the builtins for conversion. For these builtins it's the same as for
722 // the Wasm call, that is, kGCScanSlotCount = 0, so we don't have to reset it.
723 // We don't need the JS context for these builtin calls.
724
725 __ movq(valuetypes_array_ptr, MemOperand(rbp, kValueTypesArrayStartOffset));
726 // The first valuetype of the array is the return's valuetype.
727 __ movl(valuetype, Operand(valuetypes_array_ptr, 0));
728
729 Label return_kWasmI32;
730 Label return_kWasmI64;
731 Label return_kWasmF32;
732 Label return_kWasmF64;
733 Label return_kWasmRef;
734
735 __ cmpq(valuetype, Immediate(wasm::kWasmI32.raw_bit_field()));
736 __ j(equal, &return_kWasmI32);
737
738 __ cmpq(valuetype, Immediate(wasm::kWasmI64.raw_bit_field()));
739 __ j(equal, &return_kWasmI64);
740
741 __ cmpq(valuetype, Immediate(wasm::kWasmF32.raw_bit_field()));
742 __ j(equal, &return_kWasmF32);
743
744 __ cmpq(valuetype, Immediate(wasm::kWasmF64.raw_bit_field()));
745 __ j(equal, &return_kWasmF64);
746
747 __ testl(valuetype, Immediate(1));
748 __ j(not_equal, &return_kWasmRef);
749
750 // Invalid type. Wasm cannot return Simd results to JavaScript.
751 __ int3();
752
753 Label return_value_done;
754
755 __ bind(&return_kWasmI32);
756 __ movl(return_value, MemOperand(current_return_slot, 0));
757 __ addq(current_return_slot, Immediate(sizeof(int32_t)));
758 Label to_heapnumber;
759 // If pointer compression is disabled, we can convert the return to a smi.
760 if (SmiValuesAre32Bits()) {
761 __ SmiTag(return_value);
762 } else {
763 Register temp = rbx;
764 __ movq(temp, return_value);
765 // Double the return value to test if it can be a Smi.
766 __ addl(temp, return_value);
767 temp = no_reg;
768 // If there was overflow, convert to a HeapNumber.
769 __ j(overflow, &to_heapnumber);
770 // If there was no overflow, we can convert to Smi.
771 __ SmiTag(return_value);
772 }
773 __ jmp(&return_value_done);
774
775 // Handle the conversion of the I32 return value to HeapNumber when it cannot
776 // be a smi.
777 __ bind(&to_heapnumber);
778
779 PrepareForWasmToJsConversionBuiltinCall(
780 masm, return_count, result_index, current_return_slot,
781 valuetypes_array_ptr, wasm_instance, fixed_array, jsarray);
782 __ Call(BUILTIN_CODE(masm->isolate(), WasmInt32ToHeapNumber),
783 RelocInfo::CODE_TARGET);
784 RestoreAfterWasmToJsConversionBuiltinCall(
785 masm, jsarray, fixed_array, wasm_instance, valuetypes_array_ptr,
786 current_return_slot, result_index, return_count);
787 __ jmp(&return_value_done);
788
789 __ bind(&return_kWasmI64);
790 __ movq(return_value, MemOperand(current_return_slot, 0));
791 __ addq(current_return_slot, Immediate(sizeof(int64_t)));
792 PrepareForWasmToJsConversionBuiltinCall(
793 masm, return_count, result_index, current_return_slot,
794 valuetypes_array_ptr, wasm_instance, fixed_array, jsarray);
795 __ Call(BUILTIN_CODE(masm->isolate(), I64ToBigInt), RelocInfo::CODE_TARGET);
796 RestoreAfterWasmToJsConversionBuiltinCall(
797 masm, jsarray, fixed_array, wasm_instance, valuetypes_array_ptr,
798 current_return_slot, result_index, return_count);
799 __ jmp(&return_value_done);
800
801 __ bind(&return_kWasmF32);
802 __ movq(xmm1, MemOperand(current_return_slot, 0));
803 __ addq(current_return_slot, Immediate(sizeof(float)));
804 // The builtin expects the value to be in xmm0.
805 __ Movss(xmm0, xmm1);
806 PrepareForWasmToJsConversionBuiltinCall(
807 masm, return_count, result_index, current_return_slot,
808 valuetypes_array_ptr, wasm_instance, fixed_array, jsarray);
809 __ Call(BUILTIN_CODE(masm->isolate(), WasmFloat32ToNumber),
810 RelocInfo::CODE_TARGET);
811 RestoreAfterWasmToJsConversionBuiltinCall(
812 masm, jsarray, fixed_array, wasm_instance, valuetypes_array_ptr,
813 current_return_slot, result_index, return_count);
814 __ jmp(&return_value_done);
815
816 __ bind(&return_kWasmF64);
817 // The builtin expects the value to be in xmm0.
818 __ movq(xmm0, MemOperand(current_return_slot, 0));
819 __ addq(current_return_slot, Immediate(sizeof(double)));
820 PrepareForWasmToJsConversionBuiltinCall(
821 masm, return_count, result_index, current_return_slot,
822 valuetypes_array_ptr, wasm_instance, fixed_array, jsarray);
823 __ Call(BUILTIN_CODE(masm->isolate(), WasmFloat64ToNumber),
824 RelocInfo::CODE_TARGET);
825 RestoreAfterWasmToJsConversionBuiltinCall(
826 masm, jsarray, fixed_array, wasm_instance, valuetypes_array_ptr,
827 current_return_slot, result_index, return_count);
828 __ jmp(&return_value_done);
829
830 __ bind(&return_kWasmRef);
831 // Make sure slot for ref args are 64-bit aligned.
832 __ movq(rbx, current_return_slot);
833 __ andq(rbx, Immediate(0x04));
834 __ addq(current_return_slot, rbx);
835 __ movq(return_value, MemOperand(current_return_slot, 0));
836 __ addq(current_return_slot, Immediate(kSystemPointerSize));
837 // Do not modify the result in return_value.
838
839 __ bind(&return_value_done);
840 __ addq(valuetypes_array_ptr, Immediate(kValueTypeSize));
841 __ movq(MemOperand(rbp, kValueTypesArrayStartOffset), valuetypes_array_ptr);
842 __ cmpq(fixed_array, Immediate(0));
843 __ j(equal, &next_return_value);
844
845 // Store result into JSArray.
846 __ StoreTaggedField(FieldOperand(fixed_array, result_index,
847 static_cast<ScaleFactor>(kTaggedSizeLog2),
848 OFFSET_OF_DATA_START(FixedArray)),
849 return_value);
850 PrepareForWasmToJsConversionBuiltinCall(
851 masm, return_count, result_index, current_return_slot,
852 valuetypes_array_ptr, wasm_instance, fixed_array, jsarray);
853 Register slot_address = WriteBarrierDescriptor::SlotAddressRegister();
854 __ leaq(slot_address, FieldOperand(fixed_array, result_index,
855 static_cast<ScaleFactor>(kTaggedSizeLog2),
856 OFFSET_OF_DATA_START(FixedArray)));
857 __ RecordWrite(fixed_array, slot_address, return_value,
858 SaveFPRegsMode::kIgnore);
859 RestoreAfterWasmToJsConversionBuiltinCall(
860 masm, jsarray, fixed_array, wasm_instance, valuetypes_array_ptr,
861 current_return_slot, result_index, return_count);
862
863 __ jmp(&next_return_value);
864}
865
866// For compiled code, V8 generates signature-specific CWasmEntries that manage
867// the transition from C++ code to a JS function called from Wasm code and takes
868// care of handling exceptions that arise from JS (see
869// WasmWrapperGraphBuilder::BuildCWasmEntry()).
870// This builtin does the same for the Wasm interpreter and it is used for
871// Wasm-to-JS calls. It invokes GenericWasmToJSInterpreterWrapper and installs a
872// specific frame of type C_WASM_ENTRY which is used in
873// Isolate::UnwindAndFindHandler() to correctly unwind interpreter stack frames
874// and handle exceptions.
875void Builtins::Generate_WasmInterpreterCWasmEntry(MacroAssembler* masm) {
876 Label invoke, handler_entry, exit;
877 Register isolate_root = kCArgRegs[2]; // Windows: r8, Posix: rdx
878
879 // -------------------------------------------
880 // CWasmEntryFrame: (Win64)
881 // rbp-0xe8 rbx
882 // rbp-0xe0 rsi
883 // rbp-0xd8 rdi
884 // rbp-0xd0 r12
885 // rbp-0xc8 r13
886 // rbp-0xc0 r14
887 // rbp-0xb8 r15
888 // -------------------------------------------
889 // rbp-0xb0 xmm6
890 // ...
891 // rbp-0x20 xmm15
892 // -------------------------------------------
893 // rbp-0x18 rsp
894 // rbp-0x10 CEntryFp
895 // rbp-0x08 Marker(StackFrame::C_WASM_ENTRY)
896 // rbp Old RBP
897
898 // -------------------------------------------
899 // CWasmEntryFrame: (AMD64 ABI)
900 // rbp-0x40 rbx
901 // rbp-0x38 r12
902 // rbp-0x30 r13
903 // rbp-0x28 r14
904 // rbp-0x20 r15
905 // -------------------------------------------
906 // rbp-0x18 rsp
907 // rbp-0x10 CEntryFp
908 // rbp-0x08 Marker(StackFrame::C_WASM_ENTRY)
909 // rbp Old RBP
910
911#ifndef V8_OS_POSIX
912 // Offsets for arguments passed in WasmToJSCallSig. See declaration of
913 // {WasmToJSCallSig} in src/wasm/interpreter/wasm-interpreter-runtime.h.
914 constexpr int kCEntryFpParameterOffset = 0x30;
915 constexpr int kCallableOffset = 0x38;
916#endif // !V8_OS_POSIX
917
918 // Set up the stackframe.
919 __ EnterFrame(StackFrame::C_WASM_ENTRY);
920
921 // Space to store c_entry_fp and current rsp (used by exception handler).
922 __ subq(rsp, Immediate(0x10));
923
924 // Save registers
925#ifdef V8_TARGET_OS_WIN
926 // On Win64 XMM6-XMM15 are callee-save.
927 __ subq(rsp, Immediate(0xa0));
928 __ movdqu(Operand(rsp, 0x00), xmm6);
929 __ movdqu(Operand(rsp, 0x10), xmm7);
930 __ movdqu(Operand(rsp, 0x20), xmm8);
931 __ movdqu(Operand(rsp, 0x30), xmm9);
932 __ movdqu(Operand(rsp, 0x40), xmm10);
933 __ movdqu(Operand(rsp, 0x50), xmm11);
934 __ movdqu(Operand(rsp, 0x60), xmm12);
935 __ movdqu(Operand(rsp, 0x70), xmm13);
936 __ movdqu(Operand(rsp, 0x80), xmm14);
937 __ movdqu(Operand(rsp, 0x90), xmm15);
938#endif // V8_TARGET_OS_WIN
939 __ pushq(r15);
940 __ pushq(r14);
941 __ pushq(r13);
942 __ pushq(r12);
943#ifdef V8_TARGET_OS_WIN
944 __ pushq(rdi); // Only callee save in Win64 ABI, argument in AMD64 ABI.
945 __ pushq(rsi); // Only callee save in Win64 ABI, argument in AMD64 ABI.
946#endif // V8_TARGET_OS_WIN
947 __ pushq(rbx);
948
949 // InitializeRootRegister
950 __ movq(kRootRegister, isolate_root); // kRootRegister: r13
951#ifdef V8_COMPRESS_POINTERS_IN_SHARED_CAGE
952 __ LoadRootRelative(kPtrComprCageBaseRegister,
953 IsolateData::cage_base_offset());
954#endif
955 isolate_root = no_reg;
956
957 Register callable = r8;
958#ifdef V8_OS_POSIX
959 __ movq(MemOperand(rbp, WasmInterpreterCWasmEntryConstants::kCEntryFPOffset),
960 r8); // saved_c_entry_fp
961 __ movq(callable, r9); // callable
962#else // Windows
963 // Store c_entry_fp into slot
964 __ movq(rbx, MemOperand(rbp, kCEntryFpParameterOffset));
965 __ movq(MemOperand(rbp, WasmInterpreterCWasmEntryConstants::kCEntryFPOffset),
966 rbx);
967 __ movq(callable, MemOperand(rbp, kCallableOffset));
968#endif // V8_OS_POSIX
969
970 // Jump to a faked try block that does the invoke, with a faked catch
971 // block that sets the pending exception.
972 __ jmp(&invoke);
973
974 // Handler.
975 __ bind(&handler_entry);
976
977 // Store the current pc as the handler offset. It's used later to create the
978 // handler table.
979 masm->isolate()->builtins()->SetCWasmInterpreterEntryHandlerOffset(
980 handler_entry.pos());
981 // Caught exception.
982 __ jmp(&exit);
983
984 // Invoke: Link this frame into the handler chain.
985 __ bind(&invoke);
986 __ movq(MemOperand(rbp, WasmInterpreterCWasmEntryConstants::kSPFPOffset),
987 rsp);
988 __ PushStackHandler();
989 __ Call(BUILTIN_CODE(masm->isolate(), GenericWasmToJSInterpreterWrapper),
990 RelocInfo::CODE_TARGET);
991
992 // Unlink this frame from the handler chain.
993 __ PopStackHandler();
994
995 __ bind(&exit);
996 // Restore registers.
997 __ popq(rbx);
998#ifdef V8_TARGET_OS_WIN
999 __ popq(rsi);
1000 __ popq(rdi);
1001#endif // V8_TARGET_OS_WIN
1002 __ popq(r12);
1003 __ popq(r13);
1004 __ popq(r14);
1005 __ popq(r15);
1006#ifdef V8_TARGET_OS_WIN
1007 // On Win64 XMM6-XMM15 are callee-save.
1008 __ movdqu(xmm15, Operand(rsp, 0x90));
1009 __ movdqu(xmm14, Operand(rsp, 0x80));
1010 __ movdqu(xmm13, Operand(rsp, 0x70));
1011 __ movdqu(xmm12, Operand(rsp, 0x60));
1012 __ movdqu(xmm11, Operand(rsp, 0x50));
1013 __ movdqu(xmm10, Operand(rsp, 0x40));
1014 __ movdqu(xmm9, Operand(rsp, 0x30));
1015 __ movdqu(xmm8, Operand(rsp, 0x20));
1016 __ movdqu(xmm7, Operand(rsp, 0x10));
1017 __ movdqu(xmm6, Operand(rsp, 0x00));
1018#endif // V8_TARGET_OS_WIN
1019
1020 // Deconstruct the stack frame.
1021 __ LeaveFrame(StackFrame::C_WASM_ENTRY);
1022 __ ret(0);
1023}
1024
1025void Builtins::Generate_GenericWasmToJSInterpreterWrapper(
1026 MacroAssembler* masm) {
1027 Register target_js_function = kCArgRegs[0]; // Win: rcx, Posix: rdi
1028 Register packed_args = kCArgRegs[1]; // Win: rdx, Posix: rsi
1029 Register callable = rdi;
1030 Register signature = kCArgRegs[3]; // Win: r9, Posix: rcx
1031
1032 // Set up the stackframe.
1033 __ EnterFrame(StackFrame::WASM_TO_JS);
1034
1035 // -------------------------------------------
1036 // Compute offsets and prepare for GC.
1037 // -------------------------------------------
1038 // GenericWasmToJSInterpreterWrapperFrame:
1039 // rbp-N receiver ^
1040 // ... JS arg 0 | Tagged
1041 // ... ... | objects
1042 // rbp-0x68 JS arg n-1 |
1043 // rbp-0x60 context v
1044 // -------------------------------------------
1045 // rbp-0x58 current_param_slot_index
1046 // rbp-0x50 valuetypes_array_ptr
1047 // rbp-0x48 param_index/return_index
1048 // rbp-0x40 signature
1049 // rbp-0x38 param_count
1050 // rbp-0x30 return_count
1051 // rbp-0x28 expected_arity
1052 // rbp-0x20 packed_array
1053 // rbp-0x18 GC_SP
1054 // rbp-0x10 GCScanSlotLimit
1055 // rbp-0x08 Marker(StackFrame::WASM_TO_JS)
1056 // rbp Old RBP
1057
1058 constexpr int kMarkerOffset =
1059 WasmToJSInterpreterFrameConstants::kGCScanSlotLimitOffset +
1060 kSystemPointerSize;
1061 static_assert(WasmToJSInterpreterFrameConstants::kGCSPOffset ==
1062 WasmToJSInterpreterFrameConstants::kGCScanSlotLimitOffset -
1063 kSystemPointerSize);
1064 constexpr int kPackedArrayOffset =
1065 WasmToJSInterpreterFrameConstants::kGCSPOffset - kSystemPointerSize;
1066 constexpr int kExpectedArityOffset = kPackedArrayOffset - kSystemPointerSize;
1067 constexpr int kReturnCountOffset = kExpectedArityOffset - kSystemPointerSize;
1068 constexpr int kParamCountOffset = kReturnCountOffset - kSystemPointerSize;
1069 constexpr int kSignatureOffset = kParamCountOffset - kSystemPointerSize;
1070 constexpr int kParamIndexOffset = kSignatureOffset - kSystemPointerSize;
1071 // Reuse this slot when iterating over return values.
1072 constexpr int kResultIndexOffset = kParamIndexOffset;
1073 constexpr int kValueTypesArrayStartOffset =
1074 kParamIndexOffset - kSystemPointerSize;
1075 constexpr int kCurrentParamOffset =
1076 kValueTypesArrayStartOffset - kSystemPointerSize;
1077 // Reuse this slot when iterating over return values.
1078 constexpr int kCurrentResultAddressOffset = kCurrentParamOffset;
1079 constexpr int kNumSpillSlots =
1080 (kMarkerOffset - kCurrentResultAddressOffset) / kSystemPointerSize;
1081 __ subq(rsp, Immediate(kNumSpillSlots * kSystemPointerSize));
1082
1083 __ movq(MemOperand(rbp, kPackedArrayOffset), packed_args);
1084 __ Move(MemOperand(rbp, WasmToJSInterpreterFrameConstants::kGCSPOffset), 0);
1085
1086 // Points to the end of the stack frame area that contains tagged objects that
1087 // need to be visited during GC.
1088 __ movq(MemOperand(rbp,
1089 WasmToJSInterpreterFrameConstants::kGCScanSlotLimitOffset),
1090 rsp);
1091
1092#if V8_OS_POSIX
1093 // Windows has a different calling convention.
1094 signature = r9;
1095 __ movq(signature, kCArgRegs[3]);
1096 target_js_function = rcx;
1097 __ movq(target_js_function, kCArgRegs[0]);
1098 packed_args = rdx;
1099 __ movq(packed_args, kCArgRegs[1]);
1100#endif // V8_OS_POSIX
1101 __ movq(callable, r8); // Callable passed in r8.
1102
1103 Register shared_function_info = r15;
1104 __ LoadTaggedField(
1105 shared_function_info,
1106 MemOperand(
1107 target_js_function,
1108 wasm::ObjectAccess::SharedFunctionInfoOffsetInTaggedJSFunction()));
1109
1110 // Set the context of the function; the call has to run in the function
1111 // context.
1112 Register context = rsi;
1113 __ LoadTaggedField(
1114 context, FieldOperand(target_js_function, JSFunction::kContextOffset));
1115 target_js_function = no_reg;
1116
1117 // Store context to be retrieved after the call.
1118 __ pushq(context);
1119
1120 // Load global receiver if sloppy else use undefined.
1121 Label receiver_undefined;
1122 Label calculate_js_function_arity;
1123 Register receiver = r11;
1124 Register flags = rbx;
1125 __ movl(flags,
1126 FieldOperand(shared_function_info, SharedFunctionInfo::kFlagsOffset));
1127 __ testq(flags, Immediate(SharedFunctionInfo::IsNativeBit::kMask |
1128 SharedFunctionInfo::IsStrictBit::kMask));
1129 flags = no_reg;
1130 __ j(not_equal, &receiver_undefined);
1131 __ LoadGlobalProxy(receiver);
1132 __ jmp(&calculate_js_function_arity);
1133
1134 __ bind(&receiver_undefined);
1135 __ LoadRoot(receiver, RootIndex::kUndefinedValue);
1136
1137 __ bind(&calculate_js_function_arity);
1138
1139 // Load values from the signature.
1140 __ movq(MemOperand(rbp, kSignatureOffset), signature);
1141 Register valuetypes_array_ptr = signature;
1142 Register return_count = r8;
1143 Register param_count = rcx;
1144 LoadFromSignature(masm, valuetypes_array_ptr, return_count, param_count);
1145 __ movq(MemOperand(rbp, kParamCountOffset), param_count);
1146 shared_function_info = no_reg;
1147
1148 // Calculate target function arity.
1149 Register expected_arity = rbx;
1150 __ movq(expected_arity, param_count);
1151
1152 // Make room to pass args and store the context.
1153 __ movq(rax, expected_arity);
1154 __ shlq(rax, Immediate(kSystemPointerSizeLog2));
1155 __ subq(rsp, rax); // Args.
1156
1157 Register param_index = param_count;
1158 __ Move(param_index, 0);
1159
1160 // -------------------------------------------
1161 // Store signature-related values to the stack.
1162 // -------------------------------------------
1163 // We store values on the stack to restore them after function calls.
1164 __ movq(MemOperand(rbp, kReturnCountOffset), return_count);
1165 __ movq(MemOperand(rbp, kValueTypesArrayStartOffset), valuetypes_array_ptr);
1166
1167 Label prepare_for_js_call;
1168 __ Cmp(expected_arity, 0);
1169 // If we have 0 params: jump through parameter handling.
1170 __ j(equal, &prepare_for_js_call);
1171
1172 // Loop through the params starting with the first.
1173 Register current_param_slot_offset = r10;
1174 __ Move(current_param_slot_offset, Immediate(0));
1175 Register param = rax;
1176
1177 // We have to check the types of the params. The ValueType array contains
1178 // first the return then the param types.
1179 constexpr int kValueTypeSize = sizeof(wasm::ValueType);
1180 static_assert(kValueTypeSize == 4);
1181 const int32_t kValueTypeSizeLog2 = log2(kValueTypeSize);
1182
1183 // Set the ValueType array pointer to point to the first parameter.
1184 Register returns_size = return_count;
1185 return_count = no_reg;
1186 __ shlq(returns_size, Immediate(kValueTypeSizeLog2));
1187 __ addq(valuetypes_array_ptr, returns_size);
1188 returns_size = no_reg;
1189 Register valuetype = r12;
1190
1191 // -------------------------------------------
1192 // Copy reference type params first and initialize the stack for JS arguments.
1193 // -------------------------------------------
1194
1195 // Heap pointers for ref type values in packed_args can be invalidated if GC
1196 // is triggered when converting wasm numbers to JS numbers and allocating
1197 // heap numbers. So, we have to move them to the stack first.
1198 {
1199 Label loop_copy_param_ref, load_ref_param, set_and_move;
1200
1201 __ bind(&loop_copy_param_ref);
1202 __ movl(valuetype, Operand(valuetypes_array_ptr, 0));
1203 __ testl(valuetype, Immediate(1));
1204 __ j(not_equal, &load_ref_param);
1205
1206 // Initialize non-ref type slots to zero since they can be visited by GC
1207 // when converting wasm numbers into heap numbers.
1208 __ Move(param, Smi::zero());
1209
1210 Label inc_param_32bit;
1211 __ cmpq(valuetype, Immediate(wasm::kWasmI32.raw_bit_field()));
1212 __ j(equal, &inc_param_32bit);
1213 __ cmpq(valuetype, Immediate(wasm::kWasmF32.raw_bit_field()));
1214 __ j(equal, &inc_param_32bit);
1215
1216 Label inc_param_64bit;
1217 __ cmpq(valuetype, Immediate(wasm::kWasmI64.raw_bit_field()));
1218 __ j(equal, &inc_param_64bit);
1219 __ cmpq(valuetype, Immediate(wasm::kWasmF64.raw_bit_field()));
1220 __ j(equal, &inc_param_64bit);
1221
1222 // Invalid type. Wasm cannot pass Simd arguments to JavaScript.
1223 __ int3();
1224
1225 __ bind(&inc_param_32bit);
1226 __ addq(current_param_slot_offset, Immediate(sizeof(int32_t)));
1227 __ jmp(&set_and_move);
1228
1229 __ bind(&inc_param_64bit);
1230 __ addq(current_param_slot_offset, Immediate(sizeof(int64_t)));
1231 __ jmp(&set_and_move);
1232
1233 __ bind(&load_ref_param);
1234 __ movq(param,
1235 MemOperand(packed_args, current_param_slot_offset, times_1, 0));
1236 __ addq(current_param_slot_offset, Immediate(kSystemPointerSize));
1237
1238 __ bind(&set_and_move);
1239 __ movq(MemOperand(rsp, param_index, times_system_pointer_size, 0), param);
1240 __ addq(valuetypes_array_ptr, Immediate(kValueTypeSize));
1241 __ incq(param_index);
1242 __ cmpq(param_index, MemOperand(rbp, kParamCountOffset));
1243 __ j(less, &loop_copy_param_ref);
1244 }
1245
1246 // Reset pointers for the second param conversion loop.
1247 returns_size = r8;
1248 __ movq(returns_size, MemOperand(rbp, kReturnCountOffset));
1249 __ shlq(returns_size, Immediate(kValueTypeSizeLog2));
1250 __ movq(valuetypes_array_ptr, MemOperand(rbp, kValueTypesArrayStartOffset));
1251 __ addq(valuetypes_array_ptr, returns_size);
1252 returns_size = no_reg;
1253 __ movq(current_param_slot_offset, Immediate(0));
1254 __ movq(param_index, Immediate(0));
1255
1256 // -------------------------------------------
1257 // Param evaluation loop.
1258 // -------------------------------------------
1259 Label loop_through_params;
1260 __ bind(&loop_through_params);
1261
1262 __ movl(valuetype, Operand(valuetypes_array_ptr, 0));
1263
1264 // -------------------------------------------
1265 // Param conversion.
1266 // -------------------------------------------
1267 // If param is a Smi we can easily convert it. Otherwise we'll call a builtin
1268 // for conversion.
1269 Label param_conversion_done, check_ref_param, skip_ref_param, convert_param;
1270
1271 __ cmpq(valuetype, Immediate(wasm::kWasmI32.raw_bit_field()));
1272 __ j(not_equal, &check_ref_param);
1273
1274 // I32 param: change to Smi.
1275 __ movl(param,
1276 MemOperand(packed_args, current_param_slot_offset, times_1, 0));
1277 // If pointer compression is disabled, we can convert to a Smi.
1278 if (SmiValuesAre32Bits()) {
1279 __ SmiTag(param);
1280 } else {
1281 Register temp = r15;
1282 __ movq(temp, param);
1283 // Double the return value to test if it can be a Smi.
1284 __ addl(temp, param);
1285 temp = no_reg;
1286 // If there was overflow, convert the return value to a HeapNumber.
1287 __ j(overflow, &convert_param);
1288 // If there was no overflow, we can convert to Smi.
1289 __ SmiTag(param);
1290 }
1291
1292 // Place the param into the proper slot.
1293 __ movq(MemOperand(rsp, param_index, times_system_pointer_size, 0), param);
1294 __ addq(current_param_slot_offset, Immediate(sizeof(int32_t)));
1295 __ jmp(&param_conversion_done);
1296
1297 // Skip Ref params. We already copied reference params in the first loop.
1298 __ bind(&check_ref_param);
1299 __ testl(valuetype, Immediate(1));
1300 __ j(equal, &convert_param);
1301
1302 __ bind(&skip_ref_param);
1303 __ addq(current_param_slot_offset, Immediate(kSystemPointerSize));
1304
1305 // -------------------------------------------
1306 // Param conversion done.
1307 // -------------------------------------------
1308 __ bind(&param_conversion_done);
1309 __ addq(valuetypes_array_ptr, Immediate(kValueTypeSize));
1310 __ incq(param_index);
1311 __ decq(expected_arity);
1312 __ j(equal, &prepare_for_js_call);
1313 __ cmpq(param_index, MemOperand(rbp, kParamCountOffset));
1314 __ j(not_equal, &loop_through_params);
1315
1316 // -------------------------------------------
1317 // Prepare for the function call.
1318 // -------------------------------------------
1319 __ bind(&prepare_for_js_call);
1320
1321 // Reset thread_in_wasm_flag.
1322 Register thread_in_wasm_flag_addr = rcx;
1323 __ movq(
1324 thread_in_wasm_flag_addr,
1325 MemOperand(kRootRegister, Isolate::thread_in_wasm_flag_address_offset()));
1326 __ movl(MemOperand(thread_in_wasm_flag_addr, 0), Immediate(0));
1327 thread_in_wasm_flag_addr = no_reg;
1328
1329 // -------------------------------------------
1330 // Call the JS function.
1331 // -------------------------------------------
1332 // Call_ReceiverIsAny expects the arguments in the stack in this order:
1333 // rsp + offset_PC Receiver
1334 // rsp + 0x10 JS arg 0
1335 // ... ...
1336 // rsp + N JS arg n-1
1337 //
1338 // It also expects two arguments passed in registers:
1339 // rax: number of arguments + 1 (receiver)
1340 // rdi: target JSFunction|JSBoundFunction|...
1341 //
1342 __ pushq(receiver);
1343
1344 // The process of calling a JS function might increase the number of tagged
1345 // values on the stack (arguments adaptation, BuiltinExitFrame arguments,
1346 // v8::FunctionCallbackInfo implicit arguments, etc.). In any case these
1347 // additional values must be visited by GC too.
1348 // We save the current stack pointer to restore it after the call.
1349 __ movq(MemOperand(rbp, WasmToJSInterpreterFrameConstants::kGCSPOffset), rsp);
1350
1351 __ movq(rax, MemOperand(rbp, kParamCountOffset));
1352 __ incq(rax); // Count receiver.
1353 __ Call(BUILTIN_CODE(masm->isolate(), Call_ReceiverIsAny),
1354 RelocInfo::CODE_TARGET);
1355
1356 __ movq(rsp, MemOperand(rbp, WasmToJSInterpreterFrameConstants::kGCSPOffset));
1357 __ movq(MemOperand(rbp, WasmToJSInterpreterFrameConstants::kGCSPOffset),
1358 Immediate(0));
1359
1360 __ popq(receiver);
1361
1362 // Retrieve context.
1363 __ movq(context, // GC_scan_limit
1364 MemOperand(
1365 rbp, WasmToJSInterpreterFrameConstants::kGCScanSlotLimitOffset));
1366 __ subq(context, Immediate(kSystemPointerSize));
1367 __ movq(context, MemOperand(context, 0));
1368 __ movq(MemOperand(rbp,
1369 WasmToJSInterpreterFrameConstants::kGCScanSlotLimitOffset),
1370 rsp);
1371
1372 // -------------------------------------------
1373 // Return handling.
1374 // -------------------------------------------
1375 Register return_reg = rax;
1376 return_count = rcx;
1377 __ movq(return_count, MemOperand(rbp, kReturnCountOffset));
1378 __ movq(packed_args, MemOperand(rbp, kPackedArrayOffset));
1379 __ movq(signature, MemOperand(rbp, kSignatureOffset));
1380 __ movq(valuetypes_array_ptr,
1381 MemOperand(signature, wasm::FunctionSig::kRepsOffset));
1382 Register result_index = r8;
1383 __ movq(result_index, Immediate(0));
1384
1385 // If we have return values, convert them from JS types back to Wasm types.
1386 Label convert_return;
1387 Label return_done;
1388 Label all_done;
1389 Label loop_copy_return_refs;
1390 Register fixed_array = r11;
1391 __ movq(fixed_array, Immediate(0));
1392 __ cmpl(return_count, Immediate(1));
1393 __ j(less, &all_done);
1394 __ j(equal, &convert_return);
1395
1396 // We have multiple results. Convert the result into a FixedArray.
1397 // The builtin expects three args:
1398 // rax: object.
1399 // rbx: return_count as Smi.
1400 // rsi: context.
1401 __ movq(rbx, MemOperand(rbp, kReturnCountOffset));
1402 __ addq(rbx, rbx);
1403 __ pushq(return_reg); // result
1404 __ pushq(context);
1405
1406 // We can have a GC here!
1407 __ Call(BUILTIN_CODE(masm->isolate(), IterableToFixedArrayForWasm),
1408 RelocInfo::CODE_TARGET);
1409 __ movq(MemOperand(rbp,
1410 WasmToJSInterpreterFrameConstants::kGCScanSlotLimitOffset),
1411 rsp);
1412 __ movq(fixed_array, rax);
1413 __ popq(context);
1414 __ popq(return_reg);
1415 __ movq(return_count, MemOperand(rbp, kReturnCountOffset));
1416 __ movq(packed_args, MemOperand(rbp, kPackedArrayOffset));
1417 __ movq(signature, MemOperand(rbp, kSignatureOffset));
1418 __ movq(valuetypes_array_ptr,
1419 MemOperand(signature, wasm::FunctionSig::kRepsOffset));
1420 __ movq(result_index, Immediate(0));
1421
1422 __ LoadTaggedField(return_reg,
1423 FieldOperand(fixed_array, result_index,
1424 static_cast<ScaleFactor>(kTaggedSizeLog2),
1425 OFFSET_OF_DATA_START(FixedArray)));
1426 __ jmp(&convert_return);
1427
1428 // A result converted.
1429 __ bind(&return_done);
1430
1431 // Restore after builtin call
1432 __ popq(context);
1433 __ popq(fixed_array);
1434 __ movq(valuetypes_array_ptr, MemOperand(rbp, kValueTypesArrayStartOffset));
1435
1436 __ addq(valuetypes_array_ptr, Immediate(kValueTypeSize));
1437 __ movq(result_index, MemOperand(rbp, kResultIndexOffset));
1438 __ incq(result_index);
1439 __ cmpq(result_index, MemOperand(rbp, kReturnCountOffset)); // return_count
1440 __ j(greater_equal, &loop_copy_return_refs);
1441
1442 __ LoadTaggedField(return_reg,
1443 FieldOperand(fixed_array, result_index,
1444 static_cast<ScaleFactor>(kTaggedSizeLog2),
1445 OFFSET_OF_DATA_START(FixedArray)));
1446 __ jmp(&convert_return);
1447
1448 // -------------------------------------------
1449 // Update refs after calling all builtins.
1450 // -------------------------------------------
1451
1452 // Some builtin calls for return value conversion may trigger GC, and some
1453 // heap pointers of ref types might become invalid in the conversion loop.
1454 // Thus, copy the ref values again after finishing all the conversions.
1455 __ bind(&loop_copy_return_refs);
1456
1457 // If there is only one return value, there should be no heap pointer in the
1458 // packed_args while calling any builtin. So, we don't need to update refs.
1459 __ movq(return_count, MemOperand(rbp, kReturnCountOffset));
1460 __ cmpl(return_count, Immediate(1));
1461 __ j(equal, &all_done);
1462
1463 Label copy_return_if_ref, copy_return_ref, done_copy_return_ref;
1464 __ movq(packed_args, MemOperand(rbp, kPackedArrayOffset));
1465 __ movq(signature, MemOperand(rbp, kSignatureOffset));
1466 __ movq(valuetypes_array_ptr,
1467 MemOperand(signature, wasm::FunctionSig::kRepsOffset));
1468 __ movq(result_index, Immediate(0));
1469
1470 // Copy if the current return value is a ref type.
1471 __ bind(&copy_return_if_ref);
1472 __ movl(valuetype, Operand(valuetypes_array_ptr, 0));
1473
1474 __ testl(valuetype, Immediate(1));
1475 __ j(not_equal, &copy_return_ref);
1476
1477 Label inc_result_32bit;
1478 __ cmpq(valuetype, Immediate(wasm::kWasmI32.raw_bit_field()));
1479 __ j(equal, &inc_result_32bit);
1480 __ cmpq(valuetype, Immediate(wasm::kWasmF32.raw_bit_field()));
1481 __ j(equal, &inc_result_32bit);
1482
1483 Label inc_result_64bit;
1484 __ cmpq(valuetype, Immediate(wasm::kWasmI64.raw_bit_field()));
1485 __ j(equal, &inc_result_64bit);
1486 __ cmpq(valuetype, Immediate(wasm::kWasmF64.raw_bit_field()));
1487 __ j(equal, &inc_result_64bit);
1488
1489 // Invalid type. JavaScript cannot return Simd values to WebAssembly.
1490 __ int3();
1491
1492 __ bind(&inc_result_32bit);
1493 __ addq(packed_args, Immediate(sizeof(int32_t)));
1494 __ jmp(&done_copy_return_ref);
1495
1496 __ bind(&inc_result_64bit);
1497 __ addq(packed_args, Immediate(sizeof(int64_t)));
1498 __ jmp(&done_copy_return_ref);
1499
1500 __ bind(&copy_return_ref);
1501 __ LoadTaggedField(return_reg,
1502 FieldOperand(fixed_array, result_index,
1503 static_cast<ScaleFactor>(kTaggedSizeLog2),
1504 OFFSET_OF_DATA_START(FixedArray)));
1505 __ movq(MemOperand(packed_args, 0), return_reg);
1506 __ addq(packed_args, Immediate(kSystemPointerSize));
1507
1508 // Move pointers.
1509 __ bind(&done_copy_return_ref);
1510 __ addq(valuetypes_array_ptr, Immediate(kValueTypeSize));
1511 __ incq(result_index);
1512 __ cmpq(result_index, MemOperand(rbp, kReturnCountOffset));
1513 __ j(less, &copy_return_if_ref);
1514
1515 // -------------------------------------------
1516 // All done.
1517 // -------------------------------------------
1518
1519 __ bind(&all_done);
1520 // Set thread_in_wasm_flag.
1521 thread_in_wasm_flag_addr = rcx;
1522 __ movq(
1523 thread_in_wasm_flag_addr,
1524 MemOperand(kRootRegister, Isolate::thread_in_wasm_flag_address_offset()));
1525 __ movl(MemOperand(thread_in_wasm_flag_addr, 0), Immediate(1));
1526 thread_in_wasm_flag_addr = no_reg;
1527
1528 // Deconstruct the stack frame.
1529 __ LeaveFrame(StackFrame::WASM_TO_JS);
1530
1531 __ movq(rax, Immediate(WasmToJSInterpreterFrameConstants::kSuccess));
1532 __ ret(0);
1533
1534 // --------------------------------------------------------------------------
1535 // Deferred code.
1536 // --------------------------------------------------------------------------
1537
1538 // -------------------------------------------------
1539 // Param conversion builtins (Wasm type -> JS type).
1540 // -------------------------------------------------
1541 __ bind(&convert_param);
1542
1543 // Prepare for builtin call.
1544
1545 // Need to specify how many heap objects, that should be scanned by GC, are
1546 // on the top of the stack. (Only the context).
1547 // The builtin expects the parameter to be in register param = rax.
1548
1549 __ movq(MemOperand(rbp, kExpectedArityOffset), expected_arity);
1550 __ movq(MemOperand(rbp, kParamIndexOffset), param_index);
1551 __ movq(MemOperand(rbp, kValueTypesArrayStartOffset), valuetypes_array_ptr);
1552 __ movq(MemOperand(rbp, kCurrentParamOffset), current_param_slot_offset);
1553
1554 __ pushq(receiver);
1555 __ pushq(callable);
1556 __ pushq(context);
1557
1558 Label param_kWasmI32_not_smi;
1559 Label param_kWasmI64;
1560 Label param_kWasmF32;
1561 Label param_kWasmF64;
1562 Label finish_param_conversion;
1563
1564 __ cmpq(valuetype, Immediate(wasm::kWasmI32.raw_bit_field()));
1565 __ j(equal, &param_kWasmI32_not_smi);
1566
1567 __ cmpq(valuetype, Immediate(wasm::kWasmI64.raw_bit_field()));
1568 __ j(equal, &param_kWasmI64);
1569
1570 __ cmpq(valuetype, Immediate(wasm::kWasmF32.raw_bit_field()));
1571 __ j(equal, &param_kWasmF32);
1572
1573 __ cmpq(valuetype, Immediate(wasm::kWasmF64.raw_bit_field()));
1574 __ j(equal, &param_kWasmF64);
1575
1576 // Invalid type. Wasm cannot pass Simd arguments to JavaScript.
1577 __ int3();
1578
1579 __ bind(&param_kWasmI32_not_smi);
1580 __ Call(BUILTIN_CODE(masm->isolate(), WasmInt32ToHeapNumber),
1581 RelocInfo::CODE_TARGET);
1582 // Param is the result of the builtin.
1583 __ movq(rbx, Immediate(sizeof(int32_t)));
1584 __ jmp(&finish_param_conversion);
1585
1586 __ bind(&param_kWasmI64);
1587 __ movq(param,
1588 MemOperand(packed_args, current_param_slot_offset, times_1, 0));
1589 __ Call(BUILTIN_CODE(masm->isolate(), I64ToBigInt), RelocInfo::CODE_TARGET);
1590 __ movq(rbx, Immediate(sizeof(int64_t)));
1591 __ jmp(&finish_param_conversion);
1592
1593 __ bind(&param_kWasmF32);
1594 __ Movsd(xmm0,
1595 MemOperand(packed_args, current_param_slot_offset, times_1, 0));
1596 __ Call(BUILTIN_CODE(masm->isolate(), WasmFloat32ToNumber),
1597 RelocInfo::CODE_TARGET);
1598 __ movq(rbx, Immediate(sizeof(float)));
1599 __ jmp(&finish_param_conversion);
1600
1601 __ bind(&param_kWasmF64);
1602 __ movq(xmm0, MemOperand(packed_args, current_param_slot_offset, times_1, 0));
1603 __ Call(BUILTIN_CODE(masm->isolate(), WasmFloat64ToNumber),
1604 RelocInfo::CODE_TARGET);
1605 __ movq(rbx, Immediate(sizeof(double)));
1606
1607 // Restore after builtin call.
1608 __ bind(&finish_param_conversion);
1609 __ popq(context);
1610 __ popq(callable);
1611 __ popq(receiver);
1612
1613 __ movq(current_param_slot_offset, MemOperand(rbp, kCurrentParamOffset));
1614 __ addq(current_param_slot_offset, rbx);
1615 __ movq(valuetypes_array_ptr, MemOperand(rbp, kValueTypesArrayStartOffset));
1616 __ movq(param_index, MemOperand(rbp, kParamIndexOffset));
1617 __ movq(expected_arity, MemOperand(rbp, kExpectedArityOffset));
1618 __ movq(packed_args, MemOperand(rbp, kPackedArrayOffset));
1619
1620 __ movq(MemOperand(rsp, param_index, times_system_pointer_size, 0), param);
1621 __ jmp(&param_conversion_done);
1622
1623 // -------------------------------------------
1624 // Return conversions (JS type -> Wasm type).
1625 // -------------------------------------------
1626 __ bind(&convert_return);
1627
1628 // Save registers in the stack before the builtin call.
1629 __ movq(MemOperand(rbp, kResultIndexOffset), result_index);
1630 __ movq(MemOperand(rbp, kValueTypesArrayStartOffset), valuetypes_array_ptr);
1631 __ movq(MemOperand(rbp, kCurrentResultAddressOffset), packed_args);
1632
1633 __ movq(MemOperand(rbp,
1634 WasmToJSInterpreterFrameConstants::kGCScanSlotLimitOffset),
1635 rsp);
1636 __ pushq(fixed_array);
1637 __ pushq(context);
1638
1639 // The builtin expects the parameter to be in register param = rax.
1640
1641 // The first valuetype of the array is the return's valuetype.
1642 __ movl(valuetype, Operand(valuetypes_array_ptr, 0));
1643
1644 Label return_kWasmI32;
1645 Label return_kWasmI32_not_smi;
1646 Label return_kWasmI64;
1647 Label return_kWasmF32;
1648 Label return_kWasmF64;
1649 Label return_kWasmRef;
1650
1651 // Prepare for builtin call.
1652
1653 __ cmpq(valuetype, Immediate(wasm::kWasmI32.raw_bit_field()));
1654 __ j(equal, &return_kWasmI32);
1655
1656 __ cmpq(valuetype, Immediate(wasm::kWasmI64.raw_bit_field()));
1657 __ j(equal, &return_kWasmI64);
1658
1659 __ cmpq(valuetype, Immediate(wasm::kWasmF32.raw_bit_field()));
1660 __ j(equal, &return_kWasmF32);
1661
1662 __ cmpq(valuetype, Immediate(wasm::kWasmF64.raw_bit_field()));
1663 __ j(equal, &return_kWasmF64);
1664
1665 __ testl(valuetype, Immediate(1));
1666 __ j(not_equal, &return_kWasmRef);
1667
1668 // Invalid type. JavaScript cannot return Simd results to WebAssembly.
1669 __ int3();
1670
1671 __ bind(&return_kWasmI32);
1672 __ JumpIfNotSmi(return_reg, &return_kWasmI32_not_smi);
1673 // Change the param from Smi to int32.
1674 __ SmiUntag(return_reg);
1675 // Zero extend.
1676 __ movl(return_reg, return_reg);
1677 __ movl(MemOperand(packed_args, 0), return_reg);
1678 __ addq(packed_args, Immediate(sizeof(int32_t)));
1679 __ jmp(&return_done);
1680
1681 __ bind(&return_kWasmI32_not_smi);
1682 __ Call(BUILTIN_CODE(masm->isolate(), WasmTaggedNonSmiToInt32),
1683 RelocInfo::CODE_TARGET);
1684 __ AssertZeroExtended(return_reg);
1685 __ movq(packed_args, MemOperand(rbp, kCurrentResultAddressOffset));
1686 __ movl(MemOperand(packed_args, 0), return_reg);
1687 __ addq(packed_args, Immediate(sizeof(int32_t)));
1688 __ jmp(&return_done);
1689
1690 __ bind(&return_kWasmI64);
1691 __ Call(BUILTIN_CODE(masm->isolate(), BigIntToI64), RelocInfo::CODE_TARGET);
1692 __ movq(packed_args, MemOperand(rbp, kCurrentResultAddressOffset));
1693 __ movq(MemOperand(packed_args, 0), return_reg);
1694 __ addq(packed_args, Immediate(sizeof(int64_t)));
1695 __ jmp(&return_done);
1696
1697 __ bind(&return_kWasmF32);
1698 __ Call(BUILTIN_CODE(masm->isolate(), WasmTaggedToFloat32),
1699 RelocInfo::CODE_TARGET);
1700 __ movq(packed_args, MemOperand(rbp, kCurrentResultAddressOffset));
1701 __ Movss(MemOperand(packed_args, 0), xmm0);
1702 __ addq(packed_args, Immediate(sizeof(float)));
1703 __ jmp(&return_done);
1704
1705 __ bind(&return_kWasmF64);
1706 __ Call(BUILTIN_CODE(masm->isolate(), WasmTaggedToFloat64),
1707 RelocInfo::CODE_TARGET);
1708 __ movq(packed_args, MemOperand(rbp, kCurrentResultAddressOffset));
1709 __ Movsd(MemOperand(packed_args, 0), xmm0);
1710 __ addq(packed_args, Immediate(sizeof(double)));
1711 __ jmp(&return_done);
1712
1713 __ bind(&return_kWasmRef);
1714 __ movq(MemOperand(packed_args, 0), return_reg);
1715 __ addq(packed_args, Immediate(kSystemPointerSize));
1716 __ jmp(&return_done);
1717}
1718
1719#ifndef V8_DRUMBRAKE_BOUNDS_CHECKS
1720
1721namespace {
1722
1723enum IntMemoryType {
1724 kIntS8,
1725 kIntU8,
1726 kIntS16,
1727 kIntU16,
1728 kIntS32,
1729 kIntU32,
1730 kInt64
1731};
1732
1733enum IntValueType { kValueInt32, kValueInt64 };
1734
1735enum FloatType { kFloat32, kFloat64 };
1736
1737void EmitLoadInstruction(MacroAssembler* masm, Register result,
1738 Register memory_start, Register memory_index,
1739 IntValueType value_type, IntMemoryType memory_type) {
1740 switch (memory_type) {
1741 case kInt64:
1742 switch (value_type) {
1743 case kValueInt64:
1744 __ movq(result, Operand(memory_start, memory_index, times_1, 0));
1745 break;
1746 default:
1747 UNREACHABLE();
1748 }
1749 break;
1750 case kIntS32:
1751 switch (value_type) {
1752 case kValueInt64:
1753 __ movsxlq(result, Operand(memory_start, memory_index, times_1, 0));
1754 break;
1755 case kValueInt32:
1756 __ movl(result, Operand(memory_start, memory_index, times_1, 0));
1757 break;
1758 }
1759 break;
1760 case kIntU32:
1761 switch (value_type) {
1762 case kValueInt64:
1763 __ movl(result, Operand(memory_start, memory_index, times_1, 0));
1764 break;
1765 default:
1766 UNREACHABLE();
1767 }
1768 break;
1769 case kIntS16:
1770 switch (value_type) {
1771 case kValueInt64:
1772 __ movsxwq(result, Operand(memory_start, memory_index, times_1, 0));
1773 break;
1774 case kValueInt32:
1775 __ movsxwl(result, Operand(memory_start, memory_index, times_1, 0));
1776 break;
1777 }
1778 break;
1779 case kIntU16:
1780 switch (value_type) {
1781 case kValueInt64:
1782 __ movzxwq(result, Operand(memory_start, memory_index, times_1, 0));
1783 break;
1784 case kValueInt32:
1785 __ movzxwl(result, Operand(memory_start, memory_index, times_1, 0));
1786 break;
1787 }
1788 break;
1789 case kIntS8:
1790 switch (value_type) {
1791 case kValueInt64:
1792 __ movsxbq(result, Operand(memory_start, memory_index, times_1, 0));
1793 break;
1794 case kValueInt32:
1795 __ movsxbl(result, Operand(memory_start, memory_index, times_1, 0));
1796 break;
1797 }
1798 break;
1799 case kIntU8:
1800 switch (value_type) {
1801 case kValueInt64:
1802 __ movzxbq(result, Operand(memory_start, memory_index, times_1, 0));
1803 break;
1804 case kValueInt32:
1805 __ movzxbl(result, Operand(memory_start, memory_index, times_1, 0));
1806 break;
1807 }
1808 break;
1809 default:
1810 UNREACHABLE();
1811 }
1812}
1813
1814void EmitLoadInstruction(MacroAssembler* masm, Register memory_start,
1815 Register memory_offset, XMMRegister result,
1816 FloatType float_type) {
1817 switch (float_type) {
1818 case kFloat32:
1819 __ movss(xmm0, Operand(memory_start, memory_offset, times_1, 0));
1820 __ cvtss2sd(result, xmm0);
1821 break;
1822 case kFloat64:
1823 __ movsd(result, Operand(memory_start, memory_offset, times_1, 0));
1824 break;
1825 default:
1826 UNREACHABLE();
1827 }
1828}
1829
1830void EmitLoadInstruction(MacroAssembler* masm, Register memory_start,
1831 Register memory_offset, Register sp,
1832 Register slot_offset, FloatType float_type) {
1833 switch (float_type) {
1834 case kFloat32:
1835 __ movss(xmm0, Operand(memory_start, memory_offset, times_1, 0));
1836 __ movss(Operand(sp, slot_offset, times_4, 0), xmm0);
1837 break;
1838 case kFloat64:
1839 __ movsd(xmm0, Operand(memory_start, memory_offset, times_1, 0));
1840 __ movsd(Operand(sp, slot_offset, times_4, 0), xmm0);
1841 break;
1842 default:
1843 UNREACHABLE();
1844 }
1845}
1846
1847void WriteToSlot(MacroAssembler* masm, Register sp, Register slot_offset,
1848 Register value, IntValueType value_type) {
1849 switch (value_type) {
1850 case kValueInt64:
1851 __ movq(Operand(sp, slot_offset, times_4, 0), value);
1852 break;
1853 case kValueInt32:
1854 __ movl(Operand(sp, slot_offset, times_4, 0), value);
1855 break;
1856 }
1857}
1858
1859void EmitStoreInstruction(MacroAssembler* masm, Register value,
1860 Register memory_start, Register memory_index,
1861 IntMemoryType memory_type) {
1862 switch (memory_type) {
1863 case kInt64:
1864 __ movq(Operand(memory_start, memory_index, times_1, 0), value);
1865 break;
1866 case kIntS32:
1867 __ movl(Operand(memory_start, memory_index, times_1, 0), value);
1868 break;
1869 case kIntS16:
1870 __ movw(Operand(memory_start, memory_index, times_1, 0), value);
1871 break;
1872 case kIntS8:
1873 __ movb(Operand(memory_start, memory_index, times_1, 0), value);
1874 break;
1875 default:
1876 UNREACHABLE();
1877 }
1878}
1879
1880void EmitStoreInstruction(MacroAssembler* masm, XMMRegister value,
1881 Register memory_start, Register memory_index,
1882 FloatType float_type) {
1883 switch (float_type) {
1884 case kFloat32:
1885 __ movss(Operand(memory_start, memory_index, times_1, 0), value);
1886 break;
1887 case kFloat64:
1888 __ movsd(Operand(memory_start, memory_index, times_1, 0), value);
1889 break;
1890 default:
1891 UNREACHABLE();
1892 }
1893}
1894
1895void EmitLoadNextInstructionId(MacroAssembler* masm, Register next_handler_id,
1896 Register code, uint32_t code_offset) {
1897 // An InstructionHandler id is stored in the WasmBytecode as a uint16_t, so we
1898 // need to move a 16-bit word here.
1899 __ movzxwq(next_handler_id, MemOperand(code, code_offset));
1900
1901 // Currently, there cannot be more than kInstructionTableSize = 1024 different
1902 // handlers, so (for additional security) we do a bitwise AND with 1023 to
1903 // make sure some attacker might somehow generate invalid WasmBytecode data
1904 // and force an indirect jump through memory outside the handler table.
1905 __ andq(next_handler_id, Immediate(wasm::kInstructionTableMask));
1906}
1907
1908template <bool Compressed>
1909class WasmInterpreterHandlerCodeEmitter {
1910 public:
1911 static void EmitLoadSlotOffset(MacroAssembler* masm, Register slot_offset,
1912 const MemOperand& operand);
1913 static void EmitLoadMemoryOffset(MacroAssembler* masm, Register memory_offset,
1914 const MemOperand& operand);
1915};
1916
1917template <>
1918void WasmInterpreterHandlerCodeEmitter<true>::EmitLoadSlotOffset(
1919 MacroAssembler* masm, Register slot_offset, const MemOperand& operand) {
1920 __ movzxwq(slot_offset, operand);
1921}
1922template <>
1923void WasmInterpreterHandlerCodeEmitter<true>::EmitLoadMemoryOffset(
1924 MacroAssembler* masm, Register memory_offset, const MemOperand& operand) {
1925 __ movl(memory_offset, operand);
1926}
1927
1928template <>
1929void WasmInterpreterHandlerCodeEmitter<false>::EmitLoadSlotOffset(
1930 MacroAssembler* masm, Register slot_offset, const MemOperand& operand) {
1931 __ movl(slot_offset, operand);
1932}
1933template <>
1934void WasmInterpreterHandlerCodeEmitter<false>::EmitLoadMemoryOffset(
1935 MacroAssembler* masm, Register memory_offset, const MemOperand& operand) {
1936 __ movq(memory_offset, operand);
1937}
1938
1939template <bool Compressed>
1940class WasmInterpreterHandlerBuiltins {
1941 using slot_offset_t = wasm::handler_traits<Compressed>::slot_offset_t;
1942 using memory_offset32_t = wasm::handler_traits<Compressed>::memory_offset32_t;
1943 using handler_id_t = wasm::handler_traits<Compressed>::handler_id_t;
1944 using emitter = WasmInterpreterHandlerCodeEmitter<Compressed>;
1945
1946 public:
1947 static void Generate_r2r_ILoadMem(MacroAssembler* masm,
1948 IntValueType value_type,
1949 IntMemoryType memory_type) {
1950 constexpr uint32_t kMemoryOffset = 0;
1951 constexpr uint32_t kNextHandlerId =
1952 kMemoryOffset + sizeof(memory_offset32_t);
1953 constexpr uint32_t kInstructionCodeLength =
1954 kNextHandlerId + sizeof(handler_id_t);
1955
1956 Register code = rcx;
1957 Register wasm_runtime = r8;
1958 Register memory_index = r9;
1959
1960 __ movl(memory_index, memory_index);
1961
1962 Register memory_start_plus_index = memory_index;
1963 __ addq(memory_start_plus_index,
1964 MemOperand(wasm_runtime,
1965 wasm::WasmInterpreterRuntime::memory_start_offset()));
1966
1967 Register memory_offset = rax;
1968 emitter::EmitLoadMemoryOffset(masm, memory_offset,
1969 MemOperand(code, kMemoryOffset));
1970
1971 Register result = r9;
1972 EmitLoadInstruction(masm, result, memory_start_plus_index, memory_offset,
1973 value_type, memory_type);
1974
1975 Register next_handler_id = r10;
1976 EmitLoadNextInstructionId(masm, next_handler_id, code, kNextHandlerId);
1977 __ addq(code, Immediate(kInstructionCodeLength));
1978
1979 Register instr_table = rax;
1980 __ movq(
1981 instr_table,
1982 MemOperand(wasm_runtime,
1983 wasm::WasmInterpreterRuntime::instruction_table_offset()));
1984
1985 Register next_handler_addr = rax;
1986 __ movq(next_handler_addr,
1987 MemOperand(instr_table, next_handler_id, times_8, 0));
1988 __ jmp(next_handler_addr);
1989 }
1990
1991 static void Generate_r2r_FLoadMem(MacroAssembler* masm,
1992 FloatType float_type) {
1993 constexpr uint32_t kMemoryOffset = 0;
1994 constexpr uint32_t kNextHandlerId =
1995 kMemoryOffset + sizeof(memory_offset32_t);
1996 constexpr uint32_t kInstructionCodeLength =
1997 kNextHandlerId + sizeof(handler_id_t);
1998
1999 Register code = rcx;
2000 Register wasm_runtime = r8;
2001 Register memory_index = r9;
2002
2003 __ movl(memory_index, memory_index);
2004
2005 Register memory_start_plus_index = memory_index;
2006 __ addq(memory_start_plus_index,
2007 MemOperand(wasm_runtime,
2008 wasm::WasmInterpreterRuntime::memory_start_offset()));
2009
2010 Register memory_offset = rax;
2011 emitter::EmitLoadMemoryOffset(masm, memory_offset,
2012 MemOperand(code, kMemoryOffset));
2013
2014 EmitLoadInstruction(masm, memory_start_plus_index, memory_offset, xmm4,
2015 float_type);
2016
2017 Register next_handler_id = r10;
2018 EmitLoadNextInstructionId(masm, next_handler_id, code, kNextHandlerId);
2019 __ addq(code, Immediate(kInstructionCodeLength));
2020
2021 Register instr_table = rax;
2022 __ movq(
2023 instr_table,
2024 MemOperand(wasm_runtime,
2025 wasm::WasmInterpreterRuntime::instruction_table_offset()));
2026
2027 Register next_handler_addr = rax;
2028 __ movq(next_handler_addr,
2029 MemOperand(instr_table, next_handler_id, times_8, 0));
2030 __ jmp(next_handler_addr);
2031 }
2032
2033 static void Generate_r2s_ILoadMem(MacroAssembler* masm,
2034 IntValueType value_type,
2035 IntMemoryType memory_type) {
2036 constexpr uint32_t kMemoryOffset = 0;
2037 constexpr uint32_t kSlotOffset = kMemoryOffset + sizeof(memory_offset32_t);
2038 constexpr uint32_t kNextHandlerId = kSlotOffset + sizeof(slot_offset_t);
2039 constexpr uint32_t kInstructionCodeLength =
2040 kNextHandlerId + sizeof(handler_id_t);
2041
2042 Register code = rcx;
2043 Register sp = rdx;
2044 Register wasm_runtime = r8;
2045 Register memory_index = r9;
2046
2047 __ movl(memory_index, memory_index);
2048
2049 Register memory_start_plus_index = memory_index;
2050 __ addq(memory_start_plus_index,
2051 MemOperand(wasm_runtime,
2052 wasm::WasmInterpreterRuntime::memory_start_offset()));
2053
2054 Register memory_offset = rax;
2055 emitter::EmitLoadMemoryOffset(masm, memory_offset,
2056 MemOperand(code, kMemoryOffset));
2057
2058 Register value = r10;
2059 EmitLoadInstruction(masm, value, memory_start_plus_index, memory_offset,
2060 value_type, memory_type);
2061
2062 Register slot_offset = rax;
2063 emitter::EmitLoadSlotOffset(masm, slot_offset,
2064 MemOperand(code, kSlotOffset));
2065
2066 WriteToSlot(masm, sp, slot_offset, value, value_type);
2067
2068 Register next_handler_id = r10;
2069 EmitLoadNextInstructionId(masm, next_handler_id, code, kNextHandlerId);
2070 __ addq(code, Immediate(kInstructionCodeLength));
2071
2072 Register instr_table = rax;
2073 __ movq(
2074 instr_table,
2075 MemOperand(wasm_runtime,
2076 wasm::WasmInterpreterRuntime::instruction_table_offset()));
2077
2078 Register next_handler_addr = rax;
2079 __ movq(next_handler_addr,
2080 MemOperand(instr_table, next_handler_id, times_8, 0));
2081 __ jmp(next_handler_addr);
2082 }
2083
2084 static void Generate_r2s_FLoadMem(MacroAssembler* masm,
2085 FloatType float_type) {
2086 constexpr uint32_t kMemoryOffset = 0;
2087 constexpr uint32_t kSlotOffset = kMemoryOffset + sizeof(memory_offset32_t);
2088 constexpr uint32_t kNextHandlerId = kSlotOffset + sizeof(slot_offset_t);
2089 constexpr uint32_t kInstructionCodeLength =
2090 kNextHandlerId + sizeof(handler_id_t);
2091
2092 Register code = rcx;
2093 Register sp = rdx;
2094 Register wasm_runtime = r8;
2095 Register memory_index = r9;
2096
2097 __ movl(memory_index, memory_index);
2098
2099 Register memory_start_plus_index = memory_index;
2100 __ addq(memory_start_plus_index,
2101 MemOperand(wasm_runtime,
2102 wasm::WasmInterpreterRuntime::memory_start_offset()));
2103
2104 Register memory_offset = rax;
2105 emitter::EmitLoadMemoryOffset(masm, memory_offset,
2106 MemOperand(code, kMemoryOffset));
2107
2108 Register slot_offset = r11;
2109 emitter::EmitLoadSlotOffset(masm, slot_offset,
2110 MemOperand(code, kSlotOffset));
2111
2112 EmitLoadInstruction(masm, memory_start_plus_index, memory_offset, sp,
2113 slot_offset, float_type);
2114
2115 Register next_handler_id = r10;
2116 EmitLoadNextInstructionId(masm, next_handler_id, code, kNextHandlerId);
2117 __ addq(code, Immediate(kInstructionCodeLength));
2118
2119 Register instr_table = rax;
2120 __ movq(
2121 instr_table,
2122 MemOperand(wasm_runtime,
2123 wasm::WasmInterpreterRuntime::instruction_table_offset()));
2124
2125 Register next_handler_addr = rax;
2126 __ movq(next_handler_addr,
2127 MemOperand(instr_table, next_handler_id, times_8, 0));
2128 __ jmp(next_handler_addr);
2129 }
2130
2131 static void Generate_s2r_ILoadMem(MacroAssembler* masm,
2132 IntValueType value_type,
2133 IntMemoryType memory_type) {
2134 constexpr uint32_t kMemoryOffset = 0;
2135 constexpr uint32_t kMemoryIndexSlot =
2136 kMemoryOffset + sizeof(memory_offset32_t);
2137 constexpr uint32_t kNextHandlerId =
2138 kMemoryIndexSlot + sizeof(slot_offset_t);
2139 constexpr uint32_t kInstructionCodeLength =
2140 kNextHandlerId + sizeof(handler_id_t);
2141
2142 Register code = rcx;
2143 Register sp = rdx;
2144 Register wasm_runtime = r8;
2145
2146 Register memory_offset = r10;
2147 emitter::EmitLoadMemoryOffset(masm, memory_offset,
2148 MemOperand(code, kMemoryOffset));
2149
2150 Register memory_index_slot_offset = rax;
2151 emitter::EmitLoadSlotOffset(masm, memory_index_slot_offset,
2152 MemOperand(code, kMemoryIndexSlot));
2153
2154 Register memory_start_plus_offset = memory_offset;
2155 __ addq(memory_start_plus_offset,
2156 MemOperand(wasm_runtime,
2157 wasm::WasmInterpreterRuntime::memory_start_offset()));
2158
2159 Register memory_index = memory_index_slot_offset;
2160 __ movl(memory_index, Operand(sp, memory_index_slot_offset, times_4, 0));
2161
2162 Register value = r9;
2163 EmitLoadInstruction(masm, value, memory_start_plus_offset, memory_index,
2164 value_type, memory_type);
2165
2166 Register next_handler_id = r10;
2167 EmitLoadNextInstructionId(masm, next_handler_id, code, kNextHandlerId);
2168 __ addq(code, Immediate(kInstructionCodeLength));
2169
2170 Register instr_table = rax;
2171 __ movq(
2172 instr_table,
2173 MemOperand(wasm_runtime,
2174 wasm::WasmInterpreterRuntime::instruction_table_offset()));
2175
2176 Register next_handler_addr = rax;
2177 __ movq(next_handler_addr,
2178 MemOperand(instr_table, next_handler_id, times_8, 0));
2179 __ jmp(next_handler_addr);
2180 }
2181
2182 static void Generate_s2r_FLoadMem(MacroAssembler* masm,
2183 FloatType float_type) {
2184 constexpr uint32_t kMemoryOffset = 0;
2185 constexpr uint32_t kMemoryIndexSlot =
2186 kMemoryOffset + sizeof(memory_offset32_t);
2187 constexpr uint32_t kNextHandlerId =
2188 kMemoryIndexSlot + sizeof(slot_offset_t);
2189 constexpr uint32_t kInstructionCodeLength =
2190 kNextHandlerId + sizeof(handler_id_t);
2191
2192 Register code = rcx;
2193 Register sp = rdx;
2194 Register wasm_runtime = r8;
2195
2196 Register memory_offset = r10;
2197 emitter::EmitLoadMemoryOffset(masm, memory_offset,
2198 MemOperand(code, kMemoryOffset));
2199
2200 Register memory_index_slot_offset = rax;
2201 emitter::EmitLoadSlotOffset(masm, memory_index_slot_offset,
2202 MemOperand(code, kMemoryIndexSlot));
2203
2204 Register memory_start_plus_offset = memory_offset;
2205 __ addq(memory_start_plus_offset,
2206 MemOperand(wasm_runtime,
2207 wasm::WasmInterpreterRuntime::memory_start_offset()));
2208
2209 Register memory_index = memory_index_slot_offset;
2210 __ movl(memory_index, Operand(sp, memory_index_slot_offset, times_4, 0));
2211
2212 EmitLoadInstruction(masm, memory_start_plus_offset, memory_index, xmm4,
2213 float_type);
2214
2215 Register next_handler_id = r10;
2216 EmitLoadNextInstructionId(masm, next_handler_id, code, kNextHandlerId);
2217 __ addq(code, Immediate(kInstructionCodeLength));
2218
2219 Register instr_table = rax;
2220 __ movq(
2221 instr_table,
2222 MemOperand(wasm_runtime,
2223 wasm::WasmInterpreterRuntime::instruction_table_offset()));
2224
2225 Register next_handler_addr = rax;
2226 __ movq(next_handler_addr,
2227 MemOperand(instr_table, next_handler_id, times_8, 0));
2228 __ jmp(next_handler_addr);
2229 }
2230
2231 static void Generate_s2s_ILoadMem(MacroAssembler* masm,
2232 IntValueType value_type,
2233 IntMemoryType memory_type) {
2234 constexpr uint32_t kMemoryOffset = 0;
2235 constexpr uint32_t kIndexSlot = kMemoryOffset + sizeof(memory_offset32_t);
2236 constexpr uint32_t kResultSlot = kIndexSlot + sizeof(slot_offset_t);
2237 constexpr uint32_t kNextHandlerId = kResultSlot + sizeof(slot_offset_t);
2238 constexpr uint32_t kInstructionCodeLength =
2239 kNextHandlerId + sizeof(handler_id_t);
2240
2241 Register code = rcx;
2242 Register sp = rdx;
2243 Register wasm_runtime = r8;
2244
2245 Register memory_offset = r10;
2246 emitter::EmitLoadMemoryOffset(masm, memory_offset,
2247 MemOperand(code, kMemoryOffset));
2248
2249 Register memory_index_slot_offset = rax;
2250 emitter::EmitLoadSlotOffset(masm, memory_index_slot_offset,
2251 MemOperand(code, kIndexSlot));
2252
2253 Register memory_start_plus_offset = memory_offset;
2254 __ addq(memory_start_plus_offset,
2255 MemOperand(wasm_runtime,
2256 wasm::WasmInterpreterRuntime::memory_start_offset()));
2257
2258 Register memory_index = r9;
2259 __ movl(memory_index, Operand(sp, memory_index_slot_offset, times_4, 0));
2260
2261 Register result_slot_offset = r11;
2262 emitter::EmitLoadSlotOffset(masm, result_slot_offset,
2263 MemOperand(code, kResultSlot));
2264
2265 Register value = rax;
2266 EmitLoadInstruction(masm, value, memory_start_plus_offset, memory_index,
2267 value_type, memory_type);
2268
2269 WriteToSlot(masm, sp, result_slot_offset, value, value_type);
2270
2271 Register next_handler_id = r10;
2272 EmitLoadNextInstructionId(masm, next_handler_id, code, kNextHandlerId);
2273 __ addq(code, Immediate(kInstructionCodeLength));
2274
2275 Register instr_table = rax;
2276 __ movq(
2277 instr_table,
2278 MemOperand(wasm_runtime,
2279 wasm::WasmInterpreterRuntime::instruction_table_offset()));
2280
2281 Register next_handler_addr = rax;
2282 __ movq(next_handler_addr,
2283 MemOperand(instr_table, next_handler_id, times_8, 0));
2284 __ jmp(next_handler_addr);
2285 }
2286
2287 static void Generate_s2s_FLoadMem(MacroAssembler* masm,
2288 FloatType float_type) {
2289 constexpr uint32_t kMemoryOffset = 0;
2290 constexpr uint32_t kIndexSlot = kMemoryOffset + sizeof(memory_offset32_t);
2291 constexpr uint32_t kResultSlot = kIndexSlot + sizeof(slot_offset_t);
2292 constexpr uint32_t kNextHandlerId = kResultSlot + sizeof(slot_offset_t);
2293 constexpr uint32_t kInstructionCodeLength =
2294 kNextHandlerId + sizeof(handler_id_t);
2295
2296 Register code = rcx;
2297 Register sp = rdx;
2298 Register wasm_runtime = r8;
2299
2300 Register memory_offset = r10;
2301 emitter::EmitLoadMemoryOffset(masm, memory_offset,
2302 MemOperand(code, kMemoryOffset));
2303
2304 Register memory_index_slot_offset = rax;
2305 emitter::EmitLoadSlotOffset(masm, memory_index_slot_offset,
2306 MemOperand(code, kIndexSlot));
2307
2308 Register memory_start_plus_offset = memory_offset;
2309 __ addq(memory_start_plus_offset,
2310 MemOperand(wasm_runtime,
2311 wasm::WasmInterpreterRuntime::memory_start_offset()));
2312
2313 Register memory_index = r9;
2314 __ movl(memory_index, Operand(sp, memory_index_slot_offset, times_4, 0));
2315
2316 Register result_slot_offset = r11;
2317 emitter::EmitLoadSlotOffset(masm, result_slot_offset,
2318 MemOperand(code, kResultSlot));
2319
2320 EmitLoadInstruction(masm, memory_start_plus_offset, memory_index, sp,
2321 result_slot_offset, float_type);
2322
2323 Register next_handler_id = r10;
2324 EmitLoadNextInstructionId(masm, next_handler_id, code, kNextHandlerId);
2325 __ addq(code, Immediate(kInstructionCodeLength));
2326
2327 Register instr_table = rax;
2328 __ movq(
2329 instr_table,
2330 MemOperand(wasm_runtime,
2331 wasm::WasmInterpreterRuntime::instruction_table_offset()));
2332
2333 Register next_handler_addr = rax;
2334 __ movq(next_handler_addr,
2335 MemOperand(instr_table, next_handler_id, times_8, 0));
2336 __ jmp(next_handler_addr);
2337 }
2338
2339 static void Generate_s2s_ILoadMem_LocalSet(MacroAssembler* masm,
2340 IntValueType value_type,
2341 IntMemoryType memory_type) {
2342 constexpr uint32_t kMemoryOffset = 0;
2343 constexpr uint32_t kIndexSlot = kMemoryOffset + sizeof(memory_offset32_t);
2344 constexpr uint32_t kSetSlot = kIndexSlot + sizeof(slot_offset_t);
2345 constexpr uint32_t kNextHandlerId = kSetSlot + sizeof(slot_offset_t);
2346 constexpr uint32_t kInstructionCodeLength =
2347 kNextHandlerId + sizeof(handler_id_t);
2348
2349 Register code = rcx;
2350 Register sp = rdx;
2351 Register wasm_runtime = r8;
2352
2353 Register memory_offset = r10;
2354 emitter::EmitLoadMemoryOffset(masm, memory_offset,
2355 MemOperand(code, kMemoryOffset));
2356
2357 Register memory_index_slot_offset = rax;
2358 emitter::EmitLoadSlotOffset(masm, memory_index_slot_offset,
2359 MemOperand(code, kIndexSlot));
2360
2361 Register memory_start_plus_offset = memory_offset;
2362 __ addq(memory_start_plus_offset,
2363 MemOperand(wasm_runtime,
2364 wasm::WasmInterpreterRuntime::memory_start_offset()));
2365
2366 Register memory_index = memory_index_slot_offset;
2367 __ movl(memory_index, Operand(sp, memory_index_slot_offset, times_4, 0));
2368
2369 Register set_slot_offset = r11;
2370 emitter::EmitLoadSlotOffset(masm, set_slot_offset,
2371 MemOperand(code, kSetSlot));
2372
2373 Register value = rax;
2374 EmitLoadInstruction(masm, value, memory_start_plus_offset, memory_index,
2375 value_type, memory_type);
2376
2377 WriteToSlot(masm, sp, set_slot_offset, value, value_type);
2378
2379 Register next_handler_id = r10;
2380 EmitLoadNextInstructionId(masm, next_handler_id, code, kNextHandlerId);
2381 __ addq(code, Immediate(kInstructionCodeLength));
2382
2383 Register instr_table = rax;
2384 __ movq(
2385 instr_table,
2386 MemOperand(wasm_runtime,
2387 wasm::WasmInterpreterRuntime::instruction_table_offset()));
2388
2389 Register next_handler_addr = rax;
2390 __ movq(next_handler_addr,
2391 MemOperand(instr_table, next_handler_id, times_8, 0));
2392 __ jmp(next_handler_addr);
2393 }
2394
2395 static void Generate_s2s_FLoadMem_LocalSet(MacroAssembler* masm,
2396 FloatType float_type) {
2397 constexpr uint32_t kMemoryOffset = 0;
2398 constexpr uint32_t kIndexSlot = kMemoryOffset + sizeof(memory_offset32_t);
2399 constexpr uint32_t kSetSlot = kIndexSlot + sizeof(slot_offset_t);
2400 constexpr uint32_t kNextHandlerId = kSetSlot + sizeof(slot_offset_t);
2401 constexpr uint32_t kInstructionCodeLength =
2402 kNextHandlerId + sizeof(handler_id_t);
2403
2404 Register code = rcx;
2405 Register sp = rdx;
2406 Register wasm_runtime = r8;
2407
2408 Register memory_offset = r10;
2409 emitter::EmitLoadMemoryOffset(masm, memory_offset,
2410 MemOperand(code, kMemoryOffset));
2411
2412 Register memory_index_slot_offset = rax;
2413 emitter::EmitLoadSlotOffset(masm, memory_index_slot_offset,
2414 MemOperand(code, kIndexSlot));
2415
2416 Register memory_start_plus_offset = memory_offset;
2417 __ addq(memory_start_plus_offset,
2418 MemOperand(wasm_runtime,
2419 wasm::WasmInterpreterRuntime::memory_start_offset()));
2420
2421 Register memory_index = memory_index_slot_offset;
2422 __ movl(memory_index, Operand(sp, memory_index_slot_offset, times_4, 0));
2423
2424 Register set_slot_offset = r11;
2425 emitter::EmitLoadSlotOffset(masm, set_slot_offset,
2426 MemOperand(code, kSetSlot));
2427
2428 EmitLoadInstruction(masm, memory_start_plus_offset, memory_index, sp,
2429 set_slot_offset, float_type);
2430
2431 Register next_handler_id = r10;
2432 EmitLoadNextInstructionId(masm, next_handler_id, code, kNextHandlerId);
2433 __ addq(code, Immediate(kInstructionCodeLength));
2434
2435 Register instr_table = rax;
2436 __ movq(
2437 instr_table,
2438 MemOperand(wasm_runtime,
2439 wasm::WasmInterpreterRuntime::instruction_table_offset()));
2440
2441 Register next_handler_addr = rax;
2442 __ movq(next_handler_addr,
2443 MemOperand(instr_table, next_handler_id, times_8, 0));
2444 __ jmp(next_handler_addr);
2445 }
2446
2447 static void Generate_r2s_IStoreMem(MacroAssembler* masm,
2448 IntValueType /*value_type*/,
2449 IntMemoryType memory_type) {
2450 constexpr uint32_t kMemoryOffset = 0;
2451 constexpr uint32_t kMemoryIndexSlot =
2452 kMemoryOffset + sizeof(memory_offset32_t);
2453 constexpr uint32_t kNextHandlerId =
2454 kMemoryIndexSlot + sizeof(slot_offset_t);
2455 constexpr uint32_t kInstructionCodeLength =
2456 kNextHandlerId + sizeof(handler_id_t);
2457
2458 Register code = rcx;
2459 Register sp = rdx;
2460 Register wasm_runtime = r8;
2461 Register value = r9;
2462
2463 Register memory_offset = r10;
2464 emitter::EmitLoadMemoryOffset(masm, memory_offset,
2465 MemOperand(code, kMemoryOffset));
2466
2467 Register memory_index_slot_offset = r11;
2468 emitter::EmitLoadSlotOffset(masm, memory_index_slot_offset,
2469 MemOperand(code, kMemoryIndexSlot));
2470
2471 Register memory_start_plus_offset = memory_offset;
2472 __ addq(memory_start_plus_offset,
2473 MemOperand(wasm_runtime,
2474 wasm::WasmInterpreterRuntime::memory_start_offset()));
2475
2476 Register memory_index = memory_index_slot_offset;
2477 __ movl(memory_index, Operand(sp, memory_index_slot_offset, times_4, 0));
2478
2479 EmitStoreInstruction(masm, value, memory_start_plus_offset, memory_index,
2480 memory_type);
2481
2482 Register next_handler_id = r10;
2483 EmitLoadNextInstructionId(masm, next_handler_id, code, kNextHandlerId);
2484 __ addq(code, Immediate(kInstructionCodeLength));
2485
2486 Register instr_table = rax;
2487 __ movq(
2488 instr_table,
2489 MemOperand(wasm_runtime,
2490 wasm::WasmInterpreterRuntime::instruction_table_offset()));
2491
2492 Register next_handler_addr = rax;
2493 __ movq(next_handler_addr,
2494 MemOperand(instr_table, next_handler_id, times_8, 0));
2495 __ jmp(next_handler_addr);
2496 }
2497
2498 static void Generate_r2s_FStoreMem(MacroAssembler* masm,
2499 FloatType float_type) {
2500 constexpr uint32_t kMemoryOffset = 0;
2501 constexpr uint32_t kMemoryIndexSlot =
2502 kMemoryOffset + sizeof(memory_offset32_t);
2503 constexpr uint32_t kNextHandlerId =
2504 kMemoryIndexSlot + sizeof(slot_offset_t);
2505 constexpr uint32_t kInstructionCodeLength =
2506 kNextHandlerId + sizeof(handler_id_t);
2507
2508 Register code = rcx;
2509 Register sp = rdx;
2510 Register wasm_runtime = r8;
2511
2512 XMMRegister value = xmm4;
2513 if (float_type == kFloat32) {
2514 __ cvtsd2ss(value, xmm4);
2515 }
2516
2517 Register memory_offset = r10;
2518 emitter::EmitLoadMemoryOffset(masm, memory_offset,
2519 MemOperand(code, kMemoryOffset));
2520
2521 Register memory_index_slot_offset = r11;
2522 emitter::EmitLoadSlotOffset(masm, memory_index_slot_offset,
2523 MemOperand(code, kMemoryIndexSlot));
2524
2525 Register memory_start_plus_offset = memory_offset;
2526 __ addq(memory_start_plus_offset,
2527 MemOperand(wasm_runtime,
2528 wasm::WasmInterpreterRuntime::memory_start_offset()));
2529
2530 Register memory_index = memory_index_slot_offset;
2531 __ movl(memory_index, Operand(sp, memory_index_slot_offset, times_4, 0));
2532
2533 EmitStoreInstruction(masm, value, memory_start_plus_offset, memory_index,
2534 float_type);
2535
2536 Register next_handler_id = r10;
2537 EmitLoadNextInstructionId(masm, next_handler_id, code, kNextHandlerId);
2538 __ addq(code, Immediate(kInstructionCodeLength));
2539
2540 Register instr_table = rax;
2541 __ movq(
2542 instr_table,
2543 MemOperand(wasm_runtime,
2544 wasm::WasmInterpreterRuntime::instruction_table_offset()));
2545
2546 Register next_handler_addr = rax;
2547 __ movq(next_handler_addr,
2548 MemOperand(instr_table, next_handler_id, times_8, 0));
2549 __ jmp(next_handler_addr);
2550 }
2551
2552 static void Generate_s2s_IStoreMem(MacroAssembler* masm,
2553 IntValueType /*value_type*/,
2554 IntMemoryType memory_type) {
2555 constexpr uint32_t kValueSlot = 0;
2556 constexpr uint32_t kMemoryOffset = kValueSlot + sizeof(slot_offset_t);
2557 constexpr uint32_t kMemoryIndexSlot =
2558 kMemoryOffset + sizeof(memory_offset32_t);
2559 constexpr uint32_t kNextHandlerId =
2560 kMemoryIndexSlot + sizeof(slot_offset_t);
2561 constexpr uint32_t kInstructionCodeLength =
2562 kNextHandlerId + sizeof(handler_id_t);
2563
2564 Register sp = rdx;
2565 Register code = rcx;
2566 Register wasm_runtime = r8;
2567
2568 Register value_slot_offset = rax;
2569 emitter::EmitLoadSlotOffset(masm, value_slot_offset,
2570 MemOperand(code, kValueSlot));
2571
2572 Register value = value_slot_offset;
2573 switch (memory_type) {
2574 case kInt64:
2575 __ movq(value, MemOperand(sp, value_slot_offset, times_4, 0));
2576 break;
2577 case kIntS32:
2578 __ movl(value, MemOperand(sp, value_slot_offset, times_4, 0));
2579 break;
2580 case kIntS16:
2581 __ movw(value, MemOperand(sp, value_slot_offset, times_4, 0));
2582 break;
2583 case kIntS8:
2584 __ movb(value, MemOperand(sp, value_slot_offset, times_4, 0));
2585 break;
2586 default:
2587 UNREACHABLE();
2588 }
2589
2590 Register memory_offset = r10;
2591 emitter::EmitLoadMemoryOffset(masm, memory_offset,
2592 MemOperand(code, kMemoryOffset));
2593
2594 Register memory_index_slot_offset = r11;
2595 emitter::EmitLoadSlotOffset(masm, memory_index_slot_offset,
2596 MemOperand(code, kMemoryIndexSlot));
2597
2598 Register memory_start_plus_offset = memory_offset;
2599 __ addq(memory_start_plus_offset,
2600 MemOperand(wasm_runtime,
2601 wasm::WasmInterpreterRuntime::memory_start_offset()));
2602
2603 Register memory_index = memory_index_slot_offset;
2604 __ movl(memory_index, Operand(sp, memory_index_slot_offset, times_4, 0));
2605
2606 EmitStoreInstruction(masm, value, memory_start_plus_offset, memory_index,
2607 memory_type);
2608
2609 Register next_handler_id = rax;
2610 EmitLoadNextInstructionId(masm, next_handler_id, code, kNextHandlerId);
2611 __ addq(code, Immediate(kInstructionCodeLength));
2612
2613 Register instr_table = r9;
2614 __ movq(
2615 instr_table,
2616 MemOperand(wasm_runtime,
2617 wasm::WasmInterpreterRuntime::instruction_table_offset()));
2618
2619 Register next_handler_addr = rax;
2620 __ movq(next_handler_addr,
2621 MemOperand(instr_table, next_handler_id, times_8, 0));
2622 __ jmp(next_handler_addr);
2623 }
2624
2625 static void Generate_s2s_FStoreMem(MacroAssembler* masm,
2626 FloatType float_type) {
2627 constexpr uint32_t kValueSlot = 0;
2628 constexpr uint32_t kMemoryOffset = kValueSlot + sizeof(slot_offset_t);
2629 constexpr uint32_t kMemoryIndexSlot =
2630 kMemoryOffset + sizeof(memory_offset32_t);
2631 constexpr uint32_t kNextHandlerId =
2632 kMemoryIndexSlot + sizeof(slot_offset_t);
2633 constexpr uint32_t kInstructionCodeLength =
2634 kNextHandlerId + sizeof(handler_id_t);
2635
2636 Register sp = rdx;
2637 Register code = rcx;
2638 Register wasm_runtime = r8;
2639
2640 Register value_slot_offset = rax;
2641 emitter::EmitLoadSlotOffset(masm, value_slot_offset,
2642 MemOperand(code, kValueSlot));
2643
2644 XMMRegister value = xmm0;
2645 switch (float_type) {
2646 case kFloat32:
2647 __ movss(value, MemOperand(sp, value_slot_offset, times_4, 0));
2648 break;
2649 case kFloat64:
2650 __ movsd(value, MemOperand(sp, value_slot_offset, times_4, 0));
2651 break;
2652 default:
2653 UNREACHABLE();
2654 }
2655
2656 Register memory_offset = r10;
2657 emitter::EmitLoadMemoryOffset(masm, memory_offset,
2658 MemOperand(code, kMemoryOffset));
2659
2660 Register memory_index_slot_offset = r11;
2661 emitter::EmitLoadSlotOffset(masm, memory_index_slot_offset,
2662 MemOperand(code, kMemoryIndexSlot));
2663
2664 Register memory_start_plus_offset = memory_offset;
2665 __ addq(memory_start_plus_offset,
2666 MemOperand(wasm_runtime,
2667 wasm::WasmInterpreterRuntime::memory_start_offset()));
2668
2669 Register memory_index = memory_index_slot_offset;
2670 __ movl(memory_index, Operand(sp, memory_index_slot_offset, times_4, 0));
2671
2672 EmitStoreInstruction(masm, value, memory_start_plus_offset, memory_index,
2673 float_type);
2674
2675 Register next_handler_id = r10;
2676 EmitLoadNextInstructionId(masm, next_handler_id, code, kNextHandlerId);
2677 __ addq(code, Immediate(kInstructionCodeLength));
2678
2679 Register instr_table = rax;
2680 __ movq(
2681 instr_table,
2682 MemOperand(wasm_runtime,
2683 wasm::WasmInterpreterRuntime::instruction_table_offset()));
2684
2685 Register next_handler_addr = rax;
2686 __ movq(next_handler_addr,
2687 MemOperand(instr_table, next_handler_id, times_8, 0));
2688 __ jmp(next_handler_addr);
2689 }
2690
2691 static void Generate_r2s_ILoadStoreMem(MacroAssembler* masm,
2692 IntValueType value_type,
2693 IntMemoryType memory_type) {
2694 constexpr uint32_t kLoadOffset = 0;
2695 constexpr uint32_t kStoreOffset = kLoadOffset + sizeof(memory_offset32_t);
2696 constexpr uint32_t kStoreIndexSlot =
2697 kStoreOffset + sizeof(memory_offset32_t);
2698 constexpr uint32_t kNextHandlerId = kStoreIndexSlot + sizeof(slot_offset_t);
2699 constexpr uint32_t kInstructionCodeLength =
2700 kNextHandlerId + sizeof(handler_id_t);
2701
2702 Register sp = rdx;
2703 Register code = rcx;
2704 Register wasm_runtime = r8;
2705 Register load_index = r9;
2706
2707 __ movl(load_index, load_index);
2708
2709 Register memory_start_plus_load_index = load_index;
2710 __ addq(memory_start_plus_load_index,
2711 MemOperand(wasm_runtime,
2712 wasm::WasmInterpreterRuntime::memory_start_offset()));
2713
2714 Register load_offset = rax;
2715 emitter::EmitLoadMemoryOffset(masm, load_offset,
2716 MemOperand(code, kLoadOffset));
2717
2718 Register value = r10;
2719 EmitLoadInstruction(masm, value, memory_start_plus_load_index, load_offset,
2720 value_type, memory_type);
2721
2722 Register store_index_slot_offset = r9;
2723 emitter::EmitLoadSlotOffset(masm, store_index_slot_offset,
2724 MemOperand(code, kStoreIndexSlot));
2725
2726 Register store_index = store_index_slot_offset;
2727 __ movl(store_index, MemOperand(sp, store_index_slot_offset, times_4, 0));
2728
2729 Register store_offset = r11;
2730 emitter::EmitLoadMemoryOffset(masm, store_offset,
2731 MemOperand(code, kStoreOffset));
2732
2733 Register memory_start_plus_store_index = store_index;
2734 __ addq(memory_start_plus_store_index,
2735 MemOperand(wasm_runtime,
2736 wasm::WasmInterpreterRuntime::memory_start_offset()));
2737
2738 EmitStoreInstruction(masm, value, memory_start_plus_store_index,
2739 store_offset, memory_type);
2740
2741 Register next_handler_id = rax;
2742 EmitLoadNextInstructionId(masm, next_handler_id, code, kNextHandlerId);
2743 __ addq(code, Immediate(kInstructionCodeLength));
2744
2745 Register instr_table = r9;
2746 __ movq(
2747 instr_table,
2748 MemOperand(wasm_runtime,
2749 wasm::WasmInterpreterRuntime::instruction_table_offset()));
2750
2751 Register next_handler_addr = rax;
2752 __ movq(next_handler_addr,
2753 MemOperand(instr_table, next_handler_id, times_8, 0));
2754 __ jmp(next_handler_addr);
2755 }
2756
2757 static void Generate_s2s_ILoadStoreMem(MacroAssembler* masm,
2758 IntValueType value_type,
2759 IntMemoryType memory_type) {
2760 constexpr uint32_t kLoadOffset = 0;
2761 constexpr uint32_t kLoadIndexSlot = kLoadOffset + sizeof(memory_offset32_t);
2762 constexpr uint32_t kStoreOffset = kLoadIndexSlot + sizeof(slot_offset_t);
2763 constexpr uint32_t kStoreIndexSlot =
2764 kStoreOffset + sizeof(memory_offset32_t);
2765 constexpr uint32_t kNextHandlerId = kStoreIndexSlot + sizeof(slot_offset_t);
2766 constexpr uint32_t kInstructionCodeLength =
2767 kNextHandlerId + sizeof(handler_id_t);
2768
2769 Register sp = rdx;
2770 Register code = rcx;
2771 Register wasm_runtime = r8;
2772
2773 Register load_index_slot_offset = r9;
2774 emitter::EmitLoadSlotOffset(masm, load_index_slot_offset,
2775 MemOperand(code, kLoadIndexSlot));
2776
2777 Register load_index = load_index_slot_offset;
2778 __ movl(load_index, Operand(sp, load_index_slot_offset, times_4, 0));
2779
2780 Register memory_start_plus_load_index = load_index;
2781 __ addq(memory_start_plus_load_index,
2782 MemOperand(wasm_runtime,
2783 wasm::WasmInterpreterRuntime::memory_start_offset()));
2784
2785 Register load_offset = rax;
2786 emitter::EmitLoadMemoryOffset(masm, load_offset,
2787 MemOperand(code, kLoadOffset));
2788
2789 Register value = r10;
2790 EmitLoadInstruction(masm, value, memory_start_plus_load_index, load_offset,
2791 value_type, memory_type);
2792
2793 Register store_index_slot_offset = r9;
2794 emitter::EmitLoadSlotOffset(masm, store_index_slot_offset,
2795 MemOperand(code, kStoreIndexSlot));
2796
2797 Register store_index = store_index_slot_offset;
2798 __ movl(store_index, MemOperand(sp, store_index_slot_offset, times_4, 0));
2799
2800 Register store_offset = r11;
2801 emitter::EmitLoadMemoryOffset(masm, store_offset,
2802 MemOperand(code, kStoreOffset));
2803
2804 Register memory_start_plus_store_index = store_index;
2805 __ addq(memory_start_plus_store_index,
2806 MemOperand(wasm_runtime,
2807 wasm::WasmInterpreterRuntime::memory_start_offset()));
2808
2809 EmitStoreInstruction(masm, value, memory_start_plus_store_index,
2810 store_offset, memory_type);
2811
2812 Register next_handler_id = rax;
2813 EmitLoadNextInstructionId(masm, next_handler_id, code, kNextHandlerId);
2814 __ addq(code, Immediate(kInstructionCodeLength));
2815
2816 Register instr_table = r9;
2817 __ movq(
2818 instr_table,
2819 MemOperand(wasm_runtime,
2820 wasm::WasmInterpreterRuntime::instruction_table_offset()));
2821
2822 Register next_handler_addr = rax;
2823 __ movq(next_handler_addr,
2824 MemOperand(instr_table, next_handler_id, times_8, 0));
2825 __ jmp(next_handler_addr);
2826 }
2827
2828 static void Generate_r2s_FLoadStoreMem(MacroAssembler* masm,
2829 FloatType float_type) {
2830 constexpr uint32_t kLoadOffset = 0;
2831 constexpr uint32_t kStoreOffset = kLoadOffset + sizeof(memory_offset32_t);
2832 constexpr uint32_t kStoreIndexSlot =
2833 kStoreOffset + sizeof(memory_offset32_t);
2834 constexpr uint32_t kNextHandlerId = kStoreIndexSlot + sizeof(slot_offset_t);
2835 constexpr uint32_t kInstructionCodeLength =
2836 kNextHandlerId + sizeof(handler_id_t);
2837
2838 Register sp = rdx;
2839 Register code = rcx;
2840 Register wasm_runtime = r8;
2841 Register load_index = r9;
2842
2843 __ movl(load_index, load_index);
2844
2845 Register memory_start_plus_load_index = load_index;
2846 __ addq(memory_start_plus_load_index,
2847 MemOperand(wasm_runtime,
2848 wasm::WasmInterpreterRuntime::memory_start_offset()));
2849
2850 Register load_offset = rax;
2851 emitter::EmitLoadMemoryOffset(masm, load_offset,
2852 MemOperand(code, kLoadOffset));
2853
2854 XMMRegister value = xmm0;
2855 switch (float_type) {
2856 case kFloat32:
2857 __ movss(value, Operand(memory_start_plus_load_index, load_offset,
2858 times_1, 0));
2859 break;
2860 case kFloat64:
2861 __ movsd(value, Operand(memory_start_plus_load_index, load_offset,
2862 times_1, 0));
2863 break;
2864 default:
2865 UNREACHABLE();
2866 }
2867
2868 Register store_index_slot_offset = r9;
2869 emitter::EmitLoadSlotOffset(masm, store_index_slot_offset,
2870 MemOperand(code, kStoreIndexSlot));
2871
2872 Register store_index = store_index_slot_offset;
2873 __ movl(store_index, MemOperand(sp, store_index_slot_offset, times_4, 0));
2874
2875 Register store_offset = r11;
2876 emitter::EmitLoadMemoryOffset(masm, store_offset,
2877 MemOperand(code, kStoreOffset));
2878
2879 Register memory_start_plus_store_index = store_index;
2880 __ addq(memory_start_plus_store_index,
2881 MemOperand(wasm_runtime,
2882 wasm::WasmInterpreterRuntime::memory_start_offset()));
2883
2884 EmitStoreInstruction(masm, value, memory_start_plus_store_index,
2885 store_offset, float_type);
2886
2887 Register next_handler_id = rax;
2888 EmitLoadNextInstructionId(masm, next_handler_id, code, kNextHandlerId);
2889 __ addq(code, Immediate(kInstructionCodeLength));
2890
2891 Register instr_table = r9;
2892 __ movq(
2893 instr_table,
2894 MemOperand(wasm_runtime,
2895 wasm::WasmInterpreterRuntime::instruction_table_offset()));
2896
2897 Register next_handler_addr = rax;
2898 __ movq(next_handler_addr,
2899 MemOperand(instr_table, next_handler_id, times_8, 0));
2900 __ jmp(next_handler_addr);
2901 }
2902
2903 static void Generate_s2s_FLoadStoreMem(MacroAssembler* masm,
2904 FloatType float_type) {
2905 constexpr uint32_t kLoadOffset = 0;
2906 constexpr uint32_t kLoadIndexSlot = kLoadOffset + sizeof(memory_offset32_t);
2907 constexpr uint32_t kStoreOffset = kLoadIndexSlot + sizeof(slot_offset_t);
2908 constexpr uint32_t kStoreIndexSlot =
2909 kStoreOffset + sizeof(memory_offset32_t);
2910 constexpr uint32_t kNextHandlerId = kStoreIndexSlot + sizeof(slot_offset_t);
2911 constexpr uint32_t kInstructionCodeLength =
2912 kNextHandlerId + sizeof(handler_id_t);
2913
2914 Register sp = rdx;
2915 Register code = rcx;
2916 Register wasm_runtime = r8;
2917
2918 Register load_index_slot_offset = r9;
2919 emitter::EmitLoadSlotOffset(masm, load_index_slot_offset,
2920 MemOperand(code, kLoadIndexSlot));
2921
2922 Register load_index = load_index_slot_offset;
2923 __ movl(load_index, Operand(sp, load_index_slot_offset, times_4, 0));
2924
2925 Register memory_start_plus_load_index = load_index;
2926 __ addq(memory_start_plus_load_index,
2927 MemOperand(wasm_runtime,
2928 wasm::WasmInterpreterRuntime::memory_start_offset()));
2929 Register load_offset = rax;
2930 emitter::EmitLoadMemoryOffset(masm, load_offset,
2931 MemOperand(code, kLoadOffset));
2932
2933 XMMRegister value = xmm0;
2934 switch (float_type) {
2935 case kFloat32:
2936 __ movss(value, Operand(memory_start_plus_load_index, load_offset,
2937 times_1, 0));
2938 break;
2939 case kFloat64:
2940 __ movsd(value, Operand(memory_start_plus_load_index, load_offset,
2941 times_1, 0));
2942 break;
2943 default:
2944 UNREACHABLE();
2945 }
2946
2947 Register store_index_slot_offset = r9;
2948 emitter::EmitLoadSlotOffset(masm, store_index_slot_offset,
2949 MemOperand(code, kStoreIndexSlot));
2950
2951 Register store_index = store_index_slot_offset;
2952 __ movl(store_index, MemOperand(sp, store_index_slot_offset, times_4, 0));
2953
2954 Register store_offset = r11;
2955 emitter::EmitLoadMemoryOffset(masm, store_offset,
2956 MemOperand(code, kStoreOffset));
2957
2958 Register memory_start_plus_store_index = store_index;
2959 __ addq(memory_start_plus_store_index,
2960 MemOperand(wasm_runtime,
2961 wasm::WasmInterpreterRuntime::memory_start_offset()));
2962
2963 EmitStoreInstruction(masm, value, memory_start_plus_store_index,
2964 store_offset, float_type);
2965
2966 Register next_handler_id = rax;
2967 EmitLoadNextInstructionId(masm, next_handler_id, code, kNextHandlerId);
2968 __ addq(code, Immediate(kInstructionCodeLength));
2969
2970 Register instr_table = r9;
2971 __ movq(
2972 instr_table,
2973 MemOperand(wasm_runtime,
2974 wasm::WasmInterpreterRuntime::instruction_table_offset()));
2975
2976 Register next_handler_addr = rax;
2977 __ movq(next_handler_addr,
2978 MemOperand(instr_table, next_handler_id, times_8, 0));
2979 __ jmp(next_handler_addr);
2980 }
2981}; // class WasmInterpreterHandlerBuiltins<Compressed>
2982
2983} // namespace
2984
2985#define FOREACH_INT_LOADSTORE_BUILTIN(V) \
2986 V(r2r_I32LoadMem8S, Generate_r2r_ILoadMem, kValueInt32, kIntS8) \
2987 V(r2r_I32LoadMem8U, Generate_r2r_ILoadMem, kValueInt32, kIntU8) \
2988 V(r2r_I32LoadMem16S, Generate_r2r_ILoadMem, kValueInt32, kIntS16) \
2989 V(r2r_I32LoadMem16U, Generate_r2r_ILoadMem, kValueInt32, kIntU16) \
2990 V(r2r_I64LoadMem8S, Generate_r2r_ILoadMem, kValueInt64, kIntS8) \
2991 V(r2r_I64LoadMem8U, Generate_r2r_ILoadMem, kValueInt64, kIntU8) \
2992 V(r2r_I64LoadMem16S, Generate_r2r_ILoadMem, kValueInt64, kIntS16) \
2993 V(r2r_I64LoadMem16U, Generate_r2r_ILoadMem, kValueInt64, kIntU16) \
2994 V(r2r_I64LoadMem32S, Generate_r2r_ILoadMem, kValueInt64, kIntS32) \
2995 V(r2r_I64LoadMem32U, Generate_r2r_ILoadMem, kValueInt64, kIntU32) \
2996 V(r2r_I32LoadMem, Generate_r2r_ILoadMem, kValueInt32, kIntS32) \
2997 V(r2r_I64LoadMem, Generate_r2r_ILoadMem, kValueInt64, kInt64) \
2998 \
2999 V(r2s_I32LoadMem8S, Generate_r2s_ILoadMem, kValueInt32, kIntS8) \
3000 V(r2s_I32LoadMem8U, Generate_r2s_ILoadMem, kValueInt32, kIntU8) \
3001 V(r2s_I32LoadMem16S, Generate_r2s_ILoadMem, kValueInt32, kIntS16) \
3002 V(r2s_I32LoadMem16U, Generate_r2s_ILoadMem, kValueInt32, kIntU16) \
3003 V(r2s_I64LoadMem8S, Generate_r2s_ILoadMem, kValueInt64, kIntS8) \
3004 V(r2s_I64LoadMem8U, Generate_r2s_ILoadMem, kValueInt64, kIntU8) \
3005 V(r2s_I64LoadMem16S, Generate_r2s_ILoadMem, kValueInt64, kIntS16) \
3006 V(r2s_I64LoadMem16U, Generate_r2s_ILoadMem, kValueInt64, kIntU16) \
3007 V(r2s_I64LoadMem32S, Generate_r2s_ILoadMem, kValueInt64, kIntS32) \
3008 V(r2s_I64LoadMem32U, Generate_r2s_ILoadMem, kValueInt64, kIntU32) \
3009 V(r2s_I32LoadMem, Generate_r2s_ILoadMem, kValueInt32, kIntS32) \
3010 V(r2s_I64LoadMem, Generate_r2s_ILoadMem, kValueInt64, kInt64) \
3011 \
3012 V(s2r_I32LoadMem8S, Generate_s2r_ILoadMem, kValueInt32, kIntS8) \
3013 V(s2r_I32LoadMem8U, Generate_s2r_ILoadMem, kValueInt32, kIntU8) \
3014 V(s2r_I32LoadMem16S, Generate_s2r_ILoadMem, kValueInt32, kIntS16) \
3015 V(s2r_I32LoadMem16U, Generate_s2r_ILoadMem, kValueInt32, kIntU16) \
3016 V(s2r_I64LoadMem8S, Generate_s2r_ILoadMem, kValueInt64, kIntS8) \
3017 V(s2r_I64LoadMem8U, Generate_s2r_ILoadMem, kValueInt64, kIntU8) \
3018 V(s2r_I64LoadMem16S, Generate_s2r_ILoadMem, kValueInt64, kIntS16) \
3019 V(s2r_I64LoadMem16U, Generate_s2r_ILoadMem, kValueInt64, kIntU16) \
3020 V(s2r_I64LoadMem32S, Generate_s2r_ILoadMem, kValueInt64, kIntS32) \
3021 V(s2r_I64LoadMem32U, Generate_s2r_ILoadMem, kValueInt64, kIntU32) \
3022 V(s2r_I32LoadMem, Generate_s2r_ILoadMem, kValueInt32, kIntS32) \
3023 V(s2r_I64LoadMem, Generate_s2r_ILoadMem, kValueInt64, kInt64) \
3024 \
3025 V(s2s_I32LoadMem8S, Generate_s2s_ILoadMem, kValueInt32, kIntS8) \
3026 V(s2s_I32LoadMem8U, Generate_s2s_ILoadMem, kValueInt32, kIntU8) \
3027 V(s2s_I32LoadMem16S, Generate_s2s_ILoadMem, kValueInt32, kIntS16) \
3028 V(s2s_I32LoadMem16U, Generate_s2s_ILoadMem, kValueInt32, kIntU16) \
3029 V(s2s_I64LoadMem8S, Generate_s2s_ILoadMem, kValueInt64, kIntS8) \
3030 V(s2s_I64LoadMem8U, Generate_s2s_ILoadMem, kValueInt64, kIntU8) \
3031 V(s2s_I64LoadMem16S, Generate_s2s_ILoadMem, kValueInt64, kIntS16) \
3032 V(s2s_I64LoadMem16U, Generate_s2s_ILoadMem, kValueInt64, kIntU16) \
3033 V(s2s_I64LoadMem32S, Generate_s2s_ILoadMem, kValueInt64, kIntS32) \
3034 V(s2s_I64LoadMem32U, Generate_s2s_ILoadMem, kValueInt64, kIntU32) \
3035 V(s2s_I32LoadMem, Generate_s2s_ILoadMem, kValueInt32, kIntS32) \
3036 V(s2s_I64LoadMem, Generate_s2s_ILoadMem, kValueInt64, kInt64) \
3037 \
3038 V(s2s_I32LoadMem8S_LocalSet, Generate_s2s_ILoadMem_LocalSet, kValueInt32, \
3039 kIntS8) \
3040 V(s2s_I32LoadMem8U_LocalSet, Generate_s2s_ILoadMem_LocalSet, kValueInt32, \
3041 kIntU8) \
3042 V(s2s_I32LoadMem16S_LocalSet, Generate_s2s_ILoadMem_LocalSet, kValueInt32, \
3043 kIntS16) \
3044 V(s2s_I32LoadMem16U_LocalSet, Generate_s2s_ILoadMem_LocalSet, kValueInt32, \
3045 kIntU16) \
3046 V(s2s_I64LoadMem8S_LocalSet, Generate_s2s_ILoadMem_LocalSet, kValueInt64, \
3047 kIntS8) \
3048 V(s2s_I64LoadMem8U_LocalSet, Generate_s2s_ILoadMem_LocalSet, kValueInt64, \
3049 kIntU8) \
3050 V(s2s_I64LoadMem16S_LocalSet, Generate_s2s_ILoadMem_LocalSet, kValueInt64, \
3051 kIntS16) \
3052 V(s2s_I64LoadMem16U_LocalSet, Generate_s2s_ILoadMem_LocalSet, kValueInt64, \
3053 kIntU16) \
3054 V(s2s_I64LoadMem32S_LocalSet, Generate_s2s_ILoadMem_LocalSet, kValueInt64, \
3055 kIntS32) \
3056 V(s2s_I64LoadMem32U_LocalSet, Generate_s2s_ILoadMem_LocalSet, kValueInt64, \
3057 kIntU32) \
3058 V(s2s_I32LoadMem_LocalSet, Generate_s2s_ILoadMem_LocalSet, kValueInt32, \
3059 kIntS32) \
3060 V(s2s_I64LoadMem_LocalSet, Generate_s2s_ILoadMem_LocalSet, kValueInt64, \
3061 kInt64) \
3062 \
3063 V(r2s_I32StoreMem8, Generate_r2s_IStoreMem, kValueInt32, kIntS8) \
3064 V(r2s_I32StoreMem16, Generate_r2s_IStoreMem, kValueInt32, kIntS16) \
3065 V(r2s_I64StoreMem8, Generate_r2s_IStoreMem, kValueInt64, kIntS8) \
3066 V(r2s_I64StoreMem16, Generate_r2s_IStoreMem, kValueInt64, kIntS16) \
3067 V(r2s_I64StoreMem32, Generate_r2s_IStoreMem, kValueInt64, kIntS32) \
3068 V(r2s_I32StoreMem, Generate_r2s_IStoreMem, kValueInt32, kIntS32) \
3069 V(r2s_I64StoreMem, Generate_r2s_IStoreMem, kValueInt64, kInt64) \
3070 \
3071 V(s2s_I32StoreMem8, Generate_s2s_IStoreMem, kValueInt32, kIntS8) \
3072 V(s2s_I32StoreMem16, Generate_s2s_IStoreMem, kValueInt32, kIntS16) \
3073 V(s2s_I64StoreMem8, Generate_s2s_IStoreMem, kValueInt64, kIntS8) \
3074 V(s2s_I64StoreMem16, Generate_s2s_IStoreMem, kValueInt64, kIntS16) \
3075 V(s2s_I64StoreMem32, Generate_s2s_IStoreMem, kValueInt64, kIntS32) \
3076 V(s2s_I32StoreMem, Generate_s2s_IStoreMem, kValueInt32, kIntS32) \
3077 V(s2s_I64StoreMem, Generate_s2s_IStoreMem, kValueInt64, kInt64) \
3078 \
3079 V(r2s_I32LoadStoreMem, Generate_r2s_ILoadStoreMem, kValueInt32, kIntS32) \
3080 V(r2s_I64LoadStoreMem, Generate_r2s_ILoadStoreMem, kValueInt64, kInt64) \
3081 \
3082 V(s2s_I32LoadStoreMem, Generate_s2s_ILoadStoreMem, kValueInt32, kIntS32) \
3083 V(s2s_I64LoadStoreMem, Generate_s2s_ILoadStoreMem, kValueInt64, kInt64)
3084
3085#define GENERATE_INT_LOADSTORE_BUILTIN(builtin_name, generator, value_type, \
3086 memory_type) \
3087 void Builtins::Generate_##builtin_name##_s(MacroAssembler* masm) { \
3088 return WasmInterpreterHandlerBuiltins<true>::generator(masm, value_type, \
3089 memory_type); \
3090 } \
3091 void Builtins::Generate_##builtin_name##_l(MacroAssembler* masm) { \
3092 return WasmInterpreterHandlerBuiltins<false>::generator(masm, value_type, \
3093 memory_type); \
3094 }
3095
3096FOREACH_INT_LOADSTORE_BUILTIN(GENERATE_INT_LOADSTORE_BUILTIN)
3097#undef FOREACH_INT_LOADSTORE_BUILTIN
3098
3099#define FOREACH_FLOAT_LOADMEM_BUILTIN(V) \
3100 V(r2r_F32LoadMem, Generate_r2r_FLoadMem, kFloat32) \
3101 V(r2r_F64LoadMem, Generate_r2r_FLoadMem, kFloat64) \
3102 V(r2s_F32LoadMem, Generate_r2s_FLoadMem, kFloat32) \
3103 V(r2s_F64LoadMem, Generate_r2s_FLoadMem, kFloat64) \
3104 V(s2r_F32LoadMem, Generate_s2r_FLoadMem, kFloat32) \
3105 V(s2r_F64LoadMem, Generate_s2r_FLoadMem, kFloat64) \
3106 V(s2s_F32LoadMem, Generate_s2s_FLoadMem, kFloat32) \
3107 V(s2s_F64LoadMem, Generate_s2s_FLoadMem, kFloat64) \
3108 V(s2s_F32LoadMem_LocalSet, Generate_s2s_FLoadMem_LocalSet, kFloat32) \
3109 V(s2s_F64LoadMem_LocalSet, Generate_s2s_FLoadMem_LocalSet, kFloat64) \
3110 V(r2s_F32StoreMem, Generate_r2s_FStoreMem, kFloat32) \
3111 V(r2s_F64StoreMem, Generate_r2s_FStoreMem, kFloat64) \
3112 V(s2s_F32StoreMem, Generate_s2s_FStoreMem, kFloat32) \
3113 V(s2s_F64StoreMem, Generate_s2s_FStoreMem, kFloat64) \
3114 V(r2s_F32LoadStoreMem, Generate_r2s_FLoadStoreMem, kFloat32) \
3115 V(r2s_F64LoadStoreMem, Generate_r2s_FLoadStoreMem, kFloat64) \
3116 V(s2s_F32LoadStoreMem, Generate_s2s_FLoadStoreMem, kFloat32) \
3117 V(s2s_F64LoadStoreMem, Generate_s2s_FLoadStoreMem, kFloat64)
3118
3119#define GENERATE_FLOAT_LOADMEM_BUILTIN(builtin_name, generator, value_type) \
3120 void Builtins::Generate_##builtin_name##_s(MacroAssembler* masm) { \
3121 return WasmInterpreterHandlerBuiltins<true>::generator(masm, value_type); \
3122 } \
3123 void Builtins::Generate_##builtin_name##_l(MacroAssembler* masm) { \
3124 return WasmInterpreterHandlerBuiltins<false>::generator(masm, value_type); \
3125 }
3126
3127FOREACH_FLOAT_LOADMEM_BUILTIN(GENERATE_FLOAT_LOADMEM_BUILTIN)
3128#undef FOREACH_FLOAT_LOADMEM_BUILTIN
3129
3130#endif // !V8_DRUMBRAKE_BOUNDS_CHECKS
3131
3132#endif // V8_ENABLE_WEBASSEMBLY
3133
3134#undef __
3135
3136} // namespace internal
3137} // namespace v8
BUILTIN_CODE
#define BUILTIN_CODE(isolate, name)
Definition builtins.h:45
v8::base::BitField::kMask
static constexpr U kMask
Definition bit-field.h:41
v8::internal::Builtins::RecordWrite
static constexpr Builtin RecordWrite(SaveFPRegsMode fp_mode)
Definition builtins-inl.h:17
v8::internal::Builtins::Call
static constexpr Builtin Call(ConvertReceiverMode=ConvertReceiverMode::kAny)
Definition builtins-inl.h:81
v8::internal::Isolate::thread_in_wasm_flag_address_offset
static constexpr uint32_t thread_in_wasm_flag_address_offset()
Definition isolate.h:1338
v8::internal::Signature< ValueType >::kReturnCountOffset
static constexpr size_t kReturnCountOffset
Definition signature.h:140
v8::internal::Signature< ValueType >::kRepsOffset
static constexpr size_t kRepsOffset
Definition signature.h:143
v8::internal::Signature< ValueType >::kParameterCountOffset
static constexpr size_t kParameterCountOffset
Definition signature.h:141
v8::internal::Smi::zero
static constexpr Tagged< Smi > zero()
Definition smi.h:99
v8::internal::wasm::ObjectAccess::SharedFunctionInfoOffsetInTaggedJSFunction
static constexpr int SharedFunctionInfoOffsetInTaggedJSFunction()
Definition object-access.h:63
v8::internal::wasm::ObjectAccess::ToTagged
static constexpr int ToTagged(int offset)
Definition object-access.h:24
receiver
TNode< Object > receiver
Definition js-call-reducer.cc:1498
result
ZoneVector< RpoNumber > & result
Definition jump-threading.cc:21
unibrow::int32_t
int int32_t
Definition unicode.cc:40
v8::base::ieee754::log2
double log2(double x)
Definition ieee754.cc:1973
v8::internal::compiler::turboshaft::float_type
typename detail::TypeForBits< Bits >::float_type float_type
Definition types.h:159
v8::internal::wasm::kWasmF32
constexpr IndependentValueType kWasmF32
Definition value-type.h:1229
v8::internal::wasm::wasm_runtime
uint32_t WasmInterpreterRuntime * wasm_runtime
Definition wasm-interpreter.h:673
v8::internal::wasm::kWasmI32
constexpr IndependentValueType kWasmI32
Definition value-type.h:1227
v8::internal::wasm::kInstructionTableMask
static constexpr uint32_t kInstructionTableMask
Definition wasm-interpreter.h:1216
v8::internal::wasm::kWasmS128
constexpr IndependentValueType kWasmS128
Definition value-type.h:1231
v8::internal::wasm::kWasmF64
constexpr IndependentValueType kWasmF64
Definition value-type.h:1230
v8::internal::wasm::kWasmI64
constexpr IndependentValueType kWasmI64
Definition value-type.h:1228
v8::internal::no_reg
constexpr Register no_reg
Definition register-arm.h:87
v8::internal::kRootRegister
constexpr Register kRootRegister
Definition register-arm.h:332
v8::internal::kPCOnStackSize
constexpr int kPCOnStackSize
Definition globals.h:412
v8::internal::FieldOperand
Operand FieldOperand(Register object, int offset)
Definition macro-assembler-ia32.h:703
v8::internal::kSystemPointerSizeLog2
constexpr int kSystemPointerSizeLog2
Definition globals.h:494
v8::internal::kFPOnStackSize
constexpr int kFPOnStackSize
Definition globals.h:413
v8::internal::FieldMemOperand
MemOperand FieldMemOperand(Register object, int offset)
Definition macro-assembler-arm.h:32
v8::internal::kSystemPointerSize
constexpr int kSystemPointerSize
Definition globals.h:410
v8::internal::kTaggedSizeLog2
constexpr int kTaggedSizeLog2
Definition globals.h:543
v8::internal::kScratchRegister
constexpr Register kScratchRegister
Definition register-x64.h:298
v8::internal::kWasmImplicitArgRegister
constexpr Register kWasmImplicitArgRegister
Definition register-arm.h:326
v8::internal::kHeapObjectTag
const int kHeapObjectTag
Definition v8-internal.h:72
v8::internal::SmiValuesAre32Bits
constexpr bool SmiValuesAre32Bits()
Definition v8-internal.h:209
v8::internal::r11
constexpr Register r11
Definition register-arm.h:331
v8::internal::kPtrComprCageBaseRegister
constexpr Register kPtrComprCageBaseRegister
Definition register-loong64.h:239
v8::internal::kCArgRegs
constexpr Register kCArgRegs[]
Definition register-arm.h:90
UNREACHABLE
#define UNREACHABLE()
Definition logging.h:67
OFFSET_OF_DATA_START
#define OFFSET_OF_DATA_START(Type)
Definition tagged-field.h:165