Skip to content

Wasm jiterpreter cleanup and bug fixes pt. 3 #78782

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Dec 2, 2022
Merged

Wasm jiterpreter cleanup and bug fixes pt. 3 #78782

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
08d0aa0
Unroll memmove like memset
kg Nov 23, 2022
f1655f2
Add runtime option that causes interpreter and jiterpreter tiering me…
kg Nov 23, 2022
17ae775
Implement floating point relops in WASM using a C function to preserv…
kg Nov 23, 2022
c12d76c
Revert config changes
kg Nov 23, 2022
41db270
Whitespace cleanup
kg Nov 23, 2022
8113ec4
Repair merge damage
kg Nov 23, 2022
d69318d
Hoist unordered check
kg Nov 23, 2022
d4dc093
Make interp_exec_method static again, because that appears to improve…
kg Nov 24, 2022
138070d
Remove tier_instantly (not necessary) and address PR feedback
kg Nov 29, 2022
05d90cb
Remove header change
kg Nov 29, 2022
d6ff4e5
remove ts side tier instantly
kg Nov 29, 2022
d29e7ce
Enable jiterpreter if tiering is disabled, update stats
kg Dec 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions src/mono/mono/mini/interp/interp-internals.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -301,9 +301,6 @@ mono_interp_error_cleanup (MonoError *error);
gboolean
mono_interp_is_method_multicastdelegate_invoke (MonoMethod *method);

MONO_NEVER_INLINE void
mono_interp_exec_method (InterpFrame *frame, ThreadContext *context, FrameClauseArgs *clause_args);

#if HOST_BROWSER

gboolean
Expand Down
81 changes: 65 additions & 16 deletions src/mono/mono/mini/interp/interp.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,9 @@ struct FrameClauseArgs {
gboolean run_until_end;
};

static MONO_NEVER_INLINE void
mono_interp_exec_method (InterpFrame *frame, ThreadContext *context, FrameClauseArgs *clause_args);

/*
* This code synchronizes with interp_mark_stack () using compiler memory barriers.
*/
Expand Down Expand Up @@ -3698,7 +3701,7 @@ max_d (double lhs, double rhs)
* to return error information.
* FRAME is only valid until the next call to alloc_frame ().
*/
MONO_NEVER_INLINE void
static MONO_NEVER_INLINE void
mono_interp_exec_method (InterpFrame *frame, ThreadContext *context, FrameClauseArgs *clause_args)
{
InterpMethod *cmethod;
Expand Down Expand Up @@ -3797,6 +3800,11 @@ mono_interp_exec_method (InterpFrame *frame, ThreadContext *context, FrameClause
MINT_IN_CASE(MINT_DEF)
MINT_IN_CASE(MINT_DUMMY_USE)
MINT_IN_CASE(MINT_TIER_PATCHPOINT_DATA)
#ifndef HOST_BROWSER
MINT_IN_CASE(MINT_TIER_NOP_JITERPRETER)
MINT_IN_CASE(MINT_TIER_PREPARE_JITERPRETER)
MINT_IN_CASE(MINT_TIER_ENTER_JITERPRETER)
#endif
g_assert_not_reached ();
MINT_IN_BREAK;
MINT_IN_CASE(MINT_BREAK)
Expand Down Expand Up @@ -7518,6 +7526,7 @@ MINT_IN_CASE(MINT_BRTRUE_I8_SP) ZEROP_SP(gint64, !=); MINT_IN_BREAK;
ip += 5;
MINT_IN_BREAK;
}

#ifdef HOST_BROWSER
MINT_IN_CASE(MINT_TIER_NOP_JITERPRETER) {
ip += 3;
Expand Down Expand Up @@ -7608,21 +7617,6 @@ MINT_IN_CASE(MINT_BRTRUE_I8_SP) ZEROP_SP(gint64, !=); MINT_IN_BREAK;
ip = (guint16*) (((guint8*)ip) + offset);
MINT_IN_BREAK;
}
#else
MINT_IN_CASE(MINT_TIER_NOP_JITERPRETER) {
g_assert_not_reached ();
MINT_IN_BREAK;
}

MINT_IN_CASE(MINT_TIER_PREPARE_JITERPRETER) {
g_assert_not_reached ();
MINT_IN_BREAK;
}

MINT_IN_CASE(MINT_TIER_ENTER_JITERPRETER) {
g_assert_not_reached ();
MINT_IN_BREAK;
}
#endif

#if !USE_COMPUTED_GOTO
Expand Down Expand Up @@ -8611,4 +8605,59 @@ mono_interp_is_method_multicastdelegate_invoke (MonoMethod *method)
{
return is_method_multicastdelegate_invoke (method);
}

// after interp_entry_prologue the wrapper will set up all the argument values
// in the correct place and compute the stack offset, then it passes that in to this
// function in order to actually enter the interpreter and process the return value
EMSCRIPTEN_KEEPALIVE void
mono_jiterp_interp_entry (JiterpEntryData *_data, stackval *sp_args, void *res)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another option here would have been to make interp_exec_method static again as it was before the original jiterpreter commit and then have a global mono_interp_exec_method that calls it.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you want me to change it? I originally had a wrapper like you describe, but the performance impact was measurable so that's how we ended up here. I might be able to get it to inline though, I don't think I spent much time trying when I was first prototyping.

{
JiterpEntryDataHeader header;
MonoType *type;

// Copy the scratch buffer into a local variable. This is necessary for us to be
// reentrant-safe because mono_interp_exec_method could end up hitting the trampoline
// again
g_assert(_data);
header = _data->header;

g_assert(header.rmethod);
g_assert(header.rmethod->method);
g_assert(sp_args);

stackval *sp = (stackval*)header.context->stack_pointer;

InterpFrame frame = {0};
frame.imethod = header.rmethod;
frame.stack = sp;
frame.retval = sp;

header.context->stack_pointer = (guchar*)sp_args;
g_assert ((guchar*)sp_args < header.context->stack_end);

MONO_ENTER_GC_UNSAFE;
mono_interp_exec_method (&frame, header.context, NULL);
MONO_EXIT_GC_UNSAFE;

header.context->stack_pointer = (guchar*)sp;

if (header.rmethod->needs_thread_attach)
mono_threads_detach_coop (header.orig_domain, &header.attach_cookie);

mono_jiterp_check_pending_unwind (header.context);

if (mono_llvm_only) {
if (header.context->has_resume_state)
/* The exception will be handled in a frame above us */
mono_llvm_cpp_throw_exception ();
} else {
g_assert (!header.context->has_resume_state);
}

// The return value is at the bottom of the stack, after the locals space
type = header.rmethod->rtype;
if (type->type != MONO_TYPE_VOID)
mono_jiterp_stackval_to_data (type, frame.stack, res);
}

#endif
120 changes: 38 additions & 82 deletions src/mono/mono/mini/interp/jiterpreter.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ void jiterp_preserve_module (void);
#include "interp-intrins.h"
#include "tiering.h"

#include <mono/utils/mono-math.h>
#include <mono/mini/mini.h>
#include <mono/mini/mini-runtime.h>
#include <mono/mini/aot-runtime.h>
Expand Down Expand Up @@ -437,6 +438,43 @@ mono_jiterp_conv_ovf (void *dest, void *src, int opcode) {
return 0;
}

#define JITERP_RELOP(opcode, type, op, noorder) \
case opcode: \
{ \
if (is_unordered) \
return noorder; \
else \
return ((type)lhs op (type)rhs); \
}

EMSCRIPTEN_KEEPALIVE int
mono_jiterp_relop_fp (double lhs, double rhs, int opcode) {
gboolean is_unordered = mono_isunordered (lhs, rhs);
switch (opcode) {
JITERP_RELOP(MINT_CEQ_R4, float, ==, 0);
JITERP_RELOP(MINT_CEQ_R8, double, ==, 0);
JITERP_RELOP(MINT_CNE_R4, float, !=, 1);
JITERP_RELOP(MINT_CNE_R8, double, !=, 1);
JITERP_RELOP(MINT_CGT_R4, float, >, 0);
JITERP_RELOP(MINT_CGT_R8, double, >, 0);
JITERP_RELOP(MINT_CGE_R4, float, >=, 0);
JITERP_RELOP(MINT_CGE_R8, double, >=, 0);
JITERP_RELOP(MINT_CGT_UN_R4, float, >, 1);
JITERP_RELOP(MINT_CGT_UN_R8, double, >, 1);
JITERP_RELOP(MINT_CLT_R4, float, <, 0);
JITERP_RELOP(MINT_CLT_R8, double, <, 0);
JITERP_RELOP(MINT_CLT_UN_R4, float, <, 1);
JITERP_RELOP(MINT_CLT_UN_R8, double, <, 1);
JITERP_RELOP(MINT_CLE_R4, float, <=, 0);
JITERP_RELOP(MINT_CLE_R8, double, <=, 0);

default:
g_assert_not_reached();
}
}

#undef JITERP_RELOP

// we use these helpers at JIT time to figure out where to do memory loads and stores
EMSCRIPTEN_KEEPALIVE size_t
mono_jiterp_get_offset_of_vtable_initialized_flag () {
Expand Down Expand Up @@ -518,34 +556,6 @@ mono_jiterp_adjust_abort_count (MintOpcode opcode, gint32 delta) {
return jiterpreter_abort_counts[opcode];
}

typedef struct {
InterpMethod *rmethod;
ThreadContext *context;
gpointer orig_domain;
gpointer attach_cookie;
} JiterpEntryDataHeader;

// we optimize delegate calls by attempting to cache the delegate invoke
// target - this will improve performance when the same delegate is invoked
// repeatedly inside a loop
typedef struct {
MonoDelegate *delegate_invoke_is_for;
MonoMethod *delegate_invoke;
InterpMethod *delegate_invoke_rmethod;
} JiterpEntryDataCache;

// jitted interp_entry wrappers use custom tracking data structures
// that are allocated in the heap, one per wrapper
// FIXME: For thread safety we need to make these thread-local or stack-allocated
// Note that if we stack allocate these the cache will need to move somewhere else
typedef struct {
// We split the cache out from the important data so that when
// jiterp_interp_entry copies the important data it doesn't have
// to also copy the cache. This reduces overhead slightly
JiterpEntryDataHeader header;
JiterpEntryDataCache cache;
} JiterpEntryData;

// at the start of a jitted interp_entry wrapper, this is called to perform initial setup
// like resolving the target for delegates and setting up the thread context
// inlining this into the wrappers would make them unnecessarily big and complex
Expand Down Expand Up @@ -604,60 +614,6 @@ mono_jiterp_interp_entry_prologue (JiterpEntryData *data, void *this_arg)
return sp_args;
}

// after interp_entry_prologue the wrapper will set up all the argument values
// in the correct place and compute the stack offset, then it passes that in to this
// function in order to actually enter the interpreter and process the return value
EMSCRIPTEN_KEEPALIVE void
mono_jiterp_interp_entry (JiterpEntryData *_data, stackval *sp_args, void *res)
{
JiterpEntryDataHeader header;
MonoType *type;

// Copy the scratch buffer into a local variable. This is necessary for us to be
// reentrant-safe because mono_interp_exec_method could end up hitting the trampoline
// again
jiterp_assert(_data);
header = _data->header;

jiterp_assert(header.rmethod);
jiterp_assert(header.rmethod->method);
jiterp_assert(sp_args);

stackval *sp = (stackval*)header.context->stack_pointer;

InterpFrame frame = {0};
frame.imethod = header.rmethod;
frame.stack = sp;
frame.retval = sp;

header.context->stack_pointer = (guchar*)sp_args;
g_assert ((guchar*)sp_args < header.context->stack_end);

MONO_ENTER_GC_UNSAFE;
mono_interp_exec_method (&frame, header.context, NULL);
MONO_EXIT_GC_UNSAFE;

header.context->stack_pointer = (guchar*)sp;

if (header.rmethod->needs_thread_attach)
mono_threads_detach_coop (header.orig_domain, &header.attach_cookie);

mono_jiterp_check_pending_unwind (header.context);

if (mono_llvm_only) {
if (header.context->has_resume_state)
/* The exception will be handled in a frame above us */
mono_llvm_cpp_throw_exception ();
} else {
g_assert (!header.context->has_resume_state);
}

// The return value is at the bottom of the stack, after the locals space
type = header.rmethod->rtype;
if (type->type != MONO_TYPE_VOID)
mono_jiterp_stackval_to_data (type, frame.stack, res);
}

// should_abort_trace returns one of these codes depending on the opcode and current state
#define TRACE_IGNORE -1
#define TRACE_CONTINUE 0
Expand Down
35 changes: 35 additions & 0 deletions src/mono/mono/mini/interp/jiterpreter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,41 @@ mono_jiterp_do_jit_call_indirect (
gpointer cb, gpointer arg, gboolean *out_thrown
);

#ifdef __MONO_MINI_INTERPRETER_INTERNALS_H__

typedef struct {
InterpMethod *rmethod;
ThreadContext *context;
gpointer orig_domain;
gpointer attach_cookie;
} JiterpEntryDataHeader;

// we optimize delegate calls by attempting to cache the delegate invoke
// target - this will improve performance when the same delegate is invoked
// repeatedly inside a loop
typedef struct {
MonoDelegate *delegate_invoke_is_for;
MonoMethod *delegate_invoke;
InterpMethod *delegate_invoke_rmethod;
} JiterpEntryDataCache;

// jitted interp_entry wrappers use custom tracking data structures
// that are allocated in the heap, one per wrapper
// FIXME: For thread safety we need to make these thread-local or stack-allocated
// Note that if we stack allocate these the cache will need to move somewhere else
typedef struct {
// We split the cache out from the important data so that when
// jiterp_interp_entry copies the important data it doesn't have
// to also copy the cache. This reduces overhead slightly
JiterpEntryDataHeader header;
JiterpEntryDataCache cache;
} JiterpEntryData;

void
mono_jiterp_interp_entry (JiterpEntryData *_data, stackval *sp_args, void *res);

#endif // __MONO_MINI_INTERPRETER_INTERNALS_H__

extern WasmDoJitCall jiterpreter_do_jit_call;

#endif // HOST_BROWSER
Expand Down
3 changes: 1 addition & 2 deletions src/mono/mono/mini/interp/transform.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10071,8 +10071,7 @@ generate (MonoMethod *method, MonoMethodHeader *header, InterpMethod *rtm, MonoG
interp_optimize_code (td);
interp_alloc_offsets (td);
#if HOST_BROWSER
if (mono_interp_tiering_enabled ())
jiterp_insert_entry_points (td);
jiterp_insert_entry_points (td);
#endif
}

Expand Down
Loading