Skip to content

Don't pin objects permanently in ast.c #90

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jun 21, 2025
Merged

Don't pin objects permanently in ast.c #90

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1dc1459
optimize pinning in ast.c
d-netto Jun 19, 2025
2b68949
pass ast context to julia_to_scm
d-netto Jun 19, 2025
5180f3b
call pin function on AST ctx
d-netto Jun 19, 2025
367a3ca
push into pinned_objects
d-netto Jun 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 53 additions & 32 deletions src/ast.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,20 @@ typedef struct _jl_ast_context_t {
value_t ssavalue_sym;
value_t slot_sym;
jl_module_t *module; // context module for `current-julia-module-counter`
struct _jl_ast_context_t *next; // invasive list pointer for getting free contexts
arraylist_t pinned_objects;
} jl_ast_context_t;

// FIXME: Ugly hack to get a pointer to the pinned objects
arraylist_t *extract_pinned_objects_from_ast_ctx(void *ctx)
{
// This is used to extract pinned objects from the context
// for the purpose of pinning them in MMTk.
if (ctx == NULL)
return NULL;
jl_ast_context_t *jl_ctx = (jl_ast_context_t*)ctx;
return &jl_ctx->pinned_objects;
}

static jl_ast_context_t jl_ast_main_ctx;

#ifdef __clang_gcanalyzer__
Expand All @@ -153,7 +164,7 @@ struct macroctx_stack {
};

static jl_value_t *scm_to_julia(fl_context_t *fl_ctx, value_t e, jl_module_t *mod);
static value_t julia_to_scm(fl_context_t *fl_ctx, jl_value_t *v);
static value_t julia_to_scm(jl_ast_context_t *ctx, jl_value_t *v);
static jl_value_t *jl_expand_macros(jl_value_t *expr, jl_module_t *inmodule, struct macroctx_stack *macroctx, int onelevel, size_t world, int throw_load_error);

static jl_sym_t *scmsym_to_julia(fl_context_t *fl_ctx, value_t s)
Expand Down Expand Up @@ -279,27 +290,29 @@ static void jl_init_ast_ctx(jl_ast_context_t *ctx) JL_NOTSAFEPOINT
ctx->slot_sym = symbol(fl_ctx, "slot");
ctx->module = NULL;
set(symbol(fl_ctx, "*scopewarn-opt*"), fixnum(jl_options.warn_scope));
arraylist_new(&ctx->pinned_objects, 0);
}

// There should be no GC allocation while holding this lock
static uv_mutex_t flisp_lock;
static jl_ast_context_t *jl_ast_ctx_freed = NULL;
int flisp_initialized = 0;
arraylist_t jl_ast_ctx_freed;
arraylist_t jl_ast_ctx_used;

static jl_ast_context_t *jl_ast_ctx_enter(jl_module_t *m) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT
{
JL_SIGATOMIC_BEGIN();
uv_mutex_lock(&flisp_lock);
jl_ast_context_t *ctx = jl_ast_ctx_freed;
if (ctx != NULL) {
jl_ast_ctx_freed = ctx->next;
ctx->next = NULL;
}
jl_ast_context_t *ctx = (jl_ast_context_t*)arraylist_pop(&jl_ast_ctx_freed);
uv_mutex_unlock(&flisp_lock);
if (ctx == NULL) {
// Construct a new one if we can't find any
ctx = (jl_ast_context_t*)calloc(1, sizeof(jl_ast_context_t));
jl_init_ast_ctx(ctx);
}
uv_mutex_lock(&flisp_lock);
arraylist_push(&jl_ast_ctx_used, ctx);
uv_mutex_unlock(&flisp_lock);
ctx->module = m;
return ctx;
}
Expand All @@ -308,16 +321,20 @@ static void jl_ast_ctx_leave(jl_ast_context_t *ctx)
{
uv_mutex_lock(&flisp_lock);
ctx->module = NULL;
ctx->next = jl_ast_ctx_freed;
jl_ast_ctx_freed = ctx;
ctx->pinned_objects.len = 0; // clear pinned objects
arraylist_pop(&jl_ast_ctx_used);
arraylist_push(&jl_ast_ctx_freed, ctx);
uv_mutex_unlock(&flisp_lock);
JL_SIGATOMIC_END();
}

void jl_init_flisp(void)
{
if (jl_ast_ctx_freed)
if (flisp_initialized)
return;
flisp_initialized = 1;
arraylist_new(&jl_ast_ctx_freed, 0);
arraylist_new(&jl_ast_ctx_used, 0);
uv_mutex_init(&flisp_lock);
jl_init_ast_ctx(&jl_ast_main_ctx);
// To match the one in jl_ast_ctx_leave
Expand Down Expand Up @@ -674,37 +691,40 @@ static jl_value_t *scm_to_julia_(fl_context_t *fl_ctx, value_t e, jl_module_t *m
jl_error("malformed tree");
}

static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v, int check_valid);
static value_t julia_to_scm_(jl_ast_context_t *ctx, jl_value_t *v, int check_valid);

static value_t julia_to_scm(fl_context_t *fl_ctx, jl_value_t *v)
static value_t julia_to_scm(jl_ast_context_t *ctx, jl_value_t *v)
{
value_t temp;
// need try/catch to reset GC handle stack in case of error
fl_context_t *fl_ctx = &ctx->fl;
FL_TRY_EXTERN(fl_ctx) {
temp = julia_to_scm_(fl_ctx, v, 1);
temp = julia_to_scm_(ctx, v, 1);
}
FL_CATCH_EXTERN(fl_ctx) {
temp = fl_ctx->lasterror;
}
return temp;
}

static void array_to_list(fl_context_t *fl_ctx, jl_array_t *a, value_t *pv, int check_valid)
static void array_to_list(jl_ast_context_t *ctx, jl_array_t *a, value_t *pv, int check_valid)
{
value_t temp;
fl_context_t *fl_ctx = &ctx->fl;
for (long i = jl_array_nrows(a) - 1; i >= 0; i--) {
*pv = fl_cons(fl_ctx, fl_ctx->NIL, *pv);
temp = julia_to_scm_(fl_ctx, jl_array_ptr_ref(a, i), check_valid);
temp = julia_to_scm_(ctx, jl_array_ptr_ref(a, i), check_valid);
// note: must be separate statement
car_(*pv) = temp;
}
}

static value_t julia_to_list2(fl_context_t *fl_ctx, jl_value_t *a, jl_value_t *b, int check_valid)
static value_t julia_to_list2(jl_ast_context_t *ctx, jl_value_t *a, jl_value_t *b, int check_valid)
{
value_t sa = julia_to_scm_(fl_ctx, a, check_valid);
fl_context_t *fl_ctx = &ctx->fl;
value_t sa = julia_to_scm_(ctx, a, check_valid);
fl_gc_handle(fl_ctx, &sa);
value_t sb = julia_to_scm_(fl_ctx, b, check_valid);
value_t sb = julia_to_scm_(ctx, b, check_valid);
value_t l = fl_list2(fl_ctx, sa, sb);
fl_free_gc_handles(fl_ctx, 1);
return l;
Expand Down Expand Up @@ -778,12 +798,13 @@ static value_t julia_to_list2_noalloc(fl_context_t *fl_ctx, jl_value_t *a, jl_va
return l;
}

static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v, int check_valid)
static value_t julia_to_scm_(jl_ast_context_t *ctx, jl_value_t *v, int check_valid)
{
// The following code will take internal pointers to v's fields. We need to make sure
// that v will not be moved by GC.
OBJ_PIN(v);
arraylist_push(&ctx->pinned_objects, v);
value_t retval;
fl_context_t *fl_ctx = &ctx->fl;
if (julia_to_scm_noalloc1(fl_ctx, v, &retval))
return retval;
if (jl_is_expr(v)) {
Expand All @@ -792,12 +813,12 @@ static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v, int check_vali
fl_gc_handle(fl_ctx, &args);
if (jl_expr_nargs(ex) > 520000 && ex->head != jl_block_sym)
lerror(fl_ctx, symbol(fl_ctx, "error"), "expression too large");
array_to_list(fl_ctx, ex->args, &args, check_valid);
value_t hd = julia_to_scm_(fl_ctx, (jl_value_t*)ex->head, check_valid);
array_to_list(ctx, ex->args, &args, check_valid);
value_t hd = julia_to_scm_(ctx, (jl_value_t*)ex->head, check_valid);
if (ex->head == jl_lambda_sym && jl_expr_nargs(ex)>0 && jl_is_array(jl_exprarg(ex,0))) {
value_t llist = fl_ctx->NIL;
fl_gc_handle(fl_ctx, &llist);
array_to_list(fl_ctx, (jl_array_t*)jl_exprarg(ex,0), &llist, check_valid);
array_to_list(ctx, (jl_array_t*)jl_exprarg(ex,0), &llist, check_valid);
car_(args) = llist;
fl_free_gc_handles(fl_ctx, 1);
}
Expand All @@ -813,26 +834,26 @@ static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v, int check_vali
jl_value_t *line = jl_fieldref(v,0);
value_t args = julia_to_list2_noalloc(fl_ctx, line, file, check_valid);
fl_gc_handle(fl_ctx, &args);
value_t hd = julia_to_scm_(fl_ctx, (jl_value_t*)jl_line_sym, check_valid);
value_t hd = julia_to_scm_(ctx, (jl_value_t*)jl_line_sym, check_valid);
value_t scmv = fl_cons(fl_ctx, hd, args);
fl_free_gc_handles(fl_ctx, 1);
return scmv;
}
if (jl_typetagis(v, jl_gotonode_type))
return julia_to_list2_noalloc(fl_ctx, (jl_value_t*)jl_goto_sym, jl_fieldref(v,0), check_valid);
if (jl_typetagis(v, jl_quotenode_type))
return julia_to_list2(fl_ctx, (jl_value_t*)jl_inert_sym, jl_fieldref_noalloc(v,0), 0);
return julia_to_list2(ctx, (jl_value_t*)jl_inert_sym, jl_fieldref_noalloc(v,0), 0);
if (jl_typetagis(v, jl_newvarnode_type))
return julia_to_list2_noalloc(fl_ctx, (jl_value_t*)jl_newvar_sym, jl_fieldref(v,0), check_valid);
if (jl_typetagis(v, jl_globalref_type)) {
jl_module_t *m = jl_globalref_mod(v);
jl_sym_t *sym = jl_globalref_name(v);
if (m == jl_core_module)
return julia_to_list2(fl_ctx, (jl_value_t*)jl_core_sym,
return julia_to_list2(ctx, (jl_value_t*)jl_core_sym,
(jl_value_t*)sym, check_valid);
value_t args = julia_to_list2(fl_ctx, (jl_value_t*)m, (jl_value_t*)sym, check_valid);
value_t args = julia_to_list2(ctx, (jl_value_t*)m, (jl_value_t*)sym, check_valid);
fl_gc_handle(fl_ctx, &args);
value_t hd = julia_to_scm_(fl_ctx, (jl_value_t*)jl_globalref_sym, check_valid);
value_t hd = julia_to_scm_(ctx, (jl_value_t*)jl_globalref_sym, check_valid);
value_t scmv = fl_cons(fl_ctx, hd, args);
fl_free_gc_handles(fl_ctx, 1);
return scmv;
Expand Down Expand Up @@ -900,8 +921,8 @@ JL_DLLEXPORT jl_value_t *jl_fl_parse(const char *text, size_t text_len,
static jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr, jl_module_t *inmodule)
{
jl_ast_context_t *ctx = jl_ast_ctx_enter(inmodule);
value_t arg = julia_to_scm(ctx, expr);
fl_context_t *fl_ctx = &ctx->fl;
value_t arg = julia_to_scm(fl_ctx, expr);
value_t e = fl_applyn(fl_ctx, 1, symbol_value(symbol(fl_ctx, funcname)), arg);
jl_value_t *result = scm_to_julia(fl_ctx, e, inmodule);
JL_GC_PUSH1(&result);
Expand All @@ -914,8 +935,8 @@ jl_value_t *jl_call_scm_on_ast_and_loc(const char *funcname, jl_value_t *expr,
jl_module_t *inmodule, const char *file, int line)
{
jl_ast_context_t *ctx = jl_ast_ctx_enter(inmodule);
value_t arg = julia_to_scm(ctx, expr);
fl_context_t *fl_ctx = &ctx->fl;
value_t arg = julia_to_scm(fl_ctx, expr);
value_t e = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, funcname)), arg,
symbol(fl_ctx, file), fixnum(line));
jl_value_t *result = scm_to_julia(fl_ctx, e, inmodule);
Expand Down Expand Up @@ -1316,8 +1337,8 @@ JL_DLLEXPORT jl_value_t *jl_expand_with_loc_warn(jl_value_t *expr, jl_module_t *
expr = jl_copy_ast(expr);
expr = jl_expand_macros(expr, inmodule, NULL, 0, ~(size_t)0, 1);
jl_ast_context_t *ctx = jl_ast_ctx_enter(inmodule);
value_t arg = julia_to_scm(ctx, expr);
fl_context_t *fl_ctx = &ctx->fl;
value_t arg = julia_to_scm(fl_ctx, expr);
value_t e = fl_applyn(fl_ctx, 4, symbol_value(symbol(fl_ctx, "jl-expand-to-thunk-warn")), arg,
symbol(fl_ctx, file), fixnum(line), fl_ctx->F);
expr = scm_to_julia(fl_ctx, e, inmodule);
Expand Down
19 changes: 15 additions & 4 deletions src/gc-mmtk.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ JL_DLLEXPORT void jl_gc_collect(jl_gc_collection_t collection) {
// print_fragmentation();
}

void gc_pin_objects_from_inference_engine(arraylist_t *objects_pinned_by_call)
void gc_pin_objects_from_compiler_frontend(arraylist_t *objects_pinned_by_call)
{
for (size_t i = 0; i < gc_pinned_objects.len; i++) {
void *obj = gc_pinned_objects.items[i];
Expand All @@ -263,9 +263,20 @@ void gc_pin_objects_from_inference_engine(arraylist_t *objects_pinned_by_call)
arraylist_push(objects_pinned_by_call, obj);
}
}
for (size_t i = 0; i < jl_ast_ctx_used.len; i++) {
void *ctx = jl_ast_ctx_used.items[i];
arraylist_t *pinned_objects = extract_pinned_objects_from_ast_ctx(ctx);
for (size_t j = 0; j < pinned_objects->len; j++) {
void *obj = pinned_objects->items[j];
unsigned char got_pinned = mmtk_pin_object(obj);
if (got_pinned) {
arraylist_push(objects_pinned_by_call, obj);
}
}
}
}

void gc_unpin_objects_from_inference_engine(arraylist_t *objects_pinned_by_call)
void gc_unpin_objects_from_compiler_frontend(arraylist_t *objects_pinned_by_call)
{
for (size_t i = 0; i < objects_pinned_by_call->len; i++) {
void *obj = objects_pinned_by_call->items[i];
Expand Down Expand Up @@ -337,9 +348,9 @@ JL_DLLEXPORT void jl_gc_prepare_to_collect(void)
#ifndef __clang_gcanalyzer__
arraylist_t objects_pinned_by_call;
arraylist_new(&objects_pinned_by_call, 0);
gc_pin_objects_from_inference_engine(&objects_pinned_by_call);
gc_pin_objects_from_compiler_frontend(&objects_pinned_by_call);
mmtk_block_thread_for_gc();
gc_unpin_objects_from_inference_engine(&objects_pinned_by_call);
gc_unpin_objects_from_compiler_frontend(&objects_pinned_by_call);
arraylist_free(&objects_pinned_by_call);
#endif
JL_UNLOCK_NOGC(&finalizers_lock);
Expand Down
2 changes: 2 additions & 0 deletions src/julia.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2439,6 +2439,8 @@ JL_DLLEXPORT void jl_register_newmeth_tracer(void (*callback)(jl_method_t *trace

// AST access
JL_DLLEXPORT jl_value_t *jl_copy_ast(jl_value_t *expr JL_MAYBE_UNROOTED);
arraylist_t *extract_pinned_objects_from_ast_ctx(void *ctx);
extern arraylist_t jl_ast_ctx_used;

// IR representation
JL_DLLEXPORT jl_value_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code);
Expand Down