diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl index 6ae9d9a1547a3..4b8f37f6229de 100644 --- a/base/compiler/typeinfer.jl +++ b/base/compiler/typeinfer.jl @@ -1,5 +1,9 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +# Tracking of newly-inferred MethodInstances during precompilation +const track_newly_inferred = RefValue{Bool}(false) +const newly_inferred = MethodInstance[] + # build (and start inferring) the inference frame for the top-level MethodInstance function typeinf(interp::AbstractInterpreter, result::InferenceResult, cache::Symbol) frame = InferenceState(result, cache, interp) @@ -336,7 +340,7 @@ function maybe_compress_codeinfo(interp::AbstractInterpreter, linfo::MethodInsta cache_the_tree = true end if cache_the_tree - if may_compress(interp) + if JLOptions().incremental == Int8(0) && may_compress(interp) nslots = length(ci.slotflags) resize!(ci.slottypes::Vector{Any}, nslots) resize!(ci.slotnames, nslots) @@ -387,6 +391,12 @@ function cache_result!(interp::AbstractInterpreter, result::InferenceResult) if !already_inferred inferred_result = transform_result_for_cache(interp, linfo, valid_worlds, result.src) code_cache(interp)[linfo] = CodeInstance(result, inferred_result, valid_worlds) + if track_newly_inferred[] + m = linfo.def + if isa(m, Method) + m.module != Core && push!(newly_inferred, linfo) + end + end end unlock_mi_inference(interp, linfo) nothing diff --git a/base/loading.jl b/base/loading.jl index f303a4501f06c..5990484220809 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1369,6 +1369,7 @@ function include_package_for_output(pkg::PkgId, input::String, depot_path::Vecto task_local_storage()[:SOURCE_PATH] = source end + Core.Compiler.track_newly_inferred.x = true try Base.include(Base.__toplevel__, input) catch ex @@ -1376,6 +1377,8 @@ function include_package_for_output(pkg::PkgId, input::String, depot_path::Vecto @debug "Aborting `create_expr_cache'" exception=(ErrorException("Declaration of __precompile__(false) not allowed"), catch_backtrace()) exit(125) # we define status = 125 means PrecompileableError end + Core.Compiler.track_newly_inferred.x = false + ccall(:jl_set_newly_inferred, Cvoid, (Any,), Core.Compiler.newly_inferred) end const PRECOMPILE_TRACE_COMPILE = Ref{String}() diff --git a/src/codegen.cpp b/src/codegen.cpp index 28c4301fd3f76..47747874028e4 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -7415,16 +7415,22 @@ static std::pair, jl_llvm_functions_t> } } + // if we created any new roots during codegen, // copy ctx.roots into m->roots - // if we created any new roots during codegen if (ctx.roots) { jl_method_t *m = lam->def.method; + bool added = false; + bool external = jl_precompile_toplevel_module != NULL && jl_parent_module(m->module) != jl_precompile_toplevel_module; + int rootslen; JL_LOCK(&m->writelock); if (m->roots == NULL) { + rootslen = 0; m->roots = ctx.roots; jl_gc_wb(m, m->roots); + added = true; } else { + rootslen = jl_array_len(m->roots); size_t i, ilen = jl_array_dim0(ctx.roots); size_t j, jlen = jl_array_dim0(m->roots); for (i = 0; i < ilen; i++) { @@ -7434,11 +7440,19 @@ static std::pair, jl_llvm_functions_t> if (ival == jval) break; } - if (j == jlen) // not found - add to array + if (j == jlen) { // not found - add to array jl_array_ptr_1d_push(m->roots, ival); + added = true; + } } } ctx.roots = NULL; + if (added & external) { + uint64_t key = jl_precompile_toplevel_module->build_id; + if (jl_current_block_key(m) != key) { + jl_add_root_block(m, key, rootslen); + } + } JL_UNLOCK(&m->writelock); } @@ -7568,7 +7582,7 @@ jl_compile_result_t jl_emit_codeinst( jl_options.debug_level > 1) { // update the stored code if (codeinst->inferred != (jl_value_t*)src) { - if (jl_is_method(def)) + if (jl_is_method(def) && !jl_options.incremental) src = (jl_code_info_t*)jl_compress_ir(def, src); codeinst->inferred = (jl_value_t*)src; jl_gc_wb(codeinst, src); diff --git a/src/common_symbols1.inc b/src/common_symbols1.inc index 80038837be0c4..7d445289e80fa 100644 --- a/src/common_symbols1.inc +++ b/src/common_symbols1.inc @@ -97,4 +97,3 @@ jl_symbol("undef"), jl_symbol("sizeof"), jl_symbol("String"), jl_symbol("namedtuple.jl"), -jl_symbol("pop"), diff --git a/src/common_symbols2.inc b/src/common_symbols2.inc index a28f1ef50af24..c9f4e41b83e33 100644 --- a/src/common_symbols2.inc +++ b/src/common_symbols2.inc @@ -1,3 +1,4 @@ +jl_symbol("pop"), jl_symbol("inbounds"), jl_symbol("strings/string.jl"), jl_symbol("Ref"), @@ -251,4 +252,3 @@ jl_symbol("GitError"), jl_symbol("zeros"), jl_symbol("InexactError"), jl_symbol("LogLevel"), -jl_symbol("between"), diff --git a/src/dump.c b/src/dump.c index f6e467bcce78b..d3afd2958b0ff 100644 --- a/src/dump.c +++ b/src/dump.c @@ -66,8 +66,21 @@ htable_t edges_map; // list of requested ccallable signatures static arraylist_t ccallable_list; +// for methods that had already had their roots tables extended +// before deserialization started, we need to temporarily +// move the added roots aside +htable_t deferred_newroots; + +// state of (de)serialization: +// 0: not (de)serializing +// 1: (de)serializing internal methods/types/etc +// 2: (de)serializing external MethodInstances +int currently_serializing = 0; +int currently_deserializing = 0; + typedef struct { ios_t *s; + uint64_t key; jl_ptls_t ptls; jl_array_t *loaded_modules_array; } jl_serializer_state; @@ -77,6 +90,36 @@ static jl_typename_t *jl_idtable_typename = NULL; static jl_value_t *jl_bigint_type = NULL; static int gmp_limb_size = 0; +// debugging utility +static void show_method_roots(jl_array_t *mis, int deser) +{ + assert(jl_is_array(mis)); + size_t i, l = jl_array_len(mis); + htable_t umethods; + htable_new(&umethods, 0); + for (i = 0; i < l; i++) { + jl_method_instance_t *mi = (jl_method_instance_t*)jl_array_ptr_ref(mis, i); + assert(jl_is_method_instance(mi)); + jl_method_t *m = mi->def.method; + assert(jl_is_method(m)); + ptrhash_put(&umethods, m, m); + } + for (i = 0; i < umethods.size; i+=2) { + if (umethods.table[i+1] != HT_NOTFOUND) { + jl_method_t *m = (jl_method_t*)(umethods.table[i]); + assert(jl_is_method(m)); + jl_(serializer_worklist); + jl_printf(JL_STDOUT, "deser: %d, method ", deser); + jl_(m); + jl_printf(JL_STDOUT, " with root blocks:\n"); + jl_(m->external_root_blocks); + jl_printf(JL_STDOUT, " and roots:\n"); + jl_(m->roots); + } + } + htable_free(&umethods); +} + static void write_uint64(ios_t *s, uint64_t i) JL_NOTSAFEPOINT { ios_write(s, (char*)&i, 8); @@ -107,6 +150,10 @@ jl_value_t *jl_deser_symbol(uint8_t tag) return deser_symbols[tag]; } +// Inference tracks newly-inferred MethodInstances during precompilation +// and adds them by calling jl_set_newly_inferred +static jl_array_t *newly_inferred JL_GLOBALLY_ROOTED; + // --- serialize --- #define jl_serialize_value(s, v) jl_serialize_value_((s), (jl_value_t*)(v), 0) @@ -186,6 +233,54 @@ static int type_recursively_external(jl_datatype_t *dt) JL_NOTSAFEPOINT return 1; } +// Given a list of MethodInstances: +// - insert the key into the root blocks table +// - compress all CodeInfos (must occur before Method serialization so that roots are complete) +// - filter out any that are owned by a module in the worklist +// Return the number of external MethodInstances remaining in `list`. +static size_t compress_and_filter_mis(jl_array_t *list, uint64_t key) +{ + size_t i, n = 0, n0 = 0; + jl_value_t **listdata = NULL; + if (list) { + n0 = jl_array_len(list); + listdata = (jl_value_t**)jl_array_data(list); + } + int retained; + for (i = 0; i < n0; i++) { + jl_method_instance_t *mi = (jl_method_instance_t*)listdata[i]; + assert(jl_is_method_instance(mi)); + retained = 0; + if (jl_is_method(mi->def.value)) { + jl_method_t *m = mi->def.method; + retained = 0; + if (!module_in_worklist(mi->def.method->module)) { + listdata[n++] = (jl_value_t*)mi; + retained = 1; + } + size_t len = 0; + if (m->roots) + len = jl_array_len(m->roots); + jl_code_instance_t *ci = mi->cache; + while (ci) { + if (ci->inferred && ci->inferred != jl_nothing) { + assert(jl_is_code_info(ci->inferred)); // during incremental compilation we don't compress until serialization + ci->inferred = (jl_value_t*)jl_compress_ir_(m, (jl_code_info_t*)ci->inferred, key); + jl_gc_wb(ci, ci->inferred); + } + ci = ci->next; + } + if (m->roots) { + if (jl_array_len(m->roots) > len) { + assert(jl_current_block_key(m) == key || (!retained && jl_current_block_key(m) == 0)); + } + } + } + } + if (n != n0) + jl_array_del_end(list, n0 - n); + return n; +} static void jl_serialize_datatype(jl_serializer_state *s, jl_datatype_t *dt) JL_GC_DISABLED { @@ -483,7 +578,8 @@ static void jl_serialize_code_instance(jl_serializer_state *s, jl_code_instance_ write_uint8(s->s, TAG_CODE_INSTANCE); write_uint8(s->s, flags); - jl_serialize_value(s, (jl_value_t*)codeinst->def); + jl_method_instance_t *mi = codeinst->def; + jl_serialize_value(s, mi); if (write_ret_type) { jl_serialize_value(s, codeinst->inferred); jl_serialize_value(s, codeinst->rettype_const); @@ -501,6 +597,7 @@ static void jl_serialize_code_instance(jl_serializer_state *s, jl_code_instance_ enum METHOD_SERIALIZATION_MODE { METHOD_INTERNAL = 1, METHOD_EXTERNAL_MT = 2, + METHOD_HAS_NEW_ROOTS = 4, }; static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v, int as_literal) JL_GC_DISABLED @@ -634,6 +731,9 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v, int as_li uintptr_t *bp = (uintptr_t*)ptrhash_bp(&backref_table, v); assert(*bp != (uintptr_t)HT_NOTFOUND); *bp |= 1; + // check for external roots + if (jl_current_block_key(m) == s->key) //(m->external_root_blocks && jl_array_len(m->external_root_blocks) > 0) + serialization_mode |= METHOD_HAS_NEW_ROOTS; } jl_serialize_value(s, (jl_value_t*)m->sig); jl_serialize_value(s, (jl_value_t*)m->module); @@ -654,6 +754,15 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v, int as_li else { jl_serialize_value(s, (jl_value_t*)m->external_mt); } + if (serialization_mode & METHOD_HAS_NEW_ROOTS) { + // serialize the new roots and key + size_t i, l = jl_array_len(m->roots), newrootsoffset = current_block_offset(m); + write_int32(s->s, l - newrootsoffset); + for (i = newrootsoffset; i < l; i++) { + jl_serialize_value(s, (jl_value_t*)jl_array_ptr_ref(m->roots, i)); + } + write_uint64(s->s, s->key); + } if (!(serialization_mode & METHOD_INTERNAL)) return; jl_serialize_value(s, m->specializations); @@ -700,13 +809,14 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v, int as_li jl_serialize_value(s, (jl_value_t*)mi->uninferred); jl_serialize_value(s, (jl_value_t*)mi->specTypes); jl_serialize_value(s, mi->def.value); - if (!internal) + if (!internal && currently_serializing < 2) return; jl_serialize_value(s, (jl_value_t*)mi->sparam_vals); jl_array_t *backedges = mi->backedges; if (backedges) { // filter backedges to only contain pointers // to items that we will actually store (internal == 2) + // FIXME: might need adjustment for cached external methods? size_t ins, i, l = jl_array_len(backedges); jl_method_instance_t **b_edges = (jl_method_instance_t**)jl_array_data(backedges); for (ins = i = 0; i < l; i++) { @@ -1275,6 +1385,8 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t **udepsp, jl_array_t * static jl_value_t *jl_deserialize_value(jl_serializer_state *s, jl_value_t **loc) JL_GC_DISABLED; +static jl_method_t *jl_lookup_method(jl_methtable_t *mt, jl_datatype_t *sig, size_t world); + static jl_value_t *jl_deserialize_datatype(jl_serializer_state *s, int pos, jl_value_t **loc) JL_GC_DISABLED { assert(pos == backref_list.len - 1 && "nothing should have been deserialized since assigning pos"); @@ -1507,6 +1619,22 @@ static jl_value_t *jl_deserialize_value_method(jl_serializer_state *s, jl_value_ assert(loc != NULL && loc != HT_NOTFOUND); arraylist_push(&flagref_list, loc); arraylist_push(&flagref_list, (void*)pos); + if (serialization_mode & METHOD_HAS_NEW_ROOTS) { + int i, oldlen = 0, nroots = read_int32(s->s); + if (nroots != 0) { + if (!m->roots) { + m->roots = jl_alloc_vec_any(0); + jl_gc_wb(m, m->roots); + } + oldlen = jl_array_len(m->roots); + jl_array_grow_end(m->roots, nroots); + jl_value_t **rootsdata = (jl_value_t**)jl_array_data(m->roots); + for (i = 0; i < nroots; i++) { + rootsdata[i+oldlen] = jl_deserialize_value(s, &(rootsdata[i+oldlen])); + } + jl_add_root_block(m, read_uint64(s->s), oldlen); + } + } return (jl_value_t*)m; } m->specializations = (jl_svec_t*)jl_deserialize_value(s, (jl_value_t**)&m->specializations); @@ -1574,7 +1702,8 @@ static jl_value_t *jl_deserialize_value_method_instance(jl_serializer_state *s, assert(loc != NULL && loc != HT_NOTFOUND); arraylist_push(&flagref_list, loc); arraylist_push(&flagref_list, (void*)pos); - return (jl_value_t*)mi; + if (currently_deserializing < 2) + return (jl_value_t*)mi; } if (internal == 1) { @@ -1797,6 +1926,7 @@ static jl_value_t *jl_deserialize_value_any(jl_serializer_state *s, uint8_t tag, } } else { + assert(m); jl_datatype_t *dt = (jl_datatype_t*)jl_unwrap_unionall(jl_get_global(m, sym)); assert(jl_is_datatype(dt)); tn = dt->name; @@ -2256,6 +2386,14 @@ JL_DLLEXPORT void jl_init_restored_modules(jl_array_t *init_order) // --- entry points --- +// Register all newly-inferred MethodInstances +// This gets called as the final step of Base.include_package_for_output +JL_DLLEXPORT void jl_set_newly_inferred(jl_value_t* _newly_inferred) +{ + assert(_newly_inferred == NULL || jl_is_array(_newly_inferred)); + newly_inferred = (jl_array_t*) _newly_inferred; +} + JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) { JL_TIMING(SAVE_MODULE); @@ -2266,7 +2404,9 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) return 1; } JL_GC_PUSH2(&mod_array, &udeps); + uint64_t key = ((jl_module_t*)jl_array_ptr_ref(worklist, jl_array_len(worklist)-1))->build_id; mod_array = jl_get_loaded_modules(); + currently_serializing = 1; serializer_worklist = worklist; write_header(&f); @@ -2292,6 +2432,9 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) jl_symbol("BITS_PER_LIMB"))) / 8; } + // newly_inferred: discard internal MethodInstances, as these will be handled by "ordinary" module traversal + size_t n_ext_mis = compress_and_filter_mis(newly_inferred, key); + int en = jl_gc_enable(0); // edges map is not gc-safe jl_array_t *lambdas = jl_alloc_vec_any(0); jl_array_t *edges = jl_alloc_vec_any(0); @@ -2309,18 +2452,26 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) jl_collect_missing_backedges_to_mod(jl_type_type_mt); jl_collect_methtable_from_mod(lambdas, jl_nonfunction_mt); jl_collect_missing_backedges_to_mod(jl_nonfunction_mt); + for (i = 0; i < n_ext_mis; i++) + collect_backedges((jl_method_instance_t*)jl_array_ptr_ref(newly_inferred, i)); jl_collect_backedges(edges, targets); jl_serializer_state s = { &f, + key, jl_current_task->ptls, mod_array }; jl_serialize_value(&s, worklist); + currently_serializing = 2; + jl_serialize_value(&s, newly_inferred); + currently_serializing = 1; jl_serialize_value(&s, lambdas); + jl_serialize_value(&s, edges); jl_serialize_value(&s, targets); + jl_finalize_serializer(&s); serializer_worklist = NULL; @@ -2376,6 +2527,7 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) write_int32(&f, 0); // mark the end of the source text ios_close(&f); JL_GC_POP(); + currently_serializing = 0; return 0; } @@ -2549,23 +2701,128 @@ static jl_method_t *jl_lookup_method(jl_methtable_t *mt, jl_datatype_t *sig, siz return (jl_method_t*)entry->func.value; } -static jl_method_t *jl_recache_method(jl_method_t *m) +static jl_method_t *jl_recache_method(jl_method_t *m, uint64_t key) { assert(!m->is_for_opaque_closure); jl_datatype_t *sig = (jl_datatype_t*)m->sig; jl_methtable_t *mt = jl_method_get_table(m); assert((jl_value_t*)mt != jl_nothing); jl_set_typeof(m, (void*)(intptr_t)0x30); // invalidate the old value to help catch errors - jl_method_t *_new = jl_lookup_method(mt, sig, m->module->primary_world); - return _new; + jl_method_t *mreal = jl_lookup_method(mt, sig, m->module->primary_world); + if (jl_current_block_key(mreal) == key) { + // this method had already had its roots table extended prior to deserialization, e.g.: + // + // module A + // # run some code here that extends the roots table for `foo(Any)` + // using B # B also extends the roots table for `foo(Any)` + // end + // + // We can't interweave root blocks, so transiently extract the new roots added by A + // and later append them after the roots added by B + assert(ptrhash_get(&deferred_newroots, mreal) == HT_NOTFOUND); + size_t i, nroots = jl_array_len(mreal->roots), offset = current_block_offset(mreal); + // jl_printf(JL_STDOUT, "Transiently moving %ld roots out of the way for ", nroots - offset); + // jl_(mreal); + jl_array_t *rts = jl_alloc_vec_any(nroots - offset); + for (i = offset; i < nroots; i++) + jl_array_ptr_set(rts, i - offset, jl_array_ptr_ref(mreal->roots, i)); + // jl_(rts); + ptrhash_put(&deferred_newroots, mreal, rts); + jl_array_del_end(mreal->roots, nroots - offset); + if (jl_current_block_key(mreal)) { + delete_current_root_block(mreal); + } + } + return mreal; +} + +// Copy new roots to the running method +// This has to be done after recache_other +static void copy_roots(jl_method_t *dest, jl_array_t *newroots, uint64_t key) +{ + assert(jl_is_method(dest)); + // copy any new roots + if (newroots) { + assert(jl_is_array(newroots)); + // The roots may have references to recached items, fix that + size_t i, len = jl_array_len(newroots); + for (i = 0; i < len; i++) { + jl_value_t *r = jl_array_ptr_ref(newroots, i); + jl_value_t *newr = (jl_value_t*)ptrhash_get(&uniquing_table, r); + if (newr != HT_NOTFOUND) + jl_array_ptr_set(newroots, i, newr); + } + if (dest->roots) { + jl_add_root_block(dest, key, jl_array_len(dest->roots)); + jl_array_ptr_1d_append(dest->roots, newroots); + } + else { + jl_add_root_block(dest, key, 0); + dest->roots = newroots; + jl_gc_wb(dest, dest->roots); + } + } } -static jl_value_t *jl_recache_other_(jl_value_t *o); +static void copy_roots_and_blocks(jl_method_t *dest, jl_array_t *newroots, jl_array_t *newblocks) +{ + assert(jl_is_method(dest)); + // copy any new roots + if (newroots) { + assert(jl_is_array(newroots)); + assert(newblocks); + assert(jl_is_array(newblocks)); + // The roots may have references to recached items, fix that + size_t i, len = jl_array_len(newroots); + for (i = 0; i < len; i++) { + jl_value_t *r = jl_array_ptr_ref(newroots, i); + jl_value_t *newr = (jl_value_t*)ptrhash_get(&uniquing_table, r); + if (newr != HT_NOTFOUND) + jl_array_ptr_set(newroots, i, newr); + } + append_missing_blocks(dest, newroots, newblocks); + } +} -static jl_method_instance_t *jl_recache_method_instance(jl_method_instance_t *mi) +static void copy_roots_all(jl_array_t *list, jl_array_t *roots) +{ + assert(jl_is_array(list)); + htable_t umethods; + htable_new(&umethods, 0); + size_t i, len = jl_array_len(list); + for (i = 0; i < len; i++) { + jl_method_instance_t *mi = (jl_method_instance_t*)jl_array_ptr_ref(list, i); + jl_method_t *m = mi->def.method; + assert(jl_is_method(m)); + if (ptrhash_get(&umethods, m) == HT_NOTFOUND) { + ptrhash_put(&umethods, m, m); + jl_array_t *newroots = (jl_array_t*)jl_array_ptr_ref(roots, 2*i); + jl_array_t *newblocks = (jl_array_t*)jl_array_ptr_ref(roots, 2*i+1); + copy_roots_and_blocks(mi->def.method, newroots, newblocks); + } + } + htable_free(&umethods); +} + +static void restore_deferred_newroots(uint64_t key) +{ + size_t i; + for (i = 0; i < deferred_newroots.size; i += 2) { + if (deferred_newroots.table[i+1] != HT_NOTFOUND) { + jl_method_t *m = (jl_method_t*)deferred_newroots.table[i]; + jl_array_t *roots = (jl_array_t*)deferred_newroots.table[i+1]; + // jl_printf(JL_STDOUT, "Resetting %ld roots for ", jl_array_len(roots)); + copy_roots(m, roots, key); + } + } +} + +static jl_value_t *jl_recache_other_(jl_value_t *o, uint64_t key); + +static jl_method_instance_t *jl_recache_method_instance(jl_method_instance_t *mi, uint64_t key) { jl_method_t *m = mi->def.method; - m = (jl_method_t*)jl_recache_other_((jl_value_t*)m); + m = (jl_method_t*)jl_recache_other_((jl_value_t*)m, key); assert(jl_is_method(m)); jl_datatype_t *argtypes = (jl_datatype_t*)mi->specTypes; jl_set_typeof(mi, (void*)(intptr_t)0x40); // invalidate the old value to help catch errors @@ -2575,31 +2832,38 @@ static jl_method_instance_t *jl_recache_method_instance(jl_method_instance_t *mi if (ti == jl_bottom_type) env = jl_emptysvec; // the intersection may fail now if the type system had made an incorrect subtype env in the past jl_method_instance_t *_new = jl_specializations_get_linfo(m, (jl_value_t*)argtypes, env); + if (_new->cache == NULL) { + jl_code_instance_t *ci = mi->cache; + if (ci) { + _new->cache = ci; + jl_gc_wb(_new, _new->cache); + } + } return _new; } -static jl_value_t *jl_recache_other_(jl_value_t *o) +static jl_value_t *jl_recache_other_(jl_value_t *o, uint64_t key) { jl_value_t *newo = (jl_value_t*)ptrhash_get(&uniquing_table, o); if (newo != HT_NOTFOUND) return newo; if (jl_is_method(o)) { // lookup the real Method based on the placeholder sig - newo = (jl_value_t*)jl_recache_method((jl_method_t*)o); - ptrhash_put(&uniquing_table, newo, newo); + newo = (jl_value_t*)jl_recache_method((jl_method_t*)o, key); } else if (jl_is_method_instance(o)) { // lookup the real MethodInstance based on the placeholder specTypes - newo = (jl_value_t*)jl_recache_method_instance((jl_method_instance_t*)o); + newo = (jl_value_t*)jl_recache_method_instance((jl_method_instance_t*)o, key); } else { abort(); } + ptrhash_put(&uniquing_table, newo, newo); ptrhash_put(&uniquing_table, o, newo); return newo; } -static void jl_recache_other(void) +static void jl_recache_other(uint64_t key) { size_t i = 0; while (i < flagref_list.len) { @@ -2607,7 +2871,7 @@ static void jl_recache_other(void) int offs = (int)(intptr_t)flagref_list.items[i + 1]; jl_value_t *o = loc ? *loc : (jl_value_t*)backref_list.items[offs]; i += 2; - jl_value_t *newo = jl_recache_other_(o); + jl_value_t *newo = jl_recache_other_(o, key); if (loc) *loc = newo; if (offs > 0) @@ -2631,10 +2895,13 @@ static jl_value_t *_jl_restore_incremental(ios_t *f, jl_array_t *mod_array) return jl_get_exceptionf(jl_errorexception_type, "Precompile file header verification checks failed."); } - { // skip past the mod list + uint64_t key = 0; + { // skip past the mod list, except to extract the key size_t len; - while ((len = read_int32(f))) - ios_skip(f, len + 3 * sizeof(uint64_t)); + while ((len = read_int32(f))) { + ios_skip(f, len + 2 * sizeof(uint64_t)); + key = read_uint64(f); // build_id + } } { // skip past the dependency list size_t deplen = read_uint64(f); @@ -2655,6 +2922,7 @@ static jl_value_t *_jl_restore_incremental(ios_t *f, jl_array_t *mod_array) } // prepare to deserialize + currently_deserializing = 1; int en = jl_gc_enable(0); jl_gc_enable_finalizers(ct, 0); jl_atomic_fetch_add(&jl_world_counter, 1); // reserve a world age for the deserialization @@ -2665,21 +2933,30 @@ static jl_value_t *_jl_restore_incremental(ios_t *f, jl_array_t *mod_array) htable_new(&new_code_instance_validate, 0); arraylist_new(&ccallable_list, 0); htable_new(&uniquing_table, 0); + htable_new(&deferred_newroots, 0); jl_serializer_state s = { f, + key, ct->ptls, mod_array }; jl_array_t *restored = (jl_array_t*)jl_deserialize_value(&s, (jl_value_t**)&restored); serializer_worklist = restored; + assert(jl_isa((jl_value_t*)restored, jl_array_any_type)); + // restore newly-compiled external functions + currently_deserializing = 2; + jl_value_t *ext_mis = jl_deserialize_value(&s, &ext_mis); + currently_deserializing = 1; + // get list of external generic functions jl_value_t *external_methods = jl_deserialize_value(&s, &external_methods); jl_value_t *external_backedges = jl_deserialize_value(&s, &external_backedges); jl_value_t *external_edges = jl_deserialize_value(&s, &external_edges); + arraylist_t *tracee_list = NULL; if (jl_newmeth_tracer) tracee_list = arraylist_new((arraylist_t*)malloc_s(sizeof(arraylist_t)), 0); @@ -2687,10 +2964,21 @@ static jl_value_t *_jl_restore_incremental(ios_t *f, jl_array_t *mod_array) // at this point, the AST is fully reconstructed, but still completely disconnected // now all of the interconnects will be created jl_recache_types(); // make all of the types identities correct - htable_reset(&uniquing_table, 0); jl_insert_methods((jl_array_t*)external_methods); // hook up methods of external generic functions (needs to be after recache types) - jl_recache_other(); // make all of the other objects identities correct (needs to be after insert methods) - htable_free(&uniquing_table); + currently_deserializing = 2; + // Reserve the deserialized extra roots to add to the "real" method + jl_array_t *ext_mis_roots = jl_alloc_vec_any(2*jl_array_len(ext_mis)); + size_t i; + for (i = 0; i < jl_array_len(ext_mis); i++) { + jl_method_instance_t *mi = (jl_method_instance_t*)jl_array_ptr_ref(ext_mis, i); + assert(jl_is_method_instance(mi)); + jl_array_ptr_set(ext_mis_roots, 2*i, (jl_value_t*)mi->def.method->roots); + jl_array_ptr_set(ext_mis_roots, 2*i+1, (jl_value_t*)mi->def.method->external_root_blocks); + } + jl_recache_other(key); // make method identities correct (needs to be to be after insert methods) + copy_roots_all((jl_array_t*)ext_mis, ext_mis_roots); + currently_deserializing = 1; + jl_array_t *init_order = jl_finalize_deserializer(&s, tracee_list); // done with f and s (needs to be after recache) if (init_order == NULL) init_order = (jl_array_t*)jl_an_empty_vec_any; @@ -2730,6 +3018,11 @@ static jl_value_t *_jl_restore_incremental(ios_t *f, jl_array_t *mod_array) arraylist_free(&ccallable_list); jl_value_t *ret = (jl_value_t*)jl_svec(2, restored, init_order); JL_GC_POP(); + currently_deserializing = 0; + restore_deferred_newroots(key); + htable_free(&deferred_newroots); + htable_free(&uniquing_table); + // show_method_roots((jl_array_t*)ext_mis, 2); return (jl_value_t*)ret; } diff --git a/src/init.c b/src/init.c index 1eb2cccd73d68..e1dafb21fc095 100644 --- a/src/init.c +++ b/src/init.c @@ -625,6 +625,10 @@ JL_DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel) libsupport_init(); htable_new(&jl_current_modules, 0); JL_MUTEX_INIT(&jl_modules_mutex); + if (jl_generating_output()) { + jl_precompile_toplevel_module = NULL; + jl_set_newly_inferred((jl_value_t*)NULL); + } ios_set_io_wait_func = jl_set_io_wait; jl_io_loop = uv_default_loop(); // this loop will internal events (spawning process etc.), // best to call this first, since it also initializes libuv diff --git a/src/ircode.c b/src/ircode.c index 279e458728e24..3fd7dd573b2c5 100644 --- a/src/ircode.c +++ b/src/ircode.c @@ -25,6 +25,8 @@ typedef struct { ios_t *s; // method we're compressing for jl_method_t *method; + // key is nonzero if we're serializing a CodeInfo for a method owned by a non-worklist module + uint64_t key; jl_ptls_t ptls; } jl_ircode_state; @@ -32,24 +34,26 @@ typedef struct { #define jl_encode_value(s, v) jl_encode_value_((s), (jl_value_t*)(v), 0) -static int literal_val_id(jl_ircode_state *s, jl_value_t *v) JL_GC_DISABLED +static root_reference literal_val_id(jl_ircode_state *s, jl_value_t *v) JL_GC_DISABLED { jl_array_t *rs = s->method->roots; int i, l = jl_array_len(rs); if (jl_is_symbol(v) || jl_is_concrete_type(v)) { for (i = 0; i < l; i++) { if (jl_array_ptr_ref(rs, i) == v) - return i; + return get_root_reference(s->method, i); } } else { for (i = 0; i < l; i++) { if (jl_egal(jl_array_ptr_ref(rs, i), v)) - return i; + return get_root_reference(s->method, i); } } - jl_array_ptr_1d_push(rs, v); - return jl_array_len(rs) - 1; + uint64_t key = 0; + if (jl_parent_module(s->method->module)->build_id != s->key) + key = s->key; + return append_root(s->method, key, v); } static void jl_encode_int32(jl_ircode_state *s, int32_t x) @@ -321,16 +325,19 @@ static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal) if (!as_literal && !(jl_is_uniontype(v) || jl_is_newvarnode(v) || jl_is_tuple(v) || jl_is_linenode(v) || jl_is_upsilonnode(v) || jl_is_pinode(v) || jl_is_slot(v) || jl_is_ssavalue(v))) { - int id = literal_val_id(s, v); - assert(id >= 0); - if (id < 256) { + root_reference rr = literal_val_id(s, v); + if (rr.key) { + write_uint8(s->s, TAG_EXTERN_METHODROOT); + write_int64(s->s, rr.key); + } + if (rr.relative_index < 256) { write_uint8(s->s, TAG_METHODROOT); - write_uint8(s->s, id); + write_uint8(s->s, rr.relative_index); } else { - assert(id <= UINT16_MAX); + assert(rr.relative_index <= UINT16_MAX); write_uint8(s->s, TAG_LONG_METHODROOT); - write_uint16(s->s, id); + write_uint16(s->s, rr.relative_index); } return; } @@ -577,6 +584,7 @@ static jl_value_t *jl_decode_value(jl_ircode_state *s) JL_GC_DISABLED assert(!ios_eof(s->s)); jl_value_t *v; size_t i, n; + uint64_t key; uint8_t tag = read_uint8(s->s); if (tag > LAST_TAG) return jl_deser_tag(tag); @@ -589,6 +597,11 @@ static jl_value_t *jl_decode_value(jl_ircode_state *s) JL_GC_DISABLED return jl_array_ptr_ref(s->method->roots, read_uint8(s->s)); case TAG_LONG_METHODROOT: return jl_array_ptr_ref(s->method->roots, read_uint16(s->s)); + case TAG_EXTERN_METHODROOT: + key = read_uint64(s->s); + tag = read_uint8(s->s); + assert(tag == TAG_METHODROOT || tag == TAG_LONG_METHODROOT); + return fetch_root(s->method, key, tag == TAG_METHODROOT ? read_uint8(s->s) : read_uint16(s->s)); case TAG_SVEC: JL_FALLTHROUGH; case TAG_LONG_SVEC: return jl_decode_value_svec(s, tag); case TAG_COMMONSYM: @@ -688,7 +701,7 @@ static jl_value_t *jl_decode_value(jl_ircode_state *s) JL_GC_DISABLED // --- entry points --- -JL_DLLEXPORT jl_array_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code) +jl_array_t *jl_compress_ir_(jl_method_t *m, jl_code_info_t *code, uint64_t key) { JL_TIMING(AST_COMPRESS); JL_LOCK(&m->writelock); // protect the roots array (Might GC) @@ -706,6 +719,7 @@ JL_DLLEXPORT jl_array_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code) jl_ircode_state s = { &dest, m, + key, jl_current_task->ptls }; @@ -769,6 +783,11 @@ JL_DLLEXPORT jl_array_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code) return v; } +JL_DLLEXPORT jl_array_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code) +{ + return jl_compress_ir_(m, code, 0); +} + JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t *metadata, jl_array_t *data) { if (jl_is_code_info(data)) @@ -786,6 +805,7 @@ JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t jl_ircode_state s = { &src, m, + -1, jl_current_task->ptls }; diff --git a/src/jl_exported_data.inc b/src/jl_exported_data.inc index 588d2a831e225..336bd80c32214 100644 --- a/src/jl_exported_data.inc +++ b/src/jl_exported_data.inc @@ -14,6 +14,7 @@ XX(jl_argument_type) \ XX(jl_array_any_type) \ XX(jl_array_int32_type) \ + XX(jl_array_uint64_type) \ XX(jl_array_symbol_type) \ XX(jl_array_type) \ XX(jl_array_typename) \ @@ -84,6 +85,7 @@ XX(jl_pinode_type) \ XX(jl_pointer_type) \ XX(jl_pointer_typename) \ + XX(jl_precompile_toplevel_module) \ XX(jl_quotenode_type) \ XX(jl_readonlymemory_exception) \ XX(jl_ref_type) \ diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 330bd3edf7717..5b8a52a554a5f 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -2,6 +2,7 @@ #define JL_RUNTIME_EXPORTED_FUNCS(XX) \ XX(jl_active_task_stack) \ + XX(jl_add_root_block) \ XX(jl_add_standard_imports) \ XX(jl_alignment) \ XX(jl_alloc_array_1d) \ @@ -107,6 +108,7 @@ XX(jl_crc32c_sw) \ XX(jl_create_system_image) \ XX(jl_cstr_to_string) \ + XX(jl_current_block_key) \ XX(jl_current_exception) \ XX(jl_debug_method_invalidation) \ XX(jl_declare_constant) \ @@ -360,6 +362,7 @@ XX(jl_object_id_) \ XX(jl_obvious_subtype) \ XX(jl_operator_precedence) \ + XX(jl_parent_module) \ XX(jl_parse) \ XX(jl_parse_all) \ XX(jl_parse_input_line) \ diff --git a/src/jltypes.c b/src/jltypes.c index cfed8982cbc7c..678ed18b350e5 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2253,6 +2253,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_array_symbol_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_symbol_type, jl_box_long(1)); jl_array_uint8_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_uint8_type, jl_box_long(1)); jl_array_int32_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_int32_type, jl_box_long(1)); + jl_array_uint64_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_uint64_type, jl_box_long(1)); jl_an_empty_vec_any = (jl_value_t*)jl_alloc_vec_any(0); // used internally jl_atomic_store_relaxed(&jl_nonfunction_mt->leafcache, (jl_array_t*)jl_an_empty_vec_any); jl_atomic_store_relaxed(&jl_type_type_mt->leafcache, (jl_array_t*)jl_an_empty_vec_any); @@ -2392,7 +2393,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_method_type = jl_new_datatype(jl_symbol("Method"), core, jl_any_type, jl_emptysvec, - jl_perm_symsvec(26, + jl_perm_symsvec(27, "name", "module", "file", @@ -2408,6 +2409,7 @@ void jl_init_types(void) JL_GC_DISABLED "unspecialized", // !const "generator", // !const "roots", // !const + "external_root_blocks", // !const "ccallable", // !const "invokes", // !const "recursion_relation", // !const @@ -2419,7 +2421,7 @@ void jl_init_types(void) JL_GC_DISABLED "pure", "is_for_opaque_closure", "constprop"), - jl_svec(26, + jl_svec(27, jl_symbol_type, jl_module_type, jl_symbol_type, @@ -2435,6 +2437,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_any_type, // jl_method_instance_type jl_any_type, jl_array_any_type, + jl_array_uint64_type, jl_simplevector_type, jl_any_type, jl_any_type, diff --git a/src/julia.h b/src/julia.h index 5da7ad1bd1d77..70e4aff8bef99 100644 --- a/src/julia.h +++ b/src/julia.h @@ -299,6 +299,9 @@ typedef struct _jl_method_t { _Atomic(struct _jl_method_instance_t*) unspecialized; // unspecialized executable method instance, or null jl_value_t *generator; // executable code-generating function if available jl_array_t *roots; // pointers in generated code (shared to reduce memory), or null + // Mark the beginning of a block of roots added by (de)serialization of a dependent module + // Field may be NULL if no external roots have been added, otherwise it's a Vector{UInt64} + jl_array_t *external_root_blocks; // (build_id::UInt64, offset::UInt64) pairs (even/odd indexing) jl_svec_t *ccallable; // svec(rettype, sig) if a ccallable entry point is requested for this // cache of specializations of this method for invoke(), i.e. @@ -716,6 +719,7 @@ extern JL_DLLIMPORT jl_value_t *jl_array_uint8_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_array_any_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_array_symbol_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_array_int32_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_value_t *jl_array_uint64_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_expr_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_globalref_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_linenumbernode_type JL_GLOBALLY_ROOTED; @@ -1585,6 +1589,10 @@ STATIC_INLINE jl_function_t *jl_get_function(jl_module_t *m, const char *name) return (jl_function_t*)jl_get_global(m, jl_symbol(name)); } +// methods and method roots +JL_DLLEXPORT uint64_t jl_current_block_key(jl_method_t *m); +JL_DLLEXPORT void jl_add_root_block(jl_method_t *m, uint64_t key, size_t offset); + // eq hash tables JL_DLLEXPORT jl_array_t *jl_eqtable_put(jl_array_t *h, jl_value_t *key, jl_value_t *val, int *inserted); JL_DLLEXPORT jl_value_t *jl_eqtable_get(jl_array_t *h, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT; @@ -1691,6 +1699,8 @@ JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len); JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist); JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *depmods); JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, size_t sz, jl_array_t *depmods); +JL_DLLEXPORT void jl_set_newly_inferred(jl_value_t *newly_inferred); +JL_DLLEXPORT int jl_get_method_newrootsindex(jl_value_t* m); // debugging // parsing JL_DLLEXPORT jl_value_t *jl_parse_all(const char *text, size_t text_len, diff --git a/src/julia_internal.h b/src/julia_internal.h index a8c289f14a145..11cc7ad1ca534 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -505,6 +505,12 @@ typedef union { uint8_t packed; } jl_code_info_flags_t; +typedef struct { + uint64_t key; // 0 means internal + size_t block_offset; + int relative_index; +} root_reference; + // -- functions -- // // jl_code_info_flag_t code_info_flags(uint8_t pure, uint8_t propagate_inbounds, uint8_t inlineable, uint8_t inferred, uint8_t constprop); @@ -524,6 +530,14 @@ jl_code_info_t *jl_new_code_info_from_ir(jl_expr_t *ast); JL_DLLEXPORT jl_code_info_t *jl_new_code_info_uninit(void); void jl_resolve_globals_in_ir(jl_array_t *stmts, jl_module_t *m, jl_svec_t *sparam_vals, int binding_effects); +jl_array_t *jl_compress_ir_(jl_method_t *m, jl_code_info_t *code, uint64_t key); + +size_t current_block_offset(jl_method_t *m); +void delete_current_root_block(jl_method_t *m); +root_reference append_root(jl_method_t *m, uint64_t key, jl_value_t *root); +void append_missing_blocks(jl_method_t *m, jl_array_t *newroots_array, jl_array_t *newblocks_array); +jl_value_t *fetch_root(jl_method_t *m, uint64_t key, size_t offset_relative); +root_reference get_root_reference(jl_method_t *m, size_t i); int jl_valid_type_param(jl_value_t *v); @@ -606,6 +620,7 @@ jl_function_t *jl_new_generic_function_with_supertype(jl_sym_t *name, jl_module_ void jl_foreach_reachable_mtable(void (*visit)(jl_methtable_t *mt, void *env), void *env); void jl_init_main_module(void); JL_DLLEXPORT int jl_is_submodule(jl_module_t *child, jl_module_t *parent) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_module_t *jl_parent_module(jl_module_t *module); jl_array_t *jl_get_loaded_modules(void); JL_DLLEXPORT int jl_datatype_isinlinealloc(jl_datatype_t *ty, int pointerfree); @@ -645,8 +660,12 @@ jl_binding_t *jl_get_module_binding(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t JL_DLLEXPORT void jl_binding_deprecation_warning(jl_module_t *m, jl_binding_t *b); extern jl_array_t *jl_module_init_order JL_GLOBALLY_ROOTED; extern htable_t jl_current_modules JL_GLOBALLY_ROOTED; +extern JL_DLLEXPORT jl_module_t *jl_precompile_toplevel_module JL_GLOBALLY_ROOTED; int jl_compile_extern_c(void *llvmmod, void *params, void *sysimg, jl_value_t *declrt, jl_value_t *sigt); +extern int currently_serializing; +extern int currently_deserializing; + jl_opaque_closure_t *jl_new_opaque_closure(jl_tupletype_t *argt, jl_value_t *isva, jl_value_t *rt_lb, jl_value_t *rt_ub, jl_value_t *source, jl_value_t **env, size_t nenv); diff --git a/src/method.c b/src/method.c index d8cd1c30a94e1..b20f8f2e0a8ae 100644 --- a/src/method.c +++ b/src/method.c @@ -715,8 +715,12 @@ static void jl_method_set_source(jl_method_t *m, jl_code_info_t *src) jl_gc_wb(m, m->slot_syms); if (gen_only) m->source = NULL; - else - m->source = (jl_value_t*)jl_compress_ir(m, src); + else { + if (jl_options.incremental) + m->source = (jl_value_t*)src; + else + m->source = (jl_value_t*)jl_compress_ir(m, src); + } jl_gc_wb(m, m->source); JL_GC_POP(); } @@ -731,6 +735,7 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t *module) m->sig = NULL; m->slot_syms = NULL; m->roots = NULL; + m->external_root_blocks = NULL; m->ccallable = NULL; m->module = module; m->external_mt = NULL; @@ -987,6 +992,186 @@ JL_DLLEXPORT jl_method_t* jl_method_def(jl_svec_t *argdata, return m; } +// root blocks + +// debugging: check that `key` wasn't inserted previously +static int check_missing_key(uint64_t *blocks, size_t nx2, uint64_t key) +{ + size_t i; + for (i = 0; i < nx2; i+=2) { + if (blocks[i] == key) + return 0; + } + return 1; +} + +// get the key of the current root block (0 if internal) +JL_DLLEXPORT uint64_t jl_current_block_key(jl_method_t *m) +{ + if (!m->external_root_blocks) + return 0; + size_t nx2 = jl_array_len(m->external_root_blocks); + if (nx2 == 0) + return 0; + uint64_t *blocks = (uint64_t*)jl_array_data(m->external_root_blocks); + return blocks[nx2-2]; +} + +// get the offset within m->roots for the current block +size_t current_block_offset(jl_method_t *m) +{ + if (!m->external_root_blocks) + return 0; + size_t nx2 = jl_array_len(m->external_root_blocks); + if (nx2 == 0) + return 0; + uint64_t *blocks = (uint64_t*)jl_array_data(m->external_root_blocks); + return blocks[nx2-1]; +} + +// add a new block of roots +JL_DLLEXPORT void jl_add_root_block(jl_method_t *m, uint64_t key, size_t offset) +{ + JL_GC_PUSH1(&m); + assert(key != -1); + if (!m->external_root_blocks) { + m->external_root_blocks = jl_alloc_array_1d(jl_array_uint64_type, 0); + jl_gc_wb(m, m->external_root_blocks); + } + jl_array_grow_end(m->external_root_blocks, 2); + uint64_t *blocks = (uint64_t*)jl_array_data(m->external_root_blocks); + int nx2 = jl_array_len(m->external_root_blocks); + blocks[nx2-2] = key; + blocks[nx2-1] = offset; + JL_GC_POP(); +} + +void delete_current_root_block(jl_method_t *m) +{ + assert(m->external_root_blocks); + assert(jl_array_len(m->external_root_blocks) >= 2); + jl_array_del_end(m->external_root_blocks, 2); +} + +// insert a new root (might start a new block) +root_reference append_root(jl_method_t *m, uint64_t key, jl_value_t *root) +{ + assert(key != -1); + JL_GC_PUSH2(&m, &root); + if (!m->roots) { + m->roots = jl_alloc_vec_any(0); + jl_gc_wb(m, m->roots); + } + size_t len = jl_array_len(m->roots); + if (!key || !jl_options.incremental) { + // This is being called for an internal method + if (jl_options.incremental) + assert(m->external_root_blocks == NULL || jl_array_len(m->external_root_blocks) == 0); + jl_array_ptr_1d_push(m->roots, root); + root_reference rr = {key, 0, len}; + JL_GC_POP(); + return rr; + } + if (!m->external_root_blocks) { + m->external_root_blocks = jl_alloc_array_1d(jl_array_uint64_type, 0); + jl_gc_wb(m, m->external_root_blocks); + } + // key should either be new or match the last module in external_root_blocks + uint64_t *blocks = (uint64_t*)jl_array_data(m->external_root_blocks); + size_t nx2 = jl_array_len(m->external_root_blocks); + if (nx2 == 0 || blocks[nx2-2] != key) { + // Add a new external-module key + assert(check_missing_key(blocks, nx2, key)); + jl_array_grow_end(m->external_root_blocks, 2); + blocks = (uint64_t*)jl_array_data(m->external_root_blocks); // in case blocks moves + blocks[nx2++] = key; + blocks[nx2++] = len; + } + jl_array_ptr_1d_push(m->roots, root); + root_reference rr = {key, blocks[nx2-1], len - blocks[nx2-1]}; + JL_GC_POP(); + return rr; +} + +void append_missing_blocks(jl_method_t *m, jl_array_t *newroots_array, jl_array_t *newblocks_array) { + JL_GC_PUSH3(&m, &newroots_array, &newblocks_array); + uint64_t *blocks = NULL, *newblocks = NULL; + jl_value_t **newroots = NULL; + int inew, nx2 = 0, nnewx2 = 0; + size_t newlen = 0; + if (m->external_root_blocks) { + blocks = (uint64_t*)jl_array_data(m->external_root_blocks); + nx2 = jl_array_len(m->external_root_blocks); + } + if (newroots_array) { + assert(jl_is_array(newroots_array)); + newroots = (jl_value_t**)jl_array_data(newroots_array); + newlen = jl_array_len(newroots_array); + } + if (newblocks_array) { + assert(jl_is_array(newblocks_array)); + newblocks = (uint64_t*)jl_array_data(newblocks_array); + nnewx2 = jl_array_len(newblocks_array); + } + for (inew = 0; inew < nnewx2; inew += 2) { + uint64_t key = newblocks[inew]; + if (check_missing_key(blocks, nx2, key)) { + if (!m->roots) { + m->roots = jl_alloc_vec_any(0); + jl_gc_wb(m, m->roots); + } + size_t j, start = jl_array_len(m->roots), newstart = newblocks[inew+1], k; + if (inew + 2 < nnewx2) + k = newblocks[inew+3] - newstart; + else + k = newlen - newstart; + jl_array_grow_end(m->roots, k); + jl_value_t **roots = (jl_value_t**)jl_array_data(m->roots); + for (j = 0; j < k; j++) + roots[start + j] = newroots[newstart + j]; + jl_add_root_block(m, key, start); + } + } + JL_GC_POP(); +} + +// get a root, given its key and index within the block +// this is the "relocatable" way to get a root from m->roots +jl_value_t *fetch_root(jl_method_t *m, uint64_t key, size_t relative_index) +{ + assert(key != -1); + if (key == 0) + return jl_array_ptr_ref(m->roots, relative_index); + assert(m->external_root_blocks); + size_t i, nx2 = jl_array_len(m->external_root_blocks); + uint64_t *blocks = (uint64_t*)jl_array_data(m->external_root_blocks); + for (i = 0; i < nx2; i+=2) { + if (blocks[i] == key) + return jl_array_ptr_ref(m->roots, blocks[i+1] + relative_index); + } + jl_errorf("root with key %lx not found", key); +} + +// given the absolute index i of a root, retrieve its relocatable reference +root_reference get_root_reference(jl_method_t *m, size_t i) +{ + uint64_t key = 0; + size_t offset = 0; + if (!m->external_root_blocks) { + root_reference rr = {key, offset, i}; + return rr; + } + uint64_t *blocks = (uint64_t*)jl_array_data(m->external_root_blocks); + int j = 0, nx2 = jl_array_len(m->external_root_blocks); + while (j < nx2 && i > blocks[j+1]) { + key = blocks[j]; + offset = blocks[j+1]; + j += 2; + } + root_reference rr = {key, offset, i - offset}; + return rr; +} + #ifdef __cplusplus } #endif diff --git a/src/module.c b/src/module.c index 7d13a9760b153..ef77090a39a51 100644 --- a/src/module.c +++ b/src/module.c @@ -60,6 +60,13 @@ uint32_t jl_module_next_counter(jl_module_t *m) return jl_atomic_fetch_add(&m->counter, 1); } +JL_DLLEXPORT jl_module_t *jl_parent_module(jl_module_t *m) +{ + while (m->parent != jl_main_module && m->parent != m) + m = m->parent; + return m; +} + JL_DLLEXPORT jl_value_t *jl_f_new_module(jl_sym_t *name, uint8_t std_imports, uint8_t default_names) { // TODO: should we prohibit this during incremental compilation? @@ -394,6 +401,7 @@ JL_DLLEXPORT jl_binding_t *jl_get_binding_or_error(jl_module_t *m, jl_sym_t *var JL_DLLEXPORT jl_value_t *jl_module_globalref(jl_module_t *m, jl_sym_t *var) { JL_LOCK(&m->lock); + assert(jl_is_module(m)); jl_binding_t *b = (jl_binding_t*)ptrhash_get(&m->bindings, var); if (b == HT_NOTFOUND) { JL_UNLOCK(&m->lock); diff --git a/src/serialize.h b/src/serialize.h index 07ff46fdf96db..7d3a0d537b704 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -62,8 +62,9 @@ extern "C" { #define TAG_GOTOIFNOT 54 #define TAG_RETURNNODE 55 #define TAG_ARGUMENT 56 +#define TAG_EXTERN_METHODROOT 57 -#define LAST_TAG 56 +#define LAST_TAG 57 #define write_uint8(s, n) ios_putc((n), (s)) #define read_uint8(s) ((uint8_t)ios_getc(s)) diff --git a/src/staticdata.c b/src/staticdata.c index 634728991a358..37415a234808c 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -26,7 +26,7 @@ extern "C" { // TODO: put WeakRefs on the weak_refs list during deserialization // TODO: handle finalizers -#define NUM_TAGS 151 +#define NUM_TAGS 152 // An array of references that need to be restored from the sysimg // This is a manually constructed dual of the gvars array, which would be produced by codegen for Julia code, for C. @@ -106,6 +106,7 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_array_symbol_type); INSERT_TAG(jl_array_uint8_type); INSERT_TAG(jl_array_int32_type); + INSERT_TAG(jl_array_uint64_type); INSERT_TAG(jl_int32_type); INSERT_TAG(jl_int64_type); INSERT_TAG(jl_bool_type); diff --git a/src/toplevel.c b/src/toplevel.c index 363d44b3cd642..66ea236b3d8c8 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -33,6 +33,9 @@ JL_DLLEXPORT const char *jl_filename = "none"; // need to update jl_critical_err htable_t jl_current_modules; jl_mutex_t jl_modules_mutex; +// When generating output, the following is used to identify external MethodInstances +JL_DLLEXPORT jl_module_t *jl_precompile_toplevel_module; // the toplevel module currently being defined + JL_DLLEXPORT void jl_add_standard_imports(jl_module_t *m) { jl_module_t *base_module = jl_base_relative_to(m); @@ -138,11 +141,16 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex ptrhash_put(&jl_current_modules, (void*)newm, (void*)((uintptr_t)HT_NOTFOUND + 1)); JL_UNLOCK(&jl_modules_mutex); + jl_module_t *old_toplevel_module = jl_precompile_toplevel_module; + // copy parent environment info into submodule newm->uuid = parent_module->uuid; if (jl_is__toplevel__mod(parent_module)) { newm->parent = newm; jl_register_root_module(newm); + if (jl_options.incremental) { + jl_precompile_toplevel_module = newm; + } } else { newm->parent = parent_module; @@ -261,6 +269,8 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex } } + jl_precompile_toplevel_module = old_toplevel_module; + JL_GC_POP(); return (jl_value_t*)newm; } diff --git a/test/precompile.jl b/test/precompile.jl index 999bd07c9e12b..9c825ca63f06f 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1,6 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license using Test, Distributed, Random +using Base: get_world_counter Foo_module = :Foo4b3a94a1a081a8cb Foo2_module = :F2oo4b3a94a1a081a8cb @@ -535,6 +536,91 @@ precompile_test_harness(false) do dir end end +precompile_test_harness("code caching") do dir + Cache_module = :Cacheb8321416e8a3e2f1 + write(joinpath(dir, "$Cache_module.jl"), + """ + module $Cache_module + struct X end + @noinline f(dest) = push!(dest, X()) + g(dest) = push!(dest, X()) + function callboth() + f([]) + g([]) + g(X[]) + nothing + end + + precompile(callboth, ()) + end + """) + Base.compilecache(Base.PkgId(string(Cache_module))) + @eval using $Cache_module + M = getfield(Main, Cache_module) + for name in (:f, :g, :callboth) + func = getfield(M, name) + m = only(collect(methods(func))) + mi = m.specializations[1] + @test mi.cache isa Core.CodeInstance + @test mi.cache.min_world <= get_world_counter() + @test mi.cache.max_world == typemax(UInt) + if name == :g + @test m.specializations[2] isa Core.MethodInstance # for Vector{Any} and Vector{X} + @test m.specializations[2].cache isa Core.CodeInstance + end + end + mA = which(push!, (Vector{Any}, Any)) + mT = which(push!, (Vector{T} where T, Any)) + for m = (mA, mT) + hasmi = hasci = false + for i = 1:length(m.specializations) + if isassigned(m.specializations, i) + mi = m.specializations[i] + if mi.specTypes.parameters[3] == M.X + hasmi = true + hasci = isdefined(mi, :cache) && isa(mi.cache, Core.CodeInstance) + break + end + end + end + @test hasmi + @test hasci + end + @test M.X ∈ mT.roots + # PkgA loads PkgB, and both add roots to the same method (both before and after loading B) + Cache_module2 = :Cachea1544c83560f0c99 + write(joinpath(dir, "$Cache_module2.jl"), + """ + module $Cache_module2 + struct Y end + @noinline f(dest) = push!(dest, Y()) + callf() = f(Y[]) + callf() + using $(Cache_module) + struct Z end + @noinline g(dest) = push!(dest, Z()) + callg() = g(Z[]) + callg() + end + """) + Base.compilecache(Base.PkgId(string(Cache_module2))) + @eval using $Cache_module2 + M2 = getfield(Main, Cache_module2) + dest = [] + Base.invokelatest() do # use invokelatest to see the results of loading the compile + M2.f(dest) + M.f(dest) + M2.g(dest) + @test dest == [M2.Y(), M.X(), M2.Z()] + @test M2.callf() == [M2.Y()] + @test M2.callg() == [M2.Z()] + @test M.f(M.X[]) == [M.X()] + end + @test M2.Y ∈ mT.roots + @test M2.Z ∈ mT.roots + @test M.X ∈ mT.roots +end + # test --compiled-modules=no command line option precompile_test_harness("--compiled-modules=no") do dir Time_module = :Time4b3a94a1a081a8cb