Faster incremental sysimg rebuilds

base/Base.jl

@@ -458,6 +458,14 @@ function __init__()
     nothing
 end
 
+function __init_build()
+    reinit_stdio()
+    Sys.__init_build()
+    init_depot_path()
+    init_load_path()
+    init_active_project()
+end
+
 end
 
 end # baremodule Base

src/aotcompile.cpp

@@ -54,7 +54,6 @@
 #include <llvm/IR/LegacyPassManagers.h>
 #include <llvm/Transforms/Utils/Cloning.h>
 
-
 using namespace llvm;
 
 // our passes
@@ -181,10 +180,12 @@ static void emit_offset_table(Module &mod, const std::vector<GlobalValue*> &vars
         addrs[i] = ConstantExpr::getBitCast(var, T_psize);
     }
     ArrayType *vars_type = ArrayType::get(T_psize, nvars);
-    new GlobalVariable(mod, vars_type, true,
+    GlobalVariable *GV =
+        new GlobalVariable(mod, vars_type, true,
                        GlobalVariable::ExternalLinkage,
                        ConstantArray::get(vars_type, addrs),
                        name);
+    GV->setSection(JL_SYSIMG_LINK_SECTION);
 }
 
 static bool is_safe_char(unsigned char c)
@@ -276,6 +277,34 @@ static void jl_ci_cache_lookup(const jl_cgparams_t &cgparams, jl_method_instance
     *ci_out = codeinst;
 }
 
+extern FunctionType *jl_func_sig;
+
+StringRef lookup_sysimage_fname(void *ptr, jl_code_instance_t *codeinst)
+{
+    if (ptr == (void*)&jl_fptr_args) {
+        return "jl_fptr_args";
+    } else if (ptr == (void*)&jl_fptr_sparam) {
+        return "jl_fptr_sparam";
+    } else if (ptr == (void*)&jl_fptr_const_return) {
+        return "jl_fptr_const_return";
+    }
+    return jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)ptr, codeinst);
+}
+
+extern Type *T_pjlvalue;
+static void add_gv(void *ctx, void *mod, jl_value_t **gv_slot)
+{
+    jl_codegen_params_t *params = (jl_codegen_params_t*)ctx;
+    Module *M = (Module *)mod;
+
+    GlobalVariable* &lgv = params->globals[*gv_slot];
+    assert(!lgv);
+    lgv = new GlobalVariable(*M, T_pjlvalue,
+                            false, GlobalVariable::PrivateLinkage,
+                            NULL, jl_ExecutionEngine->getGlobalAtAddress((uintptr_t)gv_slot));
+    lgv->setExternallyInitialized(true);
+}
+
 // takes the running content that has collected in the shadow module and dump it to disk
 // this builds the object file portion of the sysimage files for fast startup, and can
 // also be used be extern consumers like GPUCompiler.jl to obtain a module containing
@@ -302,6 +331,10 @@ void *jl_create_native(jl_array_t *methods, const jl_cgparams_t cgparams, int _p
         imaging_mode = 1;
     std::unique_ptr<Module> clone(jl_create_llvm_module("text"));
 
+    if (jl_options.use_sysimage_native_code==JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_CHAINED) {
+        jl_foreach_sysimg_gvar_slot(add_gv, (void*)&params, (void*)clone.get());
+    }
+
     // compile all methods for the current world and type-inference world
     size_t compile_for[] = { jl_typeinf_world, jl_world_counter };
     for (int worlds = 0; worlds < 2; worlds++) {
@@ -330,6 +363,22 @@ void *jl_create_native(jl_array_t *methods, const jl_cgparams_t cgparams, int _p
                 // find and prepare the source code to compile
                 jl_code_instance_t *codeinst = NULL;
                 jl_ci_cache_lookup(cgparams, mi, params.world, &codeinst, &src);
+                if (codeinst) {
+                    jl_llvm_functions_t fnames = {
+                        lookup_sysimage_fname((void*)codeinst->invoke, codeinst).str(),
+                        codeinst->specptr.fptr ?
+                            lookup_sysimage_fname(codeinst->specptr.fptr, codeinst).str() :
+                            "",
+                    };
+                    if (fnames.functionObject.rfind("jsys", 0) != 0 &&
+                        fnames.specFunctionObject.rfind("jsys", 0) != 0) {
+                        // Skip things already in the sysimage, we'll pick it up
+                        // from there.
+                        std::unique_ptr<Module> no_module(nullptr);
+                        emitted[codeinst] = std::make_tuple(std::move(no_module), fnames);
+                        continue;
+                    }
+                }
                 if (src && !emitted.count(codeinst)) {
                     // now add it to our compilation results
                     JL_GC_PROMISE_ROOTED(codeinst->rettype);
@@ -356,11 +405,28 @@ void *jl_create_native(jl_array_t *methods, const jl_cgparams_t cgparams, int _p
     // clones the contents of the module `m` to the shadow_output collector
     // while examining and recording what kind of function pointer we have
     for (auto &def : emitted) {
-        jl_merge_module(clone.get(), std::move(std::get<0>(def.second)));
         jl_code_instance_t *this_code = def.first;
         jl_llvm_functions_t decls = std::get<1>(def.second);
         StringRef func = decls.functionObject;
         StringRef cfunc = decls.specFunctionObject;
+        if (std::get<0>(def.second))
+            jl_merge_module(clone.get(), std::move(std::get<0>(def.second)));
+        else {
+            // TODO: Probably wait until all other modules were merged
+            // TODO: These signatures aren't actually right, but it's not worth
+            // trying to compute signatures for these. Maybe declare them as
+            // void* global variables instead and have jl_merge_module know
+            // how to merge them if it comes to it?
+            assert(!func.empty());
+            Function::Create(jl_func_sig,
+                GlobalVariable::ExternalLinkage,
+                func, clone.get());
+            if (!cfunc.empty()) {
+                Function::Create(jl_func_sig,
+                    GlobalVariable::ExternalLinkage,
+                    cfunc, clone.get());
+            }
+        }
         uint32_t func_id = 0;
         uint32_t cfunc_id = 0;
         if (func == "jl_fptr_args") {
@@ -369,6 +435,9 @@ void *jl_create_native(jl_array_t *methods, const jl_cgparams_t cgparams, int _p
         else if (func == "jl_fptr_sparam") {
             func_id = -2;
         }
+        else if (func == "jl_fptr_const_return") {
+            func_id = -3;
+        }
         else {
             data->jl_sysimg_fvars.push_back(cast<Function>(clone->getNamedValue(func)));
             func_id = data->jl_sysimg_fvars.size();
@@ -389,8 +458,10 @@ void *jl_create_native(jl_array_t *methods, const jl_cgparams_t cgparams, int _p
     // and set them to be internalized and initialized at startup
     for (auto &global : gvars) {
         GlobalVariable *G = cast<GlobalVariable>(clone->getNamedValue(global));
-        G->setInitializer(ConstantPointerNull::get(cast<PointerType>(G->getValueType())));
-        G->setLinkage(GlobalVariable::InternalLinkage);
+        if (!G->isExternallyInitialized())
+            G->setInitializer(ConstantPointerNull::get(cast<PointerType>(G->getValueType())));
+        G->setLinkage(GlobalVariable::ExternalLinkage);
+        G->setVisibility(GlobalVariable::HiddenVisibility);
         data->jl_sysimg_gvars.push_back(G);
     }
 
@@ -409,7 +480,10 @@ void *jl_create_native(jl_array_t *methods, const jl_cgparams_t cgparams, int _p
     if (policy == CompilationPolicy::Default) {
         for (GlobalObject &G : clone->global_objects()) {
             if (!G.isDeclaration()) {
-                G.setLinkage(Function::InternalLinkage);
+                if (G.getLinkage() != GlobalVariable::PrivateLinkage) {
+                    G.setLinkage(Function::ExternalLinkage);
+                    G.setVisibility(GlobalVariable::HiddenVisibility);
+                }
                 makeSafeName(G);
                 addComdat(&G);
 #if defined(_OS_WINDOWS_) && defined(_CPU_X86_64_)
@@ -515,10 +589,13 @@ void jl_dump_native(void *native_code,
     std::vector<NewArchiveMember> unopt_bc_Archive;
     std::vector<std::string> outputs;
 
+    bool sysimg_chained = jl_options.use_sysimage_native_code ==
+        JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_CHAINED;
+
     if (unopt_bc_fname)
         PM.add(createBitcodeWriterPass(unopt_bc_OS));
     if (bc_fname || obj_fname || asm_fname) {
-        addOptimizationPasses(&PM, jl_options.opt_level, true, true);
+        addOptimizationPasses(&PM, jl_options.opt_level, true, true, sysimg_chained);
         addMachinePasses(&PM, TM.get());
     }
     if (bc_fname)
@@ -550,12 +627,15 @@ void jl_dump_native(void *native_code,
         // reflect the address of the jl_RTLD_DEFAULT_handle variable
         // back to the caller, so that we can check for consistency issues
         GlobalValue *jlRTLD_DEFAULT_var = jl_emit_RTLD_DEFAULT_var(data->M.get());
-        addComdat(new GlobalVariable(*data->M,
+        GlobalVariable *jlRTLD_DEFAULT_var_pointer =
+            new GlobalVariable(*data->M,
                                      jlRTLD_DEFAULT_var->getType(),
                                      true,
                                      GlobalVariable::ExternalLinkage,
                                      jlRTLD_DEFAULT_var,
-                                     "jl_RTLD_DEFAULT_handle_pointer"));
+                                     "jl_RTLD_DEFAULT_handle_pointer");
+        jlRTLD_DEFAULT_var_pointer->setSection(JL_SYSIMG_LINK_SECTION);
+        addComdat(jlRTLD_DEFAULT_var_pointer);
     }
 
     // do the actual work
@@ -627,7 +707,7 @@ void addMachinePasses(legacy::PassManagerBase *PM, TargetMachine *TM)
 // this defines the set of optimization passes defined for Julia at various optimization levels.
 // it assumes that the TLI and TTI wrapper passes have already been added.
 void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level,
-                           bool lower_intrinsics, bool dump_native)
+                           bool lower_intrinsics, bool dump_native, bool chained)
 {
 #ifdef JL_DEBUG_BUILD
     PM->add(createGCInvariantVerifierPass(true));
@@ -660,7 +740,7 @@ void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level,
             PM->add(createRemoveNIPass());
         }
         PM->add(createLowerSimdLoopPass()); // Annotate loop marked with "loopinfo" as LLVM parallel loop
-        if (dump_native)
+        if (dump_native && !chained)
             PM->add(createMultiVersioningPass());
 #if defined(JL_ASAN_ENABLED)
         PM->add(createAddressSanitizerFunctionPass());
@@ -695,7 +775,7 @@ void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level,
     // consider AggressiveInstCombinePass at optlevel > 2
     PM->add(createInstructionCombiningPass());
     PM->add(createCFGSimplificationPass());
-    if (dump_native)
+    if (dump_native && !chained)
         PM->add(createMultiVersioningPass());
     PM->add(createSROAPass());
     PM->add(createInstSimplifyLegacyPass());

src/ccall.cpp

@@ -12,7 +12,9 @@ extern const char jl_crtdll_basename[];
 // somewhat unusual variable, in that aotcompile wants to get the address of this for a sanity check
 GlobalVariable *jl_emit_RTLD_DEFAULT_var(Module *M)
 {
-    return prepare_global_in(M, jlRTLD_DEFAULT_var);
+    GlobalVariable *var = prepare_global_in(M, jlRTLD_DEFAULT_var);
+    var->setSection(JL_SYSIMG_LINK_SECTION);
+    return var;
 }
 
 // Find or create the GVs for the library and symbol lookup.

src/cgutils.cpp

@@ -68,6 +68,7 @@ static Value *stringConstPtr(
     StringRef ctxt(txt.c_str(), txt.size() + 1);
     Constant *Data = ConstantDataArray::get(jl_LLVMContext, arrayRefFromStringRef(ctxt));
     GlobalVariable *gv = get_pointer_to_constant(emission_context, Data, "_j_str", *M);
+    gv->setLinkage(GlobalVariable::PrivateLinkage);
     Value *zero = ConstantInt::get(Type::getInt32Ty(jl_LLVMContext), 0);
     Value *Args[] = { zero, zero };
     return irbuilder.CreateInBoundsGEP(gv->getValueType(), gv, Args);
@@ -206,16 +207,25 @@ static Value *julia_pgv(jl_codectx_t &ctx, const char *cname, void *addr)
     if (!gv) {
         raw_string_ostream(gvname) << cname << ctx.global_targets.size();
         localname = StringRef(gvname);
+        gv = new GlobalVariable(*M, T_pjlvalue,
+                                false, GlobalVariable::InternalLinkage,
+                                NULL, localname);
     }
     else {
         localname = gv->getName();
-        if (gv->getParent() != M)
+        if (gv->getParent() != M) {
+            GlobalVariable *oldgv = gv;
             gv = cast_or_null<GlobalVariable>(M->getNamedValue(localname));
+            if (!gv) {
+                gv = new GlobalVariable(*M, T_pjlvalue,
+                                        false, oldgv->getLinkage(),
+                                        NULL, localname);
+                if (oldgv->isExternallyInitialized()) {
+                    gv->setExternallyInitialized(true);
+                }
+            }
+        }
     }
-    if (gv == nullptr)
-        gv = new GlobalVariable(*M, T_pjlvalue,
-                                false, GlobalVariable::PrivateLinkage,
-                                NULL, localname);
     // LLVM passes sometimes strip metadata when moving load around
     // since the load at the new location satisfy the same condition as the origional one.
     // Mark the global as constant to LLVM code using our own metadata
@@ -295,7 +305,7 @@ static Value *literal_pointer_val_slot(jl_codectx_t &ctx, jl_value_t *p)
         return julia_pgv(ctx, "jl_sym#", addr, NULL, p);
     }
     // something else gets just a generic name
-    return julia_pgv(ctx, "jl_global#", p);
+    return julia_pgv(ctx, "jl_global#abc#", p);
 }
 
 static size_t dereferenceable_size(jl_value_t *jt)
@@ -1524,8 +1534,11 @@ static Value *data_pointer(jl_codectx_t &ctx, const jl_cgval_t &x)
     Value *data = x.V;
     if (x.constant) {
         Constant *val = julia_const_to_llvm(ctx, x.constant);
-        if (val)
-            data = get_pointer_to_constant(ctx.emission_context, val, "_j_const", *jl_Module);
+        if (val) {
+            GlobalVariable *gv = get_pointer_to_constant(ctx.emission_context, val, "_j_const", *jl_Module);
+            gv->setLinkage(GlobalVariable::PrivateLinkage);
+            data = gv;
+        }
         else
             data = literal_pointer_val(ctx, x.constant);
     }

src/codegen.cpp

@@ -165,13 +165,13 @@ static DataLayout &jl_data_layout = *(new DataLayout(""));
 
 // types
 static Type *T_jlvalue;
-static Type *T_pjlvalue;
+Type *T_pjlvalue;
 static Type *T_prjlvalue;
 static Type *T_ppjlvalue;
 static Type *T_pprjlvalue;
 static Type *jl_array_llvmt;
 static Type *jl_parray_llvmt;
-static FunctionType *jl_func_sig;
+FunctionType *jl_func_sig;
 static FunctionType *jl_func_sig_sparams;
 static Type *T_pvoidfunc;
 
@@ -1294,7 +1294,9 @@ static inline jl_cgval_t value_to_pointer(jl_codectx_t &ctx, Value *v, jl_value_
 {
     Value *loc;
     if (valid_as_globalinit(v)) { // llvm can't handle all the things that could be inside a ConstantExpr
-        loc = get_pointer_to_constant(ctx.emission_context, cast<Constant>(v), "_j_const", *jl_Module);
+        GlobalVariable *gv = get_pointer_to_constant(ctx.emission_context, cast<Constant>(v), "_j_const", *jl_Module);
+        gv->setLinkage(GlobalVariable::PrivateLinkage);
+        loc = gv;
     }
     else {
         loc = emit_static_alloca(ctx, v->getType());

src/debuginfo.cpp

@@ -734,7 +734,7 @@ openDebugInfo(StringRef debuginfopath, const debug_link_info &info)
             std::move(SplitFile.get()));
 }
 
-static uint64_t jl_sysimage_base;
+uint64_t jl_sysimage_base;
 static jl_sysimg_fptrs_t sysimg_fptrs;
 static jl_method_instance_t **sysimg_fvars_linfo;
 static size_t sysimg_fvars_n;