diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs
index e9e8ade09b77b..45ce24c153bc2 100644
--- a/compiler/rustc_codegen_llvm/src/back/lto.rs
+++ b/compiler/rustc_codegen_llvm/src/back/lto.rs
@@ -45,10 +45,22 @@ pub fn crate_type_allows_lto(crate_type: CrateType) -> bool {
}
}
+struct SerializedModuleInfo {
+ module: SerializedModule<ModuleBuffer>,
+ name: CString,
+ compiler_builtins: bool,
+}
+
+impl SerializedModuleInfo {
+ fn new(module: SerializedModule<ModuleBuffer>, name: CString, compiler_builtins: bool) -> Self {
+ Self { module, name, compiler_builtins }
+ }
+}
+
fn prepare_lto(
cgcx: &CodegenContext<LlvmCodegenBackend>,
dcx: &DiagCtxt,
-) -> Result<(Vec<CString>, Vec<(SerializedModule<ModuleBuffer>, CString)>), FatalError> {
+) -> Result<(Vec<CString>, Vec<SerializedModuleInfo>, Vec<CString>), FatalError> {
let export_threshold = match cgcx.lto {
// We're just doing LTO for our one crate
Lto::ThinLocal => SymbolExportLevel::Rust,
@@ -73,6 +85,17 @@ fn prepare_lto(
};
info!("{} symbols to preserve in this crate", symbols_below_threshold.len());
+ let compiler_builtins_exported_symbols = match cgcx.compiler_builtins {
+ Some(crate_num) => {
+ if let Some(exported_symbols) = exported_symbols.get(&crate_num) {
+ exported_symbols.iter().filter_map(symbol_filter).collect::<Vec<CString>>()
+ } else {
+ Vec::new()
+ }
+ }
+ None => Vec::new(),
+ };
+
// If we're performing LTO for the entire crate graph, then for each of our
// upstream dependencies, find the corresponding rlib and load the bitcode
// from the archive.
@@ -127,6 +150,7 @@ fn prepare_lto(
})
})
.filter(|&(name, _)| looks_like_rust_object_file(name));
+ let compiler_builtins = cgcx.compiler_builtins == Some(cnum);
for (name, child) in obj_files {
info!("adding bitcode from {}", name);
match get_bitcode_slice_from_object_data(
@@ -135,7 +159,11 @@ fn prepare_lto(
) {
Ok(data) => {
let module = SerializedModule::FromRlib(data.to_vec());
- upstream_modules.push((module, CString::new(name).unwrap()));
+ upstream_modules.push(SerializedModuleInfo::new(
+ module,
+ CString::new(name).unwrap(),
+ compiler_builtins,
+ ));
}
Err(e) => {
dcx.emit_err(e);
@@ -150,7 +178,7 @@ fn prepare_lto(
// __llvm_profile_runtime, therefore we won't know until link time if this symbol
// should have default visibility.
symbols_below_threshold.push(CString::new("__llvm_profile_counter_bias").unwrap());
- Ok((symbols_below_threshold, upstream_modules))
+ Ok((symbols_below_threshold, upstream_modules, compiler_builtins_exported_symbols))
}
fn get_bitcode_slice_from_object_data<'a>(
@@ -201,10 +229,21 @@ pub(crate) fn run_fat(
cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
) -> Result<LtoModuleCodegen<LlvmCodegenBackend>, FatalError> {
let dcx = cgcx.create_dcx();
- let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &dcx)?;
+ let (symbols_below_threshold, upstream_modules, compiler_builtins_exported_symbols) =
+ prepare_lto(cgcx, &dcx)?;
let symbols_below_threshold =
symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();
- fat_lto(cgcx, &dcx, modules, cached_modules, upstream_modules, &symbols_below_threshold)
+ let compiler_builtins_exported_symbols =
+ compiler_builtins_exported_symbols.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();
+ fat_lto(
+ cgcx,
+ &dcx,
+ modules,
+ cached_modules,
+ upstream_modules,
+ &symbols_below_threshold,
+ &compiler_builtins_exported_symbols,
+ )
}
/// Performs thin LTO by performing necessary global analysis and returning two
@@ -216,7 +255,7 @@ pub(crate) fn run_thin(
cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
) -> Result<(Vec<LtoModuleCodegen<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> {
let dcx = cgcx.create_dcx();
- let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &dcx)?;
+ let (symbols_below_threshold, upstream_modules, _) = prepare_lto(cgcx, &dcx)?;
let symbols_below_threshold =
symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();
if cgcx.opts.cg.linker_plugin_lto.enabled() {
@@ -239,8 +278,9 @@ fn fat_lto(
dcx: &DiagCtxt,
modules: Vec<FatLtoInput<LlvmCodegenBackend>>,
cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
- mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
+ mut serialized_modules: Vec<SerializedModuleInfo>,
symbols_below_threshold: &[*const libc::c_char],
+ compiler_builtins_exported_symbols: &[*const libc::c_char],
) -> Result<LtoModuleCodegen<LlvmCodegenBackend>, FatalError> {
let _timer = cgcx.prof.generic_activity("LLVM_fat_lto_build_monolithic_module");
info!("going for a fat lto");
@@ -258,7 +298,7 @@ fn fat_lto(
let mut in_memory = Vec::new();
serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| {
info!("pushing cached module {:?}", wp.cgu_name);
- (buffer, CString::new(wp.cgu_name).unwrap())
+ SerializedModuleInfo::new(buffer, CString::new(wp.cgu_name).unwrap(), false)
}));
for module in modules {
match module {
@@ -266,7 +306,11 @@ fn fat_lto(
FatLtoInput::Serialized { name, buffer } => {
info!("pushing serialized module {:?}", name);
let buffer = SerializedModule::Local(buffer);
- serialized_modules.push((buffer, CString::new(name).unwrap()));
+ serialized_modules.push(SerializedModuleInfo::new(
+ buffer,
+ CString::new(name).unwrap(),
+ false,
+ ));
}
}
}
@@ -299,10 +343,10 @@ fn fat_lto(
Some((_cost, i)) => in_memory.remove(i),
None => {
assert!(!serialized_modules.is_empty(), "must have at least one serialized module");
- let (buffer, name) = serialized_modules.remove(0);
+ let SerializedModuleInfo { module, name, .. } = serialized_modules.remove(0);
info!("no in-memory regular modules to choose from, parsing {:?}", name);
ModuleCodegen {
- module_llvm: ModuleLlvm::parse(cgcx, &name, buffer.data(), dcx)?,
+ module_llvm: ModuleLlvm::parse(cgcx, &name, module.data(), dcx)?,
name: name.into_string().unwrap(),
kind: ModuleKind::Regular,
}
@@ -330,26 +374,41 @@ fn fat_lto(
for module in in_memory {
let buffer = ModuleBuffer::new(module.module_llvm.llmod());
let llmod_id = CString::new(&module.name[..]).unwrap();
- serialized_modules.push((SerializedModule::Local(buffer), llmod_id));
+ serialized_modules.push(SerializedModuleInfo::new(
+ SerializedModule::Local(buffer),
+ llmod_id,
+ false,
+ ));
}
// Sort the modules to ensure we produce deterministic results.
- serialized_modules.sort_by(|module1, module2| module1.1.cmp(&module2.1));
+ // Place compiler_builtins at the end. After that we check if any crate wants to
+ // customize the builtin functions. Remove the function of compiler_builtins, if any.
+ serialized_modules.sort_by(|module1, module2| {
+ let compiler_builtins_cmp = module1.compiler_builtins.cmp(&module2.compiler_builtins);
+ if compiler_builtins_cmp.is_ne() {
+ return compiler_builtins_cmp;
+ }
+ module1.name.cmp(&module2.name)
+ });
// For all serialized bitcode files we parse them and link them in as we did
// above, this is all mostly handled in C++. Like above, though, we don't
// know much about the memory management here so we err on the side of being
// save and persist everything with the original module.
- let mut linker = Linker::new(llmod);
- for (bc_decoded, name) in serialized_modules {
+ let mut linker = Linker::new(llmod, compiler_builtins_exported_symbols);
+ for serialized_module in serialized_modules {
+ let SerializedModuleInfo { module, name, compiler_builtins } = serialized_module;
let _timer = cgcx
.prof
.generic_activity_with_arg_recorder("LLVM_fat_lto_link_module", |recorder| {
recorder.record_arg(format!("{name:?}"))
});
info!("linking {:?}", name);
- let data = bc_decoded.data();
- linker.add(data).map_err(|()| write::llvm_err(dcx, LlvmError::LoadBitcode { name }))?;
- serialized_bitcode.push(bc_decoded);
+ let data = module.data();
+ linker
+ .add(data, compiler_builtins)
+ .map_err(|()| write::llvm_err(dcx, LlvmError::LoadBitcode { name }))?;
+ serialized_bitcode.push(module);
}
drop(linker);
save_temp_bitcode(cgcx, &module, "lto.input");
@@ -372,16 +431,24 @@ fn fat_lto(
pub(crate) struct Linker<'a>(&'a mut llvm::Linker<'a>);
impl<'a> Linker<'a> {
- pub(crate) fn new(llmod: &'a llvm::Module) -> Self {
- unsafe { Linker(llvm::LLVMRustLinkerNew(llmod)) }
+ pub(crate) fn new(llmod: &'a llvm::Module, builtin_syms: &[*const libc::c_char]) -> Self {
+ let ptr = builtin_syms.as_ptr();
+ unsafe {
+ Linker(llvm::LLVMRustLinkerNew(
+ llmod,
+ ptr as *const *const libc::c_char,
+ builtin_syms.len() as libc::size_t,
+ ))
+ }
}
- pub(crate) fn add(&mut self, bytecode: &[u8]) -> Result<(), ()> {
+ pub(crate) fn add(&mut self, bytecode: &[u8], compiler_builtins: bool) -> Result<(), ()> {
unsafe {
if llvm::LLVMRustLinkerAdd(
self.0,
bytecode.as_ptr() as *const libc::c_char,
bytecode.len(),
+ compiler_builtins,
) {
Ok(())
} else {
@@ -433,7 +500,7 @@ fn thin_lto(
cgcx: &CodegenContext<LlvmCodegenBackend>,
dcx: &DiagCtxt,
modules: Vec<(String, ThinBuffer)>,
- serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
+ serialized_modules: Vec<SerializedModuleInfo>,
cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
symbols_below_threshold: &[*const libc::c_char],
) -> Result<(Vec<LtoModuleCodegen<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> {
@@ -479,10 +546,12 @@ fn thin_lto(
// we must always unconditionally look at the index).
let mut serialized = Vec::with_capacity(serialized_modules.len() + cached_modules.len());
- let cached_modules =
- cached_modules.into_iter().map(|(sm, wp)| (sm, CString::new(wp.cgu_name).unwrap()));
+ let cached_modules = cached_modules.into_iter().map(|(sm, wp)| {
+ SerializedModuleInfo::new(sm, CString::new(wp.cgu_name).unwrap(), false)
+ });
- for (module, name) in serialized_modules.into_iter().chain(cached_modules) {
+ for serialized_module in serialized_modules.into_iter().chain(cached_modules) {
+ let SerializedModuleInfo { module, name, .. } = serialized_module;
info!("upstream or cached module {:?}", name);
thin_modules.push(llvm::ThinLTOModule {
identifier: name.as_ptr(),
diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs
index c607533a08ed0..ff089f3d7d117 100644
--- a/compiler/rustc_codegen_llvm/src/back/write.rs
+++ b/compiler/rustc_codegen_llvm/src/back/write.rs
@@ -634,12 +634,12 @@ pub(crate) fn link(
let (first, elements) =
modules.split_first().expect("Bug! modules must contain at least one module.");
- let mut linker = Linker::new(first.module_llvm.llmod());
+ let mut linker = Linker::new(first.module_llvm.llmod(), &[]);
for module in elements {
let _timer = cgcx.prof.generic_activity_with_arg("LLVM_link_module", &*module.name);
let buffer = ModuleBuffer::new(module.module_llvm.llmod());
linker
- .add(buffer.data())
+ .add(buffer.data(), false)
.map_err(|()| llvm_err(dcx, LlvmError::SerializeModule { name: &module.name }))?;
}
drop(linker);
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
index 81702baa8c053..4bd7f7b76bd96 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -2356,11 +2356,16 @@ extern "C" {
out_len: &mut usize,
) -> *const u8;
- pub fn LLVMRustLinkerNew(M: &Module) -> &mut Linker<'_>;
+ pub fn LLVMRustLinkerNew(
+ M: &Module,
+ builtin_syms: *const *const c_char,
+ len: size_t,
+ ) -> &mut Linker<'_>;
pub fn LLVMRustLinkerAdd(
linker: &Linker<'_>,
bytecode: *const c_char,
bytecode_len: usize,
+ compiler_builtins: bool,
) -> bool;
pub fn LLVMRustLinkerFree<'a>(linker: &'a mut Linker<'a>);
#[allow(improper_ctypes)]
diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs
index 5a8db7bbf2d67..fb94d6d44b844 100644
--- a/compiler/rustc_codegen_ssa/src/back/write.rs
+++ b/compiler/rustc_codegen_ssa/src/back/write.rs
@@ -326,6 +326,7 @@ pub struct CodegenContext<B: WriteBackendMethods> {
pub opts: Arc<config::Options>,
pub crate_types: Vec<CrateType>,
pub each_linked_rlib_for_lto: Vec<(CrateNum, PathBuf)>,
+ pub compiler_builtins: Option<CrateNum>,
pub output_filenames: Arc<OutputFilenames>,
pub regular_module_config: Arc<ModuleConfig>,
pub metadata_module_config: Arc<ModuleConfig>,
@@ -1089,6 +1090,7 @@ fn start_executing_work<B: ExtraBackendMethods>(
let cgcx = CodegenContext::<B> {
crate_types: tcx.crate_types().to_vec(),
each_linked_rlib_for_lto,
+ compiler_builtins: crate_info.compiler_builtins,
lto: sess.lto(),
fewer_names: sess.fewer_names(),
save_temps: sess.opts.cg.save_temps,
diff --git a/compiler/rustc_llvm/llvm-wrapper/Linker.cpp b/compiler/rustc_llvm/llvm-wrapper/Linker.cpp
index 533df0f75f8f5..421ae607802fe 100644
--- a/compiler/rustc_llvm/llvm-wrapper/Linker.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/Linker.cpp
@@ -1,25 +1,709 @@
+// Derived from code in LLVM, which is:
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Derived from:
+// * https://github.com/llvm/llvm-project/blob/6009708b4367171ccdbf4b5905cb6a803753fe18/llvm/include/llvm/Linker/Linker.h
+// * https://github.com/llvm/llvm-project/blob/6009708b4367171ccdbf4b5905cb6a803753fe18/llvm/lib/Linker/LinkModules.cpp
+// * https://github.com/llvm/llvm-project/blob/6009708b4367171ccdbf4b5905cb6a803753fe18/llvm/lib/Linker/LinkDiagnosticInfo.h
+// * https://github.com/llvm/llvm-project/blob/6009708b4367171ccdbf4b5905cb6a803753fe18/llvm/lib/Linker/IRMover.cpp
+
#include "SuppressLLVMWarnings.h"
-#include "llvm/Linker/Linker.h"
+
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/DiagnosticPrinter.h"
+#include "llvm/Linker/IRMover.h"
+#include "llvm/Object/ModuleSymbolTable.h"
+#include "llvm/Support/Error.h"
#include "LLVMWrapper.h"
using namespace llvm;
+namespace {
+class LinkDiagnosticInfo : public DiagnosticInfo {
+ const Twine &Msg;
+
+public:
+ LinkDiagnosticInfo(DiagnosticSeverity Severity, const Twine &Msg);
+ void print(DiagnosticPrinter &DP) const override;
+};
+
+LinkDiagnosticInfo::LinkDiagnosticInfo(DiagnosticSeverity Severity,
+ const Twine &Msg)
+ : DiagnosticInfo(DK_Linker, Severity), Msg(Msg) {}
+void LinkDiagnosticInfo::print(DiagnosticPrinter &DP) const { DP << Msg; }
+
+} // namespace
+
+namespace {
+
+enum class LinkFrom { Dst, Src, Both };
+
+/// This is an implementation class for the LinkModules function, which is the
+/// entrypoint for this file.
+class ModuleLinker {
+ IRMover &Mover;
+ const StringSet<> &CompilerBuiltinsSymbols;
+ StringSet<> UserBuiltinsSymbols;
+ std::unique_ptr<Module> SrcM;
+ bool SrcIsCompilerBuiltins;
+ bool HasLinkedCompilerBuiltins = false;
+
+ SetVector<GlobalValue *> ValuesToLink;
+
+ /// For symbol clashes, prefer those from Src.
+ unsigned Flags;
+
+ /// List of global value names that should be internalized.
+ StringSet<> Internalize;
+
+ /// Function that will perform the actual internalization. The reason for a
+ /// callback is that the linker cannot call internalizeModule without
+ /// creating a circular dependency between IPO and the linker.
+ std::function<void(Module &, const StringSet<> &)> InternalizeCallback;
+
+ /// Used as the callback for lazy linking.
+ /// The mover has just hit GV and we have to decide if it, and other members
+ /// of the same comdat, should be linked. Every member to be linked is passed
+ /// to Add.
+ void addLazyFor(GlobalValue &GV, const IRMover::ValueAdder &Add);
+
+ bool shouldOverrideFromSrc() { return Flags & Linker::OverrideFromSrc; }
+ bool shouldLinkOnlyNeeded() { return Flags & Linker::LinkOnlyNeeded; }
+
+ bool shouldLinkFromSource(bool &LinkFromSrc, const GlobalValue &Dest,
+ const GlobalValue &Src);
+
+ /// Should we have mover and linker error diag info?
+ bool emitError(const Twine &Message) {
+ SrcM->getContext().diagnose(LinkDiagnosticInfo(DS_Error, Message));
+ return true;
+ }
+
+ bool getComdatLeader(Module &M, StringRef ComdatName,
+ const GlobalVariable *&GVar);
+ bool computeResultingSelectionKind(StringRef ComdatName,
+ Comdat::SelectionKind Src,
+ Comdat::SelectionKind Dst,
+ Comdat::SelectionKind &Result,
+ LinkFrom &From);
+ DenseMap<const Comdat *, std::pair<Comdat::SelectionKind, LinkFrom>>
+ ComdatsChosen;
+ bool getComdatResult(const Comdat *SrcC, Comdat::SelectionKind &SK,
+ LinkFrom &From);
+ // Keep track of the lazy linked global members of each comdat in source.
+ DenseMap<const Comdat *, std::vector<GlobalValue *>> LazyComdatMembers;
+
+ /// Given a global in the source module, return the global in the
+ /// destination module that is being linked to, if any.
+ GlobalValue *getLinkedToGlobal(const GlobalValue *SrcGV) {
+ Module &DstM = Mover.getModule();
+ // If the source has no name it can't link. If it has local linkage,
+ // there is no name match-up going on.
+ if (!SrcGV->hasName() || GlobalValue::isLocalLinkage(SrcGV->getLinkage()))
+ return nullptr;
+
+ // Otherwise see if we have a match in the destination module's symtab.
+ GlobalValue *DGV = DstM.getNamedValue(SrcGV->getName());
+ if (!DGV)
+ return nullptr;
+
+ // If we found a global with the same name in the dest module, but it has
+ // internal linkage, we are really not doing any linkage here.
+ if (DGV->hasLocalLinkage())
+ return nullptr;
+
+ // Otherwise, we do in fact link to the destination global.
+ return DGV;
+ }
+
+ /// Drop GV if it is a member of a comdat that we are dropping.
+ /// This can happen with COFF's largest selection kind.
+ void dropReplacedComdat(GlobalValue &GV,
+ const DenseSet<const Comdat *> &ReplacedDstComdats);
+
+ bool linkIfNeeded(GlobalValue &GV, SmallVectorImpl<GlobalValue *> &GVToClone);
+
+public:
+ ModuleLinker(IRMover &Mover, const StringSet<> &CompilerBuiltinsSymbols,
+ std::unique_ptr<Module> SrcM, bool SrcIsCompilerBuiltins,
+ unsigned Flags,
+ std::function<void(Module &, const StringSet<> &)>
+ InternalizeCallback = {})
+ : Mover(Mover), CompilerBuiltinsSymbols(CompilerBuiltinsSymbols),
+ SrcM(std::move(SrcM)), SrcIsCompilerBuiltins(SrcIsCompilerBuiltins),
+ Flags(Flags), InternalizeCallback(std::move(InternalizeCallback)) {}
+
+ bool run();
+};
+} // namespace
+
+static GlobalValue::VisibilityTypes
+getMinVisibility(GlobalValue::VisibilityTypes A,
+ GlobalValue::VisibilityTypes B) {
+ if (A == GlobalValue::HiddenVisibility || B == GlobalValue::HiddenVisibility)
+ return GlobalValue::HiddenVisibility;
+ if (A == GlobalValue::ProtectedVisibility ||
+ B == GlobalValue::ProtectedVisibility)
+ return GlobalValue::ProtectedVisibility;
+ return GlobalValue::DefaultVisibility;
+}
+
+bool ModuleLinker::getComdatLeader(Module &M, StringRef ComdatName,
+ const GlobalVariable *&GVar) {
+ const GlobalValue *GVal = M.getNamedValue(ComdatName);
+ if (const auto *GA = dyn_cast_or_null<GlobalAlias>(GVal)) {
+ GVal = GA->getAliaseeObject();
+ if (!GVal)
+ // We cannot resolve the size of the aliasee yet.
+ return emitError("Linking COMDATs named '" + ComdatName +
+ "': COMDAT key involves incomputable alias size.");
+ }
+
+ GVar = dyn_cast_or_null<GlobalVariable>(GVal);
+ if (!GVar)
+ return emitError(
+ "Linking COMDATs named '" + ComdatName +
+ "': GlobalVariable required for data dependent selection!");
+
+ return false;
+}
+
+bool ModuleLinker::computeResultingSelectionKind(StringRef ComdatName,
+ Comdat::SelectionKind Src,
+ Comdat::SelectionKind Dst,
+ Comdat::SelectionKind &Result,
+ LinkFrom &From) {
+ Module &DstM = Mover.getModule();
+ // The ability to mix Comdat::SelectionKind::Any with
+ // Comdat::SelectionKind::Largest is a behavior that comes from COFF.
+ bool DstAnyOrLargest = Dst == Comdat::SelectionKind::Any ||
+ Dst == Comdat::SelectionKind::Largest;
+ bool SrcAnyOrLargest = Src == Comdat::SelectionKind::Any ||
+ Src == Comdat::SelectionKind::Largest;
+ if (DstAnyOrLargest && SrcAnyOrLargest) {
+ if (Dst == Comdat::SelectionKind::Largest ||
+ Src == Comdat::SelectionKind::Largest)
+ Result = Comdat::SelectionKind::Largest;
+ else
+ Result = Comdat::SelectionKind::Any;
+ } else if (Src == Dst) {
+ Result = Dst;
+ } else {
+ return emitError("Linking COMDATs named '" + ComdatName +
+ "': invalid selection kinds!");
+ }
+
+ switch (Result) {
+ case Comdat::SelectionKind::Any:
+ // Go with Dst.
+ From = LinkFrom::Dst;
+ break;
+ case Comdat::SelectionKind::NoDeduplicate:
+ From = LinkFrom::Both;
+ break;
+ case Comdat::SelectionKind::ExactMatch:
+ case Comdat::SelectionKind::Largest:
+ case Comdat::SelectionKind::SameSize: {
+ const GlobalVariable *DstGV;
+ const GlobalVariable *SrcGV;
+ if (getComdatLeader(DstM, ComdatName, DstGV) ||
+ getComdatLeader(*SrcM, ComdatName, SrcGV))
+ return true;
+
+ const DataLayout &DstDL = DstM.getDataLayout();
+ const DataLayout &SrcDL = SrcM->getDataLayout();
+ uint64_t DstSize = DstDL.getTypeAllocSize(DstGV->getValueType());
+ uint64_t SrcSize = SrcDL.getTypeAllocSize(SrcGV->getValueType());
+ if (Result == Comdat::SelectionKind::ExactMatch) {
+ if (SrcGV->getInitializer() != DstGV->getInitializer())
+ return emitError("Linking COMDATs named '" + ComdatName +
+ "': ExactMatch violated!");
+ From = LinkFrom::Dst;
+ } else if (Result == Comdat::SelectionKind::Largest) {
+ From = SrcSize > DstSize ? LinkFrom::Src : LinkFrom::Dst;
+ } else if (Result == Comdat::SelectionKind::SameSize) {
+ if (SrcSize != DstSize)
+ return emitError("Linking COMDATs named '" + ComdatName +
+ "': SameSize violated!");
+ From = LinkFrom::Dst;
+ } else {
+ llvm::report_fatal_error("unknown selection kind");
+ }
+ break;
+ }
+ }
+
+ return false;
+}
+
+bool ModuleLinker::getComdatResult(const Comdat *SrcC,
+ Comdat::SelectionKind &Result,
+ LinkFrom &From) {
+ Module &DstM = Mover.getModule();
+ Comdat::SelectionKind SSK = SrcC->getSelectionKind();
+ StringRef ComdatName = SrcC->getName();
+ Module::ComdatSymTabType &ComdatSymTab = DstM.getComdatSymbolTable();
+ Module::ComdatSymTabType::iterator DstCI = ComdatSymTab.find(ComdatName);
+
+ if (DstCI == ComdatSymTab.end()) {
+ // Use the comdat if it is only available in one of the modules.
+ From = LinkFrom::Src;
+ Result = SSK;
+ return false;
+ }
+
+ const Comdat *DstC = &DstCI->second;
+ Comdat::SelectionKind DSK = DstC->getSelectionKind();
+ return computeResultingSelectionKind(ComdatName, SSK, DSK, Result, From);
+}
+
+bool ModuleLinker::shouldLinkFromSource(bool &LinkFromSrc,
+ const GlobalValue &Dest,
+ const GlobalValue &Src) {
+
+ // Should we unconditionally use the Src?
+ if (shouldOverrideFromSrc()) {
+ LinkFromSrc = true;
+ return false;
+ }
+
+ // We always have to add Src if it has appending linkage.
+ if (Src.hasAppendingLinkage() || Dest.hasAppendingLinkage()) {
+ LinkFromSrc = true;
+ return false;
+ }
+
+ bool SrcIsDeclaration = Src.isDeclarationForLinker();
+ bool DestIsDeclaration = Dest.isDeclarationForLinker();
+
+ if (SrcIsDeclaration) {
+ // If Src is external or if both Src & Dest are external.. Just link the
+ // external globals, we aren't adding anything.
+ if (Src.hasDLLImportStorageClass()) {
+ // If one of GVs is marked as DLLImport, result should be dllimport'ed.
+ LinkFromSrc = DestIsDeclaration;
+ return false;
+ }
+ // If the Dest is weak, use the source linkage.
+ if (Dest.hasExternalWeakLinkage()) {
+ LinkFromSrc = true;
+ return false;
+ }
+ // Link an available_externally over a declaration.
+ LinkFromSrc = !Src.isDeclaration() && Dest.isDeclaration();
+ return false;
+ }
+
+ if (DestIsDeclaration) {
+ // If Dest is external but Src is not:
+ LinkFromSrc = true;
+ return false;
+ }
+
+ if (Src.hasCommonLinkage()) {
+ if (Dest.hasLinkOnceLinkage() || Dest.hasWeakLinkage()) {
+ LinkFromSrc = true;
+ return false;
+ }
+
+ if (!Dest.hasCommonLinkage()) {
+ LinkFromSrc = false;
+ return false;
+ }
+
+ const DataLayout &DL = Dest.getParent()->getDataLayout();
+ uint64_t DestSize = DL.getTypeAllocSize(Dest.getValueType());
+ uint64_t SrcSize = DL.getTypeAllocSize(Src.getValueType());
+ LinkFromSrc = SrcSize > DestSize;
+ return false;
+ }
+
+ if (Src.isWeakForLinker()) {
+ assert(!Dest.hasExternalWeakLinkage());
+ assert(!Dest.hasAvailableExternallyLinkage());
+
+ if (Dest.hasLinkOnceLinkage() && Src.hasWeakLinkage()) {
+ LinkFromSrc = true;
+ return false;
+ }
+
+ LinkFromSrc = false;
+ return false;
+ }
+
+ if (Dest.isWeakForLinker()) {
+ assert(Src.hasExternalLinkage());
+ LinkFromSrc = true;
+ return false;
+ }
+
+ assert(!Src.hasExternalWeakLinkage());
+ assert(!Dest.hasExternalWeakLinkage());
+ assert(Dest.hasExternalLinkage() && Src.hasExternalLinkage() &&
+ "Unexpected linkage type!");
+ return emitError("Linking globals named '" + Src.getName() +
+ "': symbol multiply defined!");
+}
+
+bool ModuleLinker::linkIfNeeded(GlobalValue &GV,
+ SmallVectorImpl<GlobalValue *> &GVToClone) {
+ // If a builtin symbol is defined in a non-compiler-builtins, the symbol of
+ // compiler-builtins is a non-prevailing symbol.
+ if (SrcIsCompilerBuiltins && UserBuiltinsSymbols.contains(GV.getName()))
+ return false;
+ GlobalValue *DGV = getLinkedToGlobal(&GV);
+
+ if (shouldLinkOnlyNeeded()) {
+ // Always import variables with appending linkage.
+ if (!GV.hasAppendingLinkage()) {
+ // Don't import globals unless they are referenced by the destination
+ // module.
+ if (!DGV)
+ return false;
+ // Don't import globals that are already defined in the destination module
+ if (!DGV->isDeclaration())
+ return false;
+ }
+ }
+
+ if (DGV && !GV.hasLocalLinkage() && !GV.hasAppendingLinkage()) {
+ auto *DGVar = dyn_cast<GlobalVariable>(DGV);
+ auto *SGVar = dyn_cast<GlobalVariable>(&GV);
+ if (DGVar && SGVar) {
+ if (DGVar->isDeclaration() && SGVar->isDeclaration() &&
+ (!DGVar->isConstant() || !SGVar->isConstant())) {
+ DGVar->setConstant(false);
+ SGVar->setConstant(false);
+ }
+ if (DGVar->hasCommonLinkage() && SGVar->hasCommonLinkage()) {
+ MaybeAlign DAlign = DGVar->getAlign();
+ MaybeAlign SAlign = SGVar->getAlign();
+ MaybeAlign Align = std::nullopt;
+ if (DAlign || SAlign)
+ Align = std::max(DAlign.valueOrOne(), SAlign.valueOrOne());
+
+ SGVar->setAlignment(Align);
+ DGVar->setAlignment(Align);
+ }
+ }
+
+ GlobalValue::VisibilityTypes Visibility =
+ getMinVisibility(DGV->getVisibility(), GV.getVisibility());
+ DGV->setVisibility(Visibility);
+ GV.setVisibility(Visibility);
+
+ GlobalValue::UnnamedAddr UnnamedAddr = GlobalValue::getMinUnnamedAddr(
+ DGV->getUnnamedAddr(), GV.getUnnamedAddr());
+ DGV->setUnnamedAddr(UnnamedAddr);
+ GV.setUnnamedAddr(UnnamedAddr);
+ }
+
+ if (!DGV && !shouldOverrideFromSrc() &&
+ (GV.hasLocalLinkage() || GV.hasLinkOnceLinkage() ||
+ GV.hasAvailableExternallyLinkage()))
+ return false;
+
+ if (GV.isDeclaration())
+ return false;
+
+ LinkFrom ComdatFrom = LinkFrom::Dst;
+ if (const Comdat *SC = GV.getComdat()) {
+ std::tie(std::ignore, ComdatFrom) = ComdatsChosen[SC];
+ if (ComdatFrom == LinkFrom::Dst)
+ return false;
+ }
+
+ bool LinkFromSrc = true;
+ if (DGV && shouldLinkFromSource(LinkFromSrc, *DGV, GV))
+ return true;
+ if (DGV && ComdatFrom == LinkFrom::Both)
+ GVToClone.push_back(LinkFromSrc ? DGV : &GV);
+ if (LinkFromSrc)
+ ValuesToLink.insert(&GV);
+ return false;
+}
+
+void ModuleLinker::addLazyFor(GlobalValue &GV, const IRMover::ValueAdder &Add) {
+ // Add these to the internalize list
+ if (!GV.hasLinkOnceLinkage() && !GV.hasAvailableExternallyLinkage() &&
+ !shouldLinkOnlyNeeded())
+ return;
+
+ if (InternalizeCallback)
+ Internalize.insert(GV.getName());
+ Add(GV);
+
+ const Comdat *SC = GV.getComdat();
+ if (!SC)
+ return;
+ for (GlobalValue *GV2 : LazyComdatMembers[SC]) {
+ GlobalValue *DGV = getLinkedToGlobal(GV2);
+ bool LinkFromSrc = true;
+ if (DGV && shouldLinkFromSource(LinkFromSrc, *DGV, *GV2))
+ return;
+ if (!LinkFromSrc)
+ continue;
+ if (InternalizeCallback)
+ Internalize.insert(GV2->getName());
+ Add(*GV2);
+ }
+}
+
+void ModuleLinker::dropReplacedComdat(
+ GlobalValue &GV, const DenseSet<const Comdat *> &ReplacedDstComdats) {
+ Comdat *C = GV.getComdat();
+ if (!C)
+ return;
+ if (!ReplacedDstComdats.count(C))
+ return;
+ if (GV.use_empty()) {
+ GV.eraseFromParent();
+ return;
+ }
+
+ if (auto *F = dyn_cast<Function>(&GV)) {
+ F->deleteBody();
+ } else if (auto *Var = dyn_cast<GlobalVariable>(&GV)) {
+ Var->setInitializer(nullptr);
+ } else {
+ auto &Alias = cast<GlobalAlias>(GV);
+ Module &M = *Alias.getParent();
+ GlobalValue *Declaration;
+ if (auto *FTy = dyn_cast<FunctionType>(Alias.getValueType())) {
+ Declaration = Function::Create(FTy, GlobalValue::ExternalLinkage, "", &M);
+ } else {
+ Declaration =
+ new GlobalVariable(M, Alias.getValueType(), /*isConstant*/ false,
+ GlobalValue::ExternalLinkage,
+ /*Initializer*/ nullptr);
+ }
+ Declaration->takeName(&Alias);
+ Alias.replaceAllUsesWith(Declaration);
+ Alias.eraseFromParent();
+ }
+}
+
+bool ModuleLinker::run() {
+ Module &DstM = Mover.getModule();
+ DenseSet<const Comdat *> ReplacedDstComdats;
+
+ for (const auto &SMEC : SrcM->getComdatSymbolTable()) {
+ const Comdat &C = SMEC.getValue();
+ if (ComdatsChosen.count(&C))
+ continue;
+ Comdat::SelectionKind SK;
+ LinkFrom From;
+ if (getComdatResult(&C, SK, From))
+ return true;
+ ComdatsChosen[&C] = std::make_pair(SK, From);
+
+ if (From != LinkFrom::Src)
+ continue;
+
+ Module::ComdatSymTabType &ComdatSymTab = DstM.getComdatSymbolTable();
+ Module::ComdatSymTabType::iterator DstCI = ComdatSymTab.find(C.getName());
+ if (DstCI == ComdatSymTab.end())
+ continue;
+
+ // The source comdat is replacing the dest one.
+ const Comdat *DstC = &DstCI->second;
+ ReplacedDstComdats.insert(DstC);
+ }
+
+ if (!SrcIsCompilerBuiltins && HasLinkedCompilerBuiltins)
+ llvm::report_fatal_error(
+ "Expect only compiler-builtins to be linked at the end.");
+ // We promise that compiler-builtins is linked at the end, so we only need to
+ // compute it once.
+ if (SrcIsCompilerBuiltins && !HasLinkedCompilerBuiltins) {
+ ModuleSymbolTable SymbolTable;
+ SymbolTable.addModule(&DstM);
+ for (auto &Sym : SymbolTable.symbols()) {
+ uint32_t Flags = SymbolTable.getSymbolFlags(Sym);
+ if ((Flags & object::BasicSymbolRef::SF_Weak) ||
+ !(Flags & object::BasicSymbolRef::SF_Global))
+ continue;
+ if (GlobalValue *GV = dyn_cast_if_present<GlobalValue *>(Sym)) {
+ if (CompilerBuiltinsSymbols.contains(GV->getName()))
+ UserBuiltinsSymbols.insert(GV->getName());
+ } else if (auto *AS =
+ dyn_cast_if_present<ModuleSymbolTable::AsmSymbol *>(Sym)) {
+ if (CompilerBuiltinsSymbols.contains(AS->first))
+ UserBuiltinsSymbols.insert(AS->first);
+ } else {
+ llvm::report_fatal_error("unknown symbol type");
+ }
+ }
+ }
+
+ // Alias have to go first, since we are not able to find their comdats
+ // otherwise.
+ for (GlobalAlias &GV : llvm::make_early_inc_range(DstM.aliases()))
+ dropReplacedComdat(GV, ReplacedDstComdats);
+
+ for (GlobalVariable &GV : llvm::make_early_inc_range(DstM.globals()))
+ dropReplacedComdat(GV, ReplacedDstComdats);
+
+ for (Function &GV : llvm::make_early_inc_range(DstM))
+ dropReplacedComdat(GV, ReplacedDstComdats);
+
+ for (GlobalVariable &GV : SrcM->globals())
+ if (GV.hasLinkOnceLinkage())
+ if (const Comdat *SC = GV.getComdat())
+ LazyComdatMembers[SC].push_back(&GV);
+
+ for (Function &SF : *SrcM)
+ if (SF.hasLinkOnceLinkage())
+ if (const Comdat *SC = SF.getComdat())
+ LazyComdatMembers[SC].push_back(&SF);
+
+ for (GlobalAlias &GA : SrcM->aliases())
+ if (GA.hasLinkOnceLinkage())
+ if (const Comdat *SC = GA.getComdat())
+ LazyComdatMembers[SC].push_back(&GA);
+
+ // Insert all of the globals in src into the DstM module... without linking
+ // initializers (which could refer to functions not yet mapped over).
+ SmallVector<GlobalValue *, 0> GVToClone;
+ for (GlobalVariable &GV : SrcM->globals())
+ if (linkIfNeeded(GV, GVToClone))
+ return true;
+
+ for (Function &SF : *SrcM)
+ if (linkIfNeeded(SF, GVToClone))
+ return true;
+
+ for (GlobalAlias &GA : SrcM->aliases())
+ if (linkIfNeeded(GA, GVToClone))
+ return true;
+
+ for (GlobalIFunc &GI : SrcM->ifuncs())
+ if (linkIfNeeded(GI, GVToClone))
+ return true;
+
+ // For a variable in a comdat nodeduplicate, its initializer should be
+ // preserved (its content may be implicitly used by other members) even if
+ // symbol resolution does not pick it. Clone it into an unnamed private
+ // variable.
+ for (GlobalValue *GV : GVToClone) {
+ if (auto *Var = dyn_cast<GlobalVariable>(GV)) {
+ auto *NewVar = new GlobalVariable(*Var->getParent(), Var->getValueType(),
+ Var->isConstant(), Var->getLinkage(),
+ Var->getInitializer());
+ NewVar->copyAttributesFrom(Var);
+ NewVar->setVisibility(GlobalValue::DefaultVisibility);
+ NewVar->setLinkage(GlobalValue::PrivateLinkage);
+ NewVar->setDSOLocal(true);
+ NewVar->setComdat(Var->getComdat());
+ if (Var->getParent() != &Mover.getModule())
+ ValuesToLink.insert(NewVar);
+ } else {
+ emitError("linking '" + GV->getName() +
+ "': non-variables in comdat nodeduplicate are not handled");
+ }
+ }
+
+ for (unsigned I = 0; I < ValuesToLink.size(); ++I) {
+ GlobalValue *GV = ValuesToLink[I];
+ const Comdat *SC = GV->getComdat();
+ if (!SC)
+ continue;
+ for (GlobalValue *GV2 : LazyComdatMembers[SC]) {
+ GlobalValue *DGV = getLinkedToGlobal(GV2);
+ bool LinkFromSrc = true;
+ if (DGV && shouldLinkFromSource(LinkFromSrc, *DGV, *GV2))
+ return true;
+ if (LinkFromSrc)
+ ValuesToLink.insert(GV2);
+ }
+ }
+
+ if (InternalizeCallback) {
+ for (GlobalValue *GV : ValuesToLink)
+ Internalize.insert(GV->getName());
+ }
+
+ // FIXME: Propagate Errors through to the caller instead of emitting
+ // diagnostics.
+ bool HasErrors = false;
+ if (Error E =
+ Mover.move(std::move(SrcM), ValuesToLink.getArrayRef(),
+ IRMover::LazyCallback(
+ [this](GlobalValue &GV, IRMover::ValueAdder Add) {
+ addLazyFor(GV, Add);
+ }),
+ /* IsPerformingImport */ false)) {
+ handleAllErrors(std::move(E), [&](ErrorInfoBase &EIB) {
+ DstM.getContext().diagnose(LinkDiagnosticInfo(DS_Error, EIB.message()));
+ HasErrors = true;
+ });
+ }
+ if (SrcIsCompilerBuiltins)
+ HasLinkedCompilerBuiltins = true;
+ if (HasErrors)
+ return true;
+
+ if (InternalizeCallback)
+ InternalizeCallback(DstM, Internalize);
+
+ return false;
+}
+
+namespace {
+
struct RustLinker {
- Linker L;
+ IRMover Mover;
LLVMContext &Ctx;
+ StringSet<> CompilerBuiltinsSymbols;
+
+ enum Flags {
+ None = 0,
+ OverrideFromSrc = (1 << 0),
+ LinkOnlyNeeded = (1 << 1),
+ };
- RustLinker(Module &M) :
- L(M),
- Ctx(M.getContext())
- {}
+ /// Link \p Src into the composite.
+ ///
+ /// Passing OverrideSymbols as true will have symbols from Src
+ /// shadow those in the Dest.
+ ///
+ /// Passing InternalizeCallback will have the linker call the function with
+ /// the new module and a list of global value names to be internalized by the
+ /// callback.
+ ///
+ /// Returns true on error.
+ bool linkInModule(std::unique_ptr<Module> Src, bool SrcIsCompilerBuiltins,
+ unsigned Flags = Flags::None,
+ std::function<void(Module &, const StringSet<> &)>
+ InternalizeCallback = {});
+
+ RustLinker(Module &M, StringSet<> CompilerBuiltinsSymbols)
+ : Mover(M), Ctx(M.getContext()),
+ CompilerBuiltinsSymbols(CompilerBuiltinsSymbols) {}
};
-extern "C" RustLinker*
-LLVMRustLinkerNew(LLVMModuleRef DstRef) {
- Module *Dst = unwrap(DstRef);
+} // namespace
- return new RustLinker(*Dst);
+bool RustLinker::linkInModule(
+ std::unique_ptr<Module> Src, bool SrcIsCompilerBuiltins, unsigned Flags,
+ std::function<void(Module &, const StringSet<> &)> InternalizeCallback) {
+ ModuleLinker ModLinker(Mover, CompilerBuiltinsSymbols, std::move(Src),
+ SrcIsCompilerBuiltins, Flags,
+ std::move(InternalizeCallback));
+ return ModLinker.run();
+}
+
+extern "C" RustLinker *LLVMRustLinkerNew(LLVMModuleRef DstRef, char **Symbols,
+ size_t Len) {
+ Module *Dst = unwrap(DstRef);
+ StringSet<> CompilerBuiltinsSymbols;
+ for (size_t I = 0; I < Len; I++) {
+ CompilerBuiltinsSymbols.insert(Symbols[I]);
+ }
+ return new RustLinker(*Dst, CompilerBuiltinsSymbols);
}
extern "C" void
@@ -27,8 +711,8 @@ LLVMRustLinkerFree(RustLinker *L) {
delete L;
}
-extern "C" bool
-LLVMRustLinkerAdd(RustLinker *L, char *BC, size_t Len) {
+extern "C" bool LLVMRustLinkerAdd(RustLinker *L, char *BC, size_t Len,
+ bool CompilerBuiltins) {
std::unique_ptr<MemoryBuffer> Buf =
MemoryBuffer::getMemBufferCopy(StringRef(BC, Len));
@@ -41,7 +725,7 @@ LLVMRustLinkerAdd(RustLinker *L, char *BC, size_t Len) {
auto Src = std::move(*SrcOrError);
- if (L->L.linkInModule(std::move(Src))) {
+ if (L->linkInModule(std::move(Src), CompilerBuiltins)) {
LLVMRustSetLastError("");
return false;
}
diff --git a/tests/assembly/lto-custom-builtins.rs b/tests/assembly/lto-custom-builtins.rs
new file mode 100644
index 0000000000000..8b053026c27bc
--- /dev/null
+++ b/tests/assembly/lto-custom-builtins.rs
@@ -0,0 +1,45 @@
+// assembly-output: emit-asm
+// compile-flags: --crate-type cdylib -C lto=fat -C prefer-dynamic=no
+// only-x86_64-unknown-linux-gnu
+
+#![feature(lang_items, linkage)]
+#![no_std]
+#![no_main]
+
+#![crate_type = "bin"]
+
+// We want to use customized __subdf3.
+// CHECK: .globl __subdf3
+// CHECK-NEXT: __subdf3:
+// CHECK-NEXT: movq $2, %rax
+core::arch::global_asm!(".global __subdf3", "__subdf3:", "mov rax, 2");
+
+// We want to use customized __addsf3.
+// CHECK: .globl __addsf3
+// CHECK: __addsf3:
+// CHECK: xorl %eax, %eax
+// CHECK-NEXT retq
+#[no_mangle]
+pub extern "C" fn __addsf3() -> i32 {
+ 0
+}
+
+// We want to use __adddf3 of compiler-builtins.
+// CHECK: .globl __adddf3
+// CHECK: __adddf3:
+// CHECK-NEXT: .cfi_startproc
+// CHECK-NOT: movl $1, %eax
+// CHECK: movq %xmm0, %rdx
+#[no_mangle]
+#[linkage = "weak"]
+pub extern "C" fn __adddf3() -> i32 {
+ 1
+}
+
+#[panic_handler]
+fn panic(_panic: &core::panic::PanicInfo<'_>) -> ! {
+ loop {}
+}
+
+#[lang = "eh_personality"]
+extern "C" fn eh_personality() {}