Skip to content

ModuleInterface: teach -compile-module-from-interface to emit forwarding module #32985

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 19, 2020
Merged

ModuleInterface: teach -compile-module-from-interface to emit forwarding module #32985

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions include/swift/AST/SearchPathOptions.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,9 @@ class SearchPathOptions {
/// The paths to a set of explicitly built modules from interfaces.
std::vector<std::string> ExplicitSwiftModules;

/// A set of compiled modules that may be ready to use.
std::vector<std::string> CandidateCompiledModules;

/// A map of explict Swift module information.
std::string ExplicitSwiftModuleMap;
private:
Expand Down
8 changes: 8 additions & 0 deletions include/swift/Frontend/ModuleInterfaceLoader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,14 @@ class ModuleInterfaceLoader : public SerializedModuleLoaderBase {
std::vector<std::string>
getCompiledModuleCandidatesForInterface(StringRef moduleName,
StringRef interfacePath) override;

/// Given a list of potential ready-to-use compiled modules for \p interfacePath,
/// check if any one of them is up-to-date. If so, emit a forwarding module
/// to the candidate binary module to \p outPath.
bool tryEmitForwardingModule(StringRef moduleName,
StringRef interfacePath,
ArrayRef<std::string> candidates,
StringRef outPath) override;
};

struct InterfaceSubContextDelegateImpl: InterfaceSubContextDelegate {
Expand Down
4 changes: 4 additions & 0 deletions include/swift/Option/FrontendOptions.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -719,4 +719,8 @@ def target_variant_sdk_version : Separate<["-"], "target-variant-sdk-version">,

def extra_clang_options_only : Flag<["-"], "only-use-extra-clang-opts">,
HelpText<"Options passed via -Xcc are sufficient for Clang configuration">;

def candidate_module_file
: Separate<["-"], "candidate-module-file">, MetaVarName<"<path>">,
HelpText<"Specify Swift module may be ready to use for an interface">;
} // end let Flags = [FrontendOption, NoDriverOption, HelpHidden]
6 changes: 6 additions & 0 deletions include/swift/Serialization/SerializedModuleLoader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,12 @@ class SerializedModuleLoaderBase : public ModuleLoader {
StringRef interfacePath) {
return std::vector<std::string>();
}
virtual bool tryEmitForwardingModule(StringRef moduleName,
StringRef interfacePath,
ArrayRef<std::string> candidates,
StringRef outPath) {
return false;
}
};

/// Imports serialized Swift modules into an ASTContext.
Expand Down
4 changes: 4 additions & 0 deletions lib/Frontend/CompilerInvocation.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -900,6 +900,10 @@ static bool ParseSearchPathArgs(SearchPathOptions &Opts,
}
if (const Arg *A = Args.getLastArg(OPT_explict_swift_module_map))
Opts.ExplicitSwiftModuleMap = A->getValue();
for (auto A: Args.filtered(OPT_candidate_module_file)) {
Opts.CandidateCompiledModules.push_back(resolveSearchPath(A->getValue()));
}

// Opts.RuntimeIncludePath is set by calls to
// setRuntimeIncludePath() or setMainExecutablePath().
// Opts.RuntimeImportPath is set by calls to
Expand Down
16 changes: 13 additions & 3 deletions lib/Frontend/ModuleInterfaceBuilder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,8 @@ bool ModuleInterfaceBuilder::collectDepsForSerialization(

bool ModuleInterfaceBuilder::buildSwiftModuleInternal(
StringRef OutPath, bool ShouldSerializeDeps,
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer) {
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
ArrayRef<std::string> CompiledCandidates) {

auto outerPrettyStackState = llvm::SavePrettyStackState();

Expand All @@ -167,6 +168,13 @@ bool ModuleInterfaceBuilder::buildSwiftModuleInternal(
[&](SubCompilerInstanceInfo &info) {
auto &SubInstance = *info.Instance;
auto subInvocation = SubInstance.getInvocation();
// Try building forwarding module first. If succeed, return.
if (static_cast<ModuleInterfaceLoader*>(SubInstance.getASTContext()
.getModuleInterfaceLoader())->tryEmitForwardingModule(moduleName,
interfacePath,
CompiledCandidates, OutPath)) {
return false;
}
FrontendOptions &FEOpts = subInvocation.getFrontendOptions();
const auto &InputInfo = FEOpts.InputsAndOutputs.firstInput();
StringRef InPath = InputInfo.file();
Expand Down Expand Up @@ -256,12 +264,14 @@ bool ModuleInterfaceBuilder::buildSwiftModuleInternal(
bool ModuleInterfaceBuilder::buildSwiftModule(StringRef OutPath,
bool ShouldSerializeDeps,
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
llvm::function_ref<void()> RemarkRebuild) {
llvm::function_ref<void()> RemarkRebuild,
ArrayRef<std::string> CompiledCandidates) {
auto build = [&]() {
if (RemarkRebuild) {
RemarkRebuild();
}
return buildSwiftModuleInternal(OutPath, ShouldSerializeDeps, ModuleBuffer);
return buildSwiftModuleInternal(OutPath, ShouldSerializeDeps, ModuleBuffer,
CompiledCandidates);
};
if (disableInterfaceFileLock) {
return build();
Expand Down
6 changes: 4 additions & 2 deletions lib/Frontend/ModuleInterfaceBuilder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,8 @@ class ModuleInterfaceBuilder {
bool IsHashBased);

bool buildSwiftModuleInternal(StringRef OutPath, bool ShouldSerializeDeps,
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer);
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
ArrayRef<std::string> CandidateModules);
public:
ModuleInterfaceBuilder(SourceManager &sourceMgr, DiagnosticEngine &diags,
InterfaceSubContextDelegate &subASTDelegate,
Expand All @@ -111,7 +112,8 @@ class ModuleInterfaceBuilder {

bool buildSwiftModule(StringRef OutPath, bool ShouldSerializeDeps,
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
llvm::function_ref<void()> RemarkRebuild = nullptr);
llvm::function_ref<void()> RemarkRebuild = nullptr,
ArrayRef<std::string> CandidateModules = {});
};

} // end namespace swift
Expand Down
39 changes: 37 additions & 2 deletions lib/Frontend/ModuleInterfaceLoader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1029,7 +1029,6 @@ ModuleInterfaceLoader::getCompiledModuleCandidatesForInterface(StringRef moduleN
dependencyTracker,
llvm::is_contained(PreferInterfaceForModules, moduleName) ?
ModuleLoadingMode::PreferInterface : LoadMode);
SmallVector<FileDependency, 16> allDeps;
std::vector<std::string> results;
auto pair = Impl.getCompiledModuleCandidates();
// Add compiled module candidates only when they are non-empty.
Expand All @@ -1040,6 +1039,41 @@ ModuleInterfaceLoader::getCompiledModuleCandidatesForInterface(StringRef moduleN
return results;
}

bool ModuleInterfaceLoader::tryEmitForwardingModule(StringRef moduleName,
StringRef interfacePath,
ArrayRef<std::string> candidates,
StringRef outputPath) {
// Derive .swiftmodule path from the .swiftinterface path.
auto newExt = file_types::getExtension(file_types::TY_SwiftModuleFile);
llvm::SmallString<32> modulePath = interfacePath;
llvm::sys::path::replace_extension(modulePath, newExt);
ModuleInterfaceLoaderImpl Impl(
Ctx, modulePath, interfacePath, moduleName,
CacheDir, PrebuiltCacheDir, SourceLoc(),
Opts,
dependencyTracker,
llvm::is_contained(PreferInterfaceForModules, moduleName) ?
ModuleLoadingMode::PreferInterface : LoadMode);
SmallVector<FileDependency, 16> deps;
std::unique_ptr<llvm::MemoryBuffer> moduleBuffer;
for (auto mod: candidates) {
// Check if the candidate compiled module is still up-to-date.
if (Impl.swiftModuleIsUpToDate(mod, deps, moduleBuffer)) {
// If so, emit a forwarding module to the candidate.
ForwardingModule FM(mod);
auto hadError = withOutputFile(Ctx.Diags, outputPath,
[&](llvm::raw_pwrite_stream &out) {
llvm::yaml::Output yamlWriter(out);
yamlWriter << FM;
return false;
});
if (!hadError)
return true;
}
}
return false;
}

bool ModuleInterfaceLoader::buildSwiftModuleFromSwiftInterface(
SourceManager &SourceMgr, DiagnosticEngine &Diags,
const SearchPathOptions &SearchPathOpts, const LangOptions &LangOpts,
Expand Down Expand Up @@ -1069,7 +1103,8 @@ bool ModuleInterfaceLoader::buildSwiftModuleFromSwiftInterface(
// FIXME: We really only want to serialize 'important' dependencies here, if
// we want to ship the built swiftmodules to another machine.
return builder.buildSwiftModule(OutPath, /*shouldSerializeDeps*/true,
/*ModuleBuffer*/nullptr);
/*ModuleBuffer*/nullptr, nullptr,
SearchPathOpts.CandidateCompiledModules);
}

void ModuleInterfaceLoader::collectVisibleTopLevelModuleNames(
Expand Down
26 changes: 26 additions & 0 deletions test/ModuleInterface/emit-forward-modules.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
// RUN: %empty-directory(%t)
// RUN: %empty-directory(%t/Foo.swiftmodule)
// RUN: %empty-directory(%t/ResourceDir/%target-sdk-name/prebuilt-modules/Foo.swiftmodule)
// RUN: echo "public func foo() {}" > %t/Foo.swift
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test has broken the macOS CI:

FAIL: Swift(macosx-x86_64) :: ModuleInterface/emit-forward-modules.swift (699 of 13520)
******************** TEST 'Swift(macosx-x86_64) :: ModuleInterface/emit-forward-modules.swift' FAILED ********************
--snip commands--
Command Output (stdout):
--
swiftmodule '/Volumes/swift-ci/jenkins/workspace/swift-PR-osx/branch-master/buildbot_incremental/swift-macosx-x86_64/test-macosx-x86_64/ModuleInterface/Output/emit-forward-modules.swift.tmp/Foo.swiftmodule/x86_64-apple-macos.swiftmodule' is not a forwarding module!

--
Command Output (stderr):
--
<unknown>:0: warning: module interfaces are only supported with Swift language version 5 or later (currently using -swift-version 4)
<unknown>:0: warning: module interfaces are only supported with -enable-library-evolution

Two completely unrelated pulls fail the CI because of this one test.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I attempted to contact some people that can revert this if necessary

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks.


import Foo

// Step 1: build swift interface from the source
// RUN: %target-swift-frontend -emit-module %t/Foo.swift -module-name Foo -emit-module-interface-path %t/Foo.swiftmodule/%target-swiftinterface-name

// Step 2: building a module from the interface, and the module should not be a forwarding module.
// RUN: %target-swift-frontend -compile-module-from-interface %t/Foo.swiftmodule/%target-swiftinterface-name -o %t/Foo.swiftmodule/%target-swiftmodule-name -module-name Foo

// RUN: not %{python} %S/ModuleCache/Inputs/check-is-forwarding-module.py %t/Foo.swiftmodule/%target-swiftmodule-name

// Step 3: given the adjacent binary module as a candidate, building a module from the interface should give us a forwarding module
// RUN: %target-swift-frontend -compile-module-from-interface %t/Foo.swiftmodule/%target-swiftinterface-name -o %t/Foo-from-interface.swiftmodule -module-name Foo -candidate-module-file %t/Foo.swiftmodule/%target-swiftmodule-name

// RUN: %{python} %S/ModuleCache/Inputs/check-is-forwarding-module.py %t/Foo-from-interface.swiftmodule

// Step 4: given the stale adjacent binary module as a candidate, building a module from the interface should give us a binary module
// RUN: touch %t/Foo.swiftmodule/%target-swiftinterface-name

// RUN: %target-swift-frontend -compile-module-from-interface %t/Foo.swiftmodule/%target-swiftinterface-name -o %t/Foo-from-interface.swiftmodule -module-name Foo -candidate-module-file %t/Foo.swiftmodule/%target-swiftmodule-name

// RUN: not %{python} %S/ModuleCache/Inputs/check-is-forwarding-module.py %t/Foo-from-interface.swiftmodule