Skip to content

Commit 6907c02

Browse files
[ScanDependencies] Get accurate macro dependency
Build an accurate macro dependency for swift caching. Specifically, do not include not used macro plugins into the dependency, which might cause false negatives for cache hits. This also builds the foundation for future improvement when dependency scanning will determine the macro plugin to load and swift-frontend do not need to redo the work. rdar://127116512
1 parent 2618ae5 commit 6907c02

File tree

8 files changed

+144
-80
lines changed

8 files changed

+144
-80
lines changed

include/swift/AST/ModuleDependencies.h

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,13 @@ enum class ModuleDependencyKind : int8_t {
8888
LastKind = SwiftPlaceholder + 1
8989
};
9090

91+
/// This is used to idenfity a specific macro plugin dependency.
92+
struct MacroPluginDependency {
93+
std::string LibraryPath;
94+
std::string ExecutablePath;
95+
std::string Identity;
96+
};
97+
9198
/// This is used to identify a specific module.
9299
struct ModuleDependencyID {
93100
std::string ModuleName;
@@ -205,6 +212,9 @@ struct CommonSwiftTextualModuleDependencyDetails {
205212
/// (Clang) modules on which the bridging header depends.
206213
std::vector<std::string> bridgingModuleDependencies;
207214

215+
/// The macro dependencies.
216+
llvm::StringMap<MacroPluginDependency> macroDependencies;
217+
208218
/// The Swift frontend invocation arguments to build the Swift module from the
209219
/// interface.
210220
std::vector<std::string> buildCommandLine;
@@ -267,6 +277,12 @@ class SwiftInterfaceModuleDependenciesStorage :
267277
void updateCommandLine(const std::vector<std::string> &newCommandLine) {
268278
textualModuleDetails.buildCommandLine = newCommandLine;
269279
}
280+
281+
void addMacroDependency(StringRef module, StringRef id, StringRef libraryPath,
282+
StringRef executablePath) {
283+
textualModuleDetails.macroDependencies.insert(
284+
{module, {libraryPath.str(), executablePath.str(), id.str()}});
285+
}
270286
};
271287

272288
/// Describes the dependencies of a Swift module
@@ -318,6 +334,12 @@ class SwiftSourceModuleDependenciesStorage :
318334
void addTestableImport(ImportPath::Module module) {
319335
testableImports.insert(module.front().Item.str());
320336
}
337+
338+
void addMacroDependency(StringRef module, StringRef id, StringRef libraryPath,
339+
StringRef executablePath) {
340+
textualModuleDetails.macroDependencies.insert(
341+
{module, {libraryPath.str(), executablePath.str(), id.str()}});
342+
}
321343
};
322344

323345
/// Describes the dependencies of a pre-built Swift module (with no .swiftinterface).
@@ -700,6 +722,10 @@ class ModuleDependencyInfo {
700722
/// For a Source dependency, register a `Testable` import
701723
void addTestableImport(ImportPath::Module module);
702724

725+
/// For a Source dependency, register a macro dependency.
726+
void addMacroDependency(StringRef module, StringRef id, StringRef libraryPath,
727+
StringRef executablePath);
728+
703729
/// Whether or not a queried module name is a `@Testable` import dependency
704730
/// of this module. Can only return `true` for Swift source modules.
705731
bool isTestableImport(StringRef moduleName) const;

include/swift/AST/PluginRegistry.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ class LoadedLibraryPlugin {
4343
/// Finds the address of the given symbol within the library.
4444
void *getAddressOfSymbol(const char *symbolName);
4545

46+
/// Get the identify of the library plugin.
47+
std::string getIdentity() const;
48+
4649
NullTerminatedStringRef getLibraryPath() {
4750
return {LibraryPath.c_str(), LibraryPath.size()};
4851
}
@@ -115,6 +118,9 @@ class LoadedExecutablePlugin {
115118
return LastModificationTime;
116119
}
117120

121+
/// Get the identify of the executable plugin.
122+
std::string getIdentity() const;
123+
118124
/// Indicates that the current process is usable.
119125
bool isAlive() const { return Process != nullptr && !Process->isStale; }
120126

lib/AST/ModuleDependencies.cpp

Lines changed: 67 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,11 @@
1717
#include "swift/AST/Decl.h"
1818
#include "swift/AST/DiagnosticsFrontend.h"
1919
#include "swift/AST/DiagnosticsSema.h"
20+
#include "swift/AST/Evaluator.h"
21+
#include "swift/AST/MacroDefinition.h"
22+
#include "swift/AST/PluginLoader.h"
2023
#include "swift/AST/SourceFile.h"
24+
#include "swift/AST/TypeCheckRequests.h"
2125
#include "swift/Frontend/Frontend.h"
2226
#include "llvm/CAS/CASProvidingFileSystem.h"
2327
#include "llvm/CAS/CachingOnDiskFileSystem.h"
@@ -103,6 +107,21 @@ void ModuleDependencyInfo::addTestableImport(ImportPath::Module module) {
103107
dyn_cast<SwiftSourceModuleDependenciesStorage>(storage.get())->addTestableImport(module);
104108
}
105109

110+
void ModuleDependencyInfo::addMacroDependency(StringRef module, StringRef id,
111+
StringRef libraryPath,
112+
StringRef executablePath) {
113+
if (auto swiftSourceStorage =
114+
dyn_cast<SwiftSourceModuleDependenciesStorage>(storage.get()))
115+
swiftSourceStorage->addMacroDependency(module, id, libraryPath,
116+
executablePath);
117+
else if (auto swiftInterfaceStorage =
118+
dyn_cast<SwiftInterfaceModuleDependenciesStorage>(storage.get()))
119+
swiftInterfaceStorage->addMacroDependency(module, id, libraryPath,
120+
executablePath);
121+
else
122+
llvm_unreachable("Unexpected dependency kind");
123+
}
124+
106125
bool ModuleDependencyInfo::isTestableImport(StringRef moduleName) const {
107126
if (auto swiftSourceDepStorage = getAsSwiftSourceModule())
108127
return swiftSourceDepStorage->testableImports.contains(moduleName);
@@ -132,33 +151,56 @@ void ModuleDependencyInfo::addModuleImport(
132151
SmallVector<Decl *, 32> decls;
133152
sf.getTopLevelDecls(decls);
134153
for (auto decl : decls) {
135-
auto importDecl = dyn_cast<ImportDecl>(decl);
136-
if (!importDecl)
137-
continue;
138-
139-
ImportPath::Builder scratch;
140-
auto realPath = importDecl->getRealModulePath(scratch);
141-
142-
// Explicit 'Builtin' import is not a part of the module's
143-
// dependency set, does not exist on the filesystem,
144-
// and is resolved within the compiler during compilation.
145-
SmallString<64> importedModuleName;
146-
realPath.getString(importedModuleName);
147-
if (importedModuleName == BUILTIN_NAME)
148-
continue;
149-
150-
// Ignore/diagnose tautological imports akin to import resolution
151-
if (!swift::dependencies::checkImportNotTautological(
152-
realPath, importDecl->getLoc(), sf, importDecl->isExported()))
153-
continue;
154+
if (auto importDecl = dyn_cast<ImportDecl>(decl)) {
155+
ImportPath::Builder scratch;
156+
auto realPath = importDecl->getRealModulePath(scratch);
157+
158+
// Explicit 'Builtin' import is not a part of the module's
159+
// dependency set, does not exist on the filesystem,
160+
// and is resolved within the compiler during compilation.
161+
SmallString<64> importedModuleName;
162+
realPath.getString(importedModuleName);
163+
if (importedModuleName == BUILTIN_NAME)
164+
continue;
154165

155-
addModuleImport(realPath, &alreadyAddedModules);
166+
// Ignore/diagnose tautological imports akin to import resolution
167+
if (!swift::dependencies::checkImportNotTautological(
168+
realPath, importDecl->getLoc(), sf, importDecl->isExported()))
169+
continue;
156170

157-
// Additionally, keep track of which dependencies of a Source
158-
// module are `@Testable`.
159-
if (getKind() == swift::ModuleDependencyKind::SwiftSource &&
160-
importDecl->isTestable())
161-
addTestableImport(realPath);
171+
addModuleImport(realPath, &alreadyAddedModules);
172+
173+
// Additionally, keep track of which dependencies of a Source
174+
// module are `@Testable`.
175+
if (getKind() == swift::ModuleDependencyKind::SwiftSource &&
176+
importDecl->isTestable())
177+
addTestableImport(realPath);
178+
} else if (auto macroDecl = dyn_cast<MacroDecl>(decl)) {
179+
auto macroDef = macroDecl->getDefinition();
180+
auto &ctx = macroDecl->getASTContext();
181+
if (macroDef.kind != MacroDefinition::Kind::External)
182+
continue;
183+
auto external = macroDef.getExternalMacro();
184+
CompilerPluginLoadRequest loadRequest{&macroDecl->getASTContext(),
185+
external.moduleName};
186+
CompilerPluginLoadResult loaded =
187+
evaluateOrDefault(macroDecl->getASTContext().evaluator, loadRequest,
188+
CompilerPluginLoadResult::error("request error"));
189+
PluginLoader &loader = ctx.getPluginLoader();
190+
if (auto exe = loaded.getAsExecutablePlugin()) {
191+
// Lookup again to find the library path.
192+
const auto &entry =
193+
loader.lookupPluginByModuleName(external.moduleName);
194+
addMacroDependency(external.moduleName.str(),
195+
exe->getIdentity(),
196+
entry.libraryPath, entry.executablePath);
197+
}
198+
if (auto plugin = loaded.getAsLibraryPlugin()) {
199+
addMacroDependency(external.moduleName.str(),
200+
plugin->getIdentity(),
201+
plugin->getLibraryPath(), /*executablePath=*/"");
202+
}
203+
}
162204
}
163205

164206
auto fileName = sf.getFilename();
@@ -482,58 +524,6 @@ void SwiftDependencyTracker::addCommonSearchPathDeps(
482524
// Add VFSOverlay file.
483525
for (auto &Overlay: Opts.VFSOverlayFiles)
484526
FS->status(Overlay);
485-
486-
// Add plugin dylibs from the toolchain only by look through the plugin search
487-
// directory.
488-
auto recordFiles = [&](StringRef Path) {
489-
std::error_code EC;
490-
for (auto I = FS->dir_begin(Path, EC);
491-
!EC && I != llvm::vfs::directory_iterator(); I = I.increment(EC)) {
492-
if (I->type() != llvm::sys::fs::file_type::regular_file)
493-
continue;
494-
#if defined(_WIN32)
495-
constexpr StringRef libPrefix{};
496-
constexpr StringRef libSuffix = ".dll";
497-
#else
498-
constexpr StringRef libPrefix = "lib";
499-
constexpr StringRef libSuffix = LTDL_SHLIB_EXT;
500-
#endif
501-
StringRef filename = llvm::sys::path::filename(I->path());
502-
if (filename.starts_with(libPrefix) && filename.ends_with(libSuffix))
503-
FS->status(I->path());
504-
}
505-
};
506-
for (auto &entry : Opts.PluginSearchOpts) {
507-
switch (entry.getKind()) {
508-
509-
// '-load-plugin-library <library path>'.
510-
case PluginSearchOption::Kind::LoadPluginLibrary: {
511-
auto &val = entry.get<PluginSearchOption::LoadPluginLibrary>();
512-
FS->status(val.LibraryPath);
513-
break;
514-
}
515-
516-
// '-load-plugin-executable <executable path>#<module name>, ...'.
517-
case PluginSearchOption::Kind::LoadPluginExecutable: {
518-
// We don't have executable plugin in toolchain.
519-
break;
520-
}
521-
522-
// '-plugin-path <library search path>'.
523-
case PluginSearchOption::Kind::PluginPath: {
524-
auto &val = entry.get<PluginSearchOption::PluginPath>();
525-
recordFiles(val.SearchPath);
526-
break;
527-
}
528-
529-
// '-external-plugin-path <library search path>#<server path>'.
530-
case PluginSearchOption::Kind::ExternalPluginPath: {
531-
auto &val = entry.get<PluginSearchOption::ExternalPluginPath>();
532-
recordFiles(val.SearchPath);
533-
break;
534-
}
535-
}
536-
}
537527
}
538528

539529
void SwiftDependencyTracker::startTracking() {

lib/AST/PluginLoader.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include "swift/AST/ASTContext.h"
1515
#include "swift/AST/DiagnosticEngine.h"
1616
#include "swift/AST/DiagnosticsFrontend.h"
17+
#include "swift/Basic/LLVM.h"
1718
#include "swift/Basic/SourceManager.h"
1819
#include "swift/Parse/Lexer.h"
1920
#include "llvm/Config/config.h"

lib/AST/PluginRegistry.cpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,19 @@ void *LoadedLibraryPlugin::getAddressOfSymbol(const char *symbolName) {
8383
return cached;
8484
}
8585

86+
std::string LoadedLibraryPlugin::getIdentity() const {
87+
// Using the last modification time as identify for now. Assume the stat
88+
// call should not be changed here comparing when it was dlopen'ed.
89+
llvm::sys::fs::file_status stat;
90+
if (auto err = llvm::sys::fs::status(LibraryPath, stat))
91+
return "";
92+
93+
std::string timeStr;
94+
llvm::raw_string_ostream SS(timeStr);
95+
SS << stat.getLastModificationTime();
96+
return timeStr;
97+
}
98+
8699
llvm::Expected<LoadedExecutablePlugin *>
87100
PluginRegistry::loadExecutablePlugin(StringRef path, bool disableSandbox) {
88101
llvm::sys::fs::file_status stat;
@@ -128,6 +141,13 @@ PluginRegistry::loadExecutablePlugin(StringRef path, bool disableSandbox) {
128141
return storage.get();
129142
}
130143

144+
std::string LoadedExecutablePlugin::getIdentity() const {
145+
std::string timeStr;
146+
llvm::raw_string_ostream SS(timeStr);
147+
SS << LastModificationTime;
148+
return timeStr;
149+
}
150+
131151
llvm::Error LoadedExecutablePlugin::spawnIfNeeded() {
132152
if (Process) {
133153
// See if the loaded one is still usable.

lib/DependencyScan/ScanDependencies.cpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,10 @@ static llvm::Error resolveExplicitModuleInputs(
342342
llvm::for_each(
343343
sourceDep->auxiliaryFiles,
344344
[&tracker](const std::string &file) { tracker->trackFile(file); });
345+
llvm::for_each(sourceDep->textualModuleDetails.macroDependencies,
346+
[&tracker](const auto &entry) {
347+
tracker->trackFile(entry.second.LibraryPath);
348+
});
345349
auto root = tracker->createTreeFromDependencies();
346350
if (!root)
347351
return root.takeError();
@@ -357,6 +361,10 @@ static llvm::Error resolveExplicitModuleInputs(
357361
llvm::for_each(
358362
textualDep->auxiliaryFiles,
359363
[&tracker](const std::string &file) { tracker->trackFile(file); });
364+
llvm::for_each(sourceDep->textualModuleDetails.macroDependencies,
365+
[&tracker](const auto &entry) {
366+
tracker->trackFile(entry.second.LibraryPath);
367+
});
360368
auto root = tracker->createTreeFromDependencies();
361369
if (!root)
362370
return root.takeError();

lib/Sema/TypeCheckMacros.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -271,7 +271,7 @@ initializeExecutablePlugin(ASTContext &ctx,
271271
LoadedExecutablePlugin *executablePlugin,
272272
StringRef libraryPath, Identifier moduleName) {
273273
// Lock the plugin while initializing.
274-
// Note that'executablePlugn' can be shared between multiple ASTContext.
274+
// Note that'executablePlugin' can be shared between multiple ASTContext.
275275
executablePlugin->lock();
276276
SWIFT_DEFER { executablePlugin->unlock(); };
277277

test/CAS/macro_plugin_external.swift

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
// RUN: %empty-directory(%t)
77
// RUN: %empty-directory(%t/plugins)
8+
// RUN: split-file %s %t
89
//
910
//== Build the plugin library
1011
// RUN: %host-build-swift \
@@ -15,9 +16,17 @@
1516
// RUN: %S/../Macros/Inputs/syntax_macro_definitions.swift \
1617
// RUN: -g -no-toolchain-stdlib-rpath
1718

19+
/// No macro plugin when macro not used.
1820
// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-serialized -module-name MyApp -module-cache-path %t/clang-module-cache -O \
1921
// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
20-
// RUN: %s -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -external-plugin-path %t/plugins#%swift-plugin-server
22+
// RUN: %t/nomacro.swift -o %t/deps1.json -swift-version 5 -cache-compile-job -cas-path %t/cas -external-plugin-path %t/plugins#%swift-plugin-server
23+
// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps1.json MyApp casFSRootID > %t/no_macro_fs.casid
24+
// RUN: llvm-cas -cas %t/cas -ls-tree-recursive @%t/no_macro_fs.casid | %FileCheck %s --check-prefix=NO-MACRO
25+
// NO-MACRO-NOT: MacroDefinition
26+
27+
// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-serialized -module-name MyApp -module-cache-path %t/clang-module-cache -O \
28+
// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
29+
// RUN: %t/macro.swift -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -external-plugin-path %t/plugins#%swift-plugin-server
2130

2231
// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json MyApp casFSRootID > %t/fs.casid
2332
// RUN: llvm-cas -cas %t/cas -ls-tree-recursive @%t/fs.casid | %FileCheck %s --check-prefix=FS
@@ -37,8 +46,12 @@
3746
// RUN: -external-plugin-path %t/plugins/#%swift-plugin-server \
3847
// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
3948
// RUN: -module-name MyApp -explicit-swift-module-map-file @%t/map.casid \
40-
// RUN: %s @%t/MyApp.cmd
49+
// RUN: %t/macro.swift @%t/MyApp.cmd
50+
51+
//--- nomacro.swift
52+
func test() {}
4153

54+
//--- macro.swift
4255
@attached(extension, conformances: P, names: named(requirement))
4356
macro DelegatedConformance() = #externalMacro(module: "MacroDefinition", type: "DelegatedConformanceViaExtensionMacro")
4457

0 commit comments

Comments
 (0)