[ScanDependencies] Get accurate macro dependency

Build an accurate macro dependency for swift caching. Specifically, do
not include not used macro plugins into the dependency, which might
cause false negatives for cache hits.

This also builds the foundation for future improvement when dependency
scanning will determine the macro plugin to load and swift-frontend do
not need to redo the work.

rdar://127116512

Author: cachemeifyoucan

include/swift/AST/ModuleDependencies.h

@@ -88,6 +88,13 @@ enum class ModuleDependencyKind : int8_t {
   LastKind = SwiftPlaceholder + 1
 };
 
+/// This is used to idenfity a specific macro plugin dependency.
+struct MacroPluginDependency {
+  std::string LibraryPath;
+  std::string ExecutablePath;
+  std::string Identity;
+};
+
 /// This is used to identify a specific module.
 struct ModuleDependencyID {
   std::string ModuleName;
@@ -205,6 +212,9 @@ struct CommonSwiftTextualModuleDependencyDetails {
   /// (Clang) modules on which the bridging header depends.
   std::vector<std::string> bridgingModuleDependencies;
 
+  /// The macro dependencies.
+  llvm::StringMap<MacroPluginDependency> macroDependencies;
+
   /// The Swift frontend invocation arguments to build the Swift module from the
   /// interface.
   std::vector<std::string> buildCommandLine;
@@ -267,6 +277,12 @@ class SwiftInterfaceModuleDependenciesStorage :
   void updateCommandLine(const std::vector<std::string> &newCommandLine) {
     textualModuleDetails.buildCommandLine = newCommandLine;
   }
+
+  void addMacroDependency(StringRef module, StringRef id, StringRef libraryPath,
+                          StringRef executablePath) {
+    textualModuleDetails.macroDependencies.insert(
+        {module, {libraryPath.str(), executablePath.str(), id.str()}});
+  }
 };
 
 /// Describes the dependencies of a Swift module
@@ -318,6 +334,12 @@ class SwiftSourceModuleDependenciesStorage :
   void addTestableImport(ImportPath::Module module) {
     testableImports.insert(module.front().Item.str());
   }
+
+  void addMacroDependency(StringRef module, StringRef id, StringRef libraryPath,
+                          StringRef executablePath) {
+    textualModuleDetails.macroDependencies.insert(
+        {module, {libraryPath.str(), executablePath.str(), id.str()}});
+  }
 };
 
 /// Describes the dependencies of a pre-built Swift module (with no .swiftinterface).
@@ -700,6 +722,10 @@ class ModuleDependencyInfo {
   /// For a Source dependency, register a `Testable` import
   void addTestableImport(ImportPath::Module module);
 
+  /// For a Source dependency, register a macro dependency.
+  void addMacroDependency(StringRef module, StringRef id, StringRef libraryPath,
+                          StringRef executablePath);
+
   /// Whether or not a queried module name is a `@Testable` import dependency
   /// of this module. Can only return `true` for Swift source modules.
   bool isTestableImport(StringRef moduleName) const;

include/swift/AST/PluginRegistry.h

@@ -43,6 +43,9 @@ class LoadedLibraryPlugin {
   /// Finds the address of the given symbol within the library.
   void *getAddressOfSymbol(const char *symbolName);
 
+  /// Get the identify of the library plugin.
+  std::string getIdentity() const;
+
   NullTerminatedStringRef getLibraryPath() {
     return {LibraryPath.c_str(), LibraryPath.size()};
   }
@@ -115,6 +118,9 @@ class LoadedExecutablePlugin {
     return LastModificationTime;
   }
 
+  /// Get the identify of the executable plugin.
+  std::string getIdentity() const;
+
   /// Indicates that the current process is usable.
   bool isAlive() const { return Process != nullptr && !Process->isStale; }
 

lib/AST/ModuleDependencies.cpp

@@ -17,7 +17,11 @@
 #include "swift/AST/Decl.h"
 #include "swift/AST/DiagnosticsFrontend.h"
 #include "swift/AST/DiagnosticsSema.h"
+#include "swift/AST/Evaluator.h"
+#include "swift/AST/MacroDefinition.h"
+#include "swift/AST/PluginLoader.h"
 #include "swift/AST/SourceFile.h"
+#include "swift/AST/TypeCheckRequests.h"
 #include "swift/Frontend/Frontend.h"
 #include "llvm/CAS/CASProvidingFileSystem.h"
 #include "llvm/CAS/CachingOnDiskFileSystem.h"
@@ -103,6 +107,21 @@ void ModuleDependencyInfo::addTestableImport(ImportPath::Module module) {
   dyn_cast<SwiftSourceModuleDependenciesStorage>(storage.get())->addTestableImport(module);
 }
 
+void ModuleDependencyInfo::addMacroDependency(StringRef module, StringRef id,
+                                              StringRef libraryPath,
+                                              StringRef executablePath) {
+  if (auto swiftSourceStorage =
+          dyn_cast<SwiftSourceModuleDependenciesStorage>(storage.get()))
+    swiftSourceStorage->addMacroDependency(module, id, libraryPath,
+                                           executablePath);
+  else if (auto swiftInterfaceStorage =
+               dyn_cast<SwiftInterfaceModuleDependenciesStorage>(storage.get()))
+    swiftInterfaceStorage->addMacroDependency(module, id, libraryPath,
+                                              executablePath);
+  else
+    llvm_unreachable("Unexpected dependency kind");
+}
+
 bool ModuleDependencyInfo::isTestableImport(StringRef moduleName) const {
   if (auto swiftSourceDepStorage = getAsSwiftSourceModule())
     return swiftSourceDepStorage->testableImports.contains(moduleName);
@@ -132,33 +151,55 @@ void ModuleDependencyInfo::addModuleImport(
   SmallVector<Decl *, 32> decls;
   sf.getTopLevelDecls(decls);
   for (auto decl : decls) {
-    auto importDecl = dyn_cast<ImportDecl>(decl);
-    if (!importDecl)
-      continue;
-
-    ImportPath::Builder scratch;
-    auto realPath = importDecl->getRealModulePath(scratch);
-
-    // Explicit 'Builtin' import is not a part of the module's
-    // dependency set, does not exist on the filesystem,
-    // and is resolved within the compiler during compilation.
-    SmallString<64> importedModuleName;
-    realPath.getString(importedModuleName);
-    if (importedModuleName == BUILTIN_NAME)
-      continue;
-
-    // Ignore/diagnose tautological imports akin to import resolution
-    if (!swift::dependencies::checkImportNotTautological(
-            realPath, importDecl->getLoc(), sf, importDecl->isExported()))
-      continue;
+    if (auto importDecl = dyn_cast<ImportDecl>(decl)) {
+      ImportPath::Builder scratch;
+      auto realPath = importDecl->getRealModulePath(scratch);
+
+      // Explicit 'Builtin' import is not a part of the module's
+      // dependency set, does not exist on the filesystem,
+      // and is resolved within the compiler during compilation.
+      SmallString<64> importedModuleName;
+      realPath.getString(importedModuleName);
+      if (importedModuleName == BUILTIN_NAME)
+        continue;
 
-    addModuleImport(realPath, &alreadyAddedModules);
+      // Ignore/diagnose tautological imports akin to import resolution
+      if (!swift::dependencies::checkImportNotTautological(
+              realPath, importDecl->getLoc(), sf, importDecl->isExported()))
+        continue;
 
-    // Additionally, keep track of which dependencies of a Source
-    // module are `@Testable`.
-    if (getKind() == swift::ModuleDependencyKind::SwiftSource &&
-        importDecl->isTestable())
-      addTestableImport(realPath);
+      addModuleImport(realPath, &alreadyAddedModules);
+
+      // Additionally, keep track of which dependencies of a Source
+      // module are `@Testable`.
+      if (getKind() == swift::ModuleDependencyKind::SwiftSource &&
+          importDecl->isTestable())
+        addTestableImport(realPath);
+    } else if (auto macroDecl = dyn_cast<MacroDecl>(decl)) {
+      auto macroDef = macroDecl->getDefinition();
+      auto &ctx = macroDecl->getASTContext();
+      if (macroDef.kind != MacroDefinition::Kind::External)
+        continue;
+      auto external = macroDef.getExternalMacro();
+      CompilerPluginLoadRequest loadRequest{&macroDecl->getASTContext(),
+                                            external.moduleName};
+      CompilerPluginLoadResult loaded =
+          evaluateOrDefault(macroDecl->getASTContext().evaluator, loadRequest,
+                            CompilerPluginLoadResult::error("request error"));
+      PluginLoader &loader = ctx.getPluginLoader();
+      if (auto exe = loaded.getAsExecutablePlugin()) {
+        // Lookup again to find the library path.
+        const auto &entry =
+            loader.lookupPluginByModuleName(external.moduleName);
+        addMacroDependency(external.moduleName.str(),
+                           exe->getIdentity(),
+                           entry.libraryPath, entry.executablePath);
+      } else if (auto plugin = loaded.getAsLibraryPlugin()) {
+        addMacroDependency(external.moduleName.str(),
+                           plugin->getIdentity(),
+                           plugin->getLibraryPath(), /*executablePath=*/"");
+      }
+    }
   }
 
   auto fileName = sf.getFilename();
@@ -482,58 +523,6 @@ void SwiftDependencyTracker::addCommonSearchPathDeps(
   // Add VFSOverlay file.
   for (auto &Overlay: Opts.VFSOverlayFiles)
     FS->status(Overlay);
-
-  // Add plugin dylibs from the toolchain only by look through the plugin search
-  // directory.
-  auto recordFiles = [&](StringRef Path) {
-    std::error_code EC;
-    for (auto I = FS->dir_begin(Path, EC);
-         !EC && I != llvm::vfs::directory_iterator(); I = I.increment(EC)) {
-      if (I->type() != llvm::sys::fs::file_type::regular_file)
-        continue;
-#if defined(_WIN32)
-      constexpr StringRef libPrefix{};
-      constexpr StringRef libSuffix = ".dll";
-#else
-      constexpr StringRef libPrefix = "lib";
-      constexpr StringRef libSuffix = LTDL_SHLIB_EXT;
-#endif
-      StringRef filename = llvm::sys::path::filename(I->path());
-      if (filename.starts_with(libPrefix) && filename.ends_with(libSuffix))
-        FS->status(I->path());
-    }
-  };
-  for (auto &entry : Opts.PluginSearchOpts) {
-    switch (entry.getKind()) {
-
-    // '-load-plugin-library <library path>'.
-    case PluginSearchOption::Kind::LoadPluginLibrary: {
-      auto &val = entry.get<PluginSearchOption::LoadPluginLibrary>();
-      FS->status(val.LibraryPath);
-      break;
-    }
-
-    // '-load-plugin-executable <executable path>#<module name>, ...'.
-    case PluginSearchOption::Kind::LoadPluginExecutable: {
-      // We don't have executable plugin in toolchain.
-      break;
-    }
-
-    // '-plugin-path <library search path>'.
-    case PluginSearchOption::Kind::PluginPath: {
-      auto &val = entry.get<PluginSearchOption::PluginPath>();
-      recordFiles(val.SearchPath);
-      break;
-    }
-
-    // '-external-plugin-path <library search path>#<server path>'.
-    case PluginSearchOption::Kind::ExternalPluginPath: {
-      auto &val = entry.get<PluginSearchOption::ExternalPluginPath>();
-      recordFiles(val.SearchPath);
-      break;
-    }
-    }
-  }
 }
 
 void SwiftDependencyTracker::startTracking() {

lib/AST/PluginLoader.cpp

@@ -14,6 +14,7 @@
 #include "swift/AST/ASTContext.h"
 #include "swift/AST/DiagnosticEngine.h"
 #include "swift/AST/DiagnosticsFrontend.h"
+#include "swift/Basic/LLVM.h"
 #include "swift/Basic/SourceManager.h"
 #include "swift/Parse/Lexer.h"
 #include "llvm/Config/config.h"

lib/AST/PluginRegistry.cpp

@@ -83,6 +83,19 @@ void *LoadedLibraryPlugin::getAddressOfSymbol(const char *symbolName) {
   return cached;
 }
 
+std::string LoadedLibraryPlugin::getIdentity() const {
+  // Using the last modification time as identify for now. Assume the stat
+  // call should not be changed here comparing when it was dlopen'ed.
+  llvm::sys::fs::file_status stat;
+  if (auto err = llvm::sys::fs::status(LibraryPath, stat))
+    return "";
+
+  std::string timeStr;
+  llvm::raw_string_ostream SS(timeStr);
+  SS << stat.getLastModificationTime();
+  return timeStr;
+}
+
 llvm::Expected<LoadedExecutablePlugin *>
 PluginRegistry::loadExecutablePlugin(StringRef path, bool disableSandbox) {
   llvm::sys::fs::file_status stat;
@@ -128,6 +141,13 @@ PluginRegistry::loadExecutablePlugin(StringRef path, bool disableSandbox) {
   return storage.get();
 }
 
+std::string LoadedExecutablePlugin::getIdentity() const {
+  std::string timeStr;
+  llvm::raw_string_ostream SS(timeStr);
+  SS << LastModificationTime;
+  return timeStr;
+}
+
 llvm::Error LoadedExecutablePlugin::spawnIfNeeded() {
   if (Process) {
     // See if the loaded one is still usable.

lib/DependencyScan/ScanDependencies.cpp

@@ -242,6 +242,7 @@ static llvm::Error resolveExplicitModuleInputs(
     return E;
 
   std::vector<std::string> commandLine = resolvingDepInfo.getCommandline();
+  auto dependencyInfoCopy = resolvingDepInfo;
   for (const auto &depModuleID : dependencies) {
     const auto &depInfo = cache.findKnownDependency(depModuleID);
     switch (depModuleID.Kind) {
@@ -253,6 +254,14 @@ static llvm::Error resolveExplicitModuleInputs(
                        : interfaceDepDetails->moduleCacheKey;
       commandLine.push_back("-swift-module-file=" + depModuleID.ModuleName + "=" +
                             path);
+      // Add the exported macro from interface into current module.
+      llvm::for_each(
+          interfaceDepDetails->textualModuleDetails.macroDependencies,
+          [&](const auto &entry) {
+            dependencyInfoCopy.addMacroDependency(
+                entry.first(), entry.second.Identity, entry.second.LibraryPath,
+                entry.second.ExecutablePath);
+          });
     } break;
     case swift::ModuleDependencyKind::SwiftBinary: {
       auto binaryDepDetails = depInfo.getAsSwiftBinaryModule();
@@ -316,7 +325,6 @@ static llvm::Error resolveExplicitModuleInputs(
   }
 
   // Update the dependency in the cache with the modified command-line.
-  auto dependencyInfoCopy = resolvingDepInfo;
   if (resolvingDepInfo.isSwiftInterfaceModule() ||
       resolvingDepInfo.isClangModule()) {
     if (service.hasPathMapping())
@@ -342,6 +350,10 @@ static llvm::Error resolveExplicitModuleInputs(
       llvm::for_each(
           sourceDep->auxiliaryFiles,
           [&tracker](const std::string &file) { tracker->trackFile(file); });
+      llvm::for_each(sourceDep->textualModuleDetails.macroDependencies,
+                     [&tracker](const auto &entry) {
+                       tracker->trackFile(entry.second.LibraryPath);
+                     });
       auto root = tracker->createTreeFromDependencies();
       if (!root)
         return root.takeError();

lib/Sema/TypeCheckMacros.cpp

@@ -271,7 +271,7 @@ initializeExecutablePlugin(ASTContext &ctx,
                            LoadedExecutablePlugin *executablePlugin,
                            StringRef libraryPath, Identifier moduleName) {
   // Lock the plugin while initializing.
-  // Note that'executablePlugn' can be shared between multiple ASTContext.
+  // Note that'executablePlugin' can be shared between multiple ASTContext.
   executablePlugin->lock();
   SWIFT_DEFER { executablePlugin->unlock(); };
 

test/CAS/macro_plugin_external.swift

@@ -5,6 +5,7 @@
 
 // RUN: %empty-directory(%t)
 // RUN: %empty-directory(%t/plugins)
+// RUN: split-file %s %t
 //
 //== Build the plugin library
 // RUN: %host-build-swift \
@@ -15,9 +16,17 @@
 // RUN:   %S/../Macros/Inputs/syntax_macro_definitions.swift \
 // RUN:   -g -no-toolchain-stdlib-rpath
 
+/// No macro plugin when macro not used.
 // RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-serialized -module-name MyApp -module-cache-path %t/clang-module-cache -O \
 // RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
-// RUN:   %s -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -external-plugin-path %t/plugins#%swift-plugin-server
+// RUN:   %t/nomacro.swift -o %t/deps1.json -swift-version 5 -cache-compile-job -cas-path %t/cas -external-plugin-path %t/plugins#%swift-plugin-server
+// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps1.json MyApp casFSRootID > %t/no_macro_fs.casid
+// RUN: llvm-cas -cas %t/cas -ls-tree-recursive @%t/no_macro_fs.casid | %FileCheck %s --check-prefix=NO-MACRO
+// NO-MACRO-NOT: MacroDefinition
+
+// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-serialized -module-name MyApp -module-cache-path %t/clang-module-cache -O \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   %t/macro.swift -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -external-plugin-path %t/plugins#%swift-plugin-server
 
 // RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json MyApp casFSRootID > %t/fs.casid
 // RUN: llvm-cas -cas %t/cas -ls-tree-recursive @%t/fs.casid | %FileCheck %s --check-prefix=FS
@@ -37,8 +46,12 @@
 // RUN:   -external-plugin-path %t/plugins/#%swift-plugin-server \
 // RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
 // RUN:   -module-name MyApp -explicit-swift-module-map-file @%t/map.casid \
-// RUN:   %s @%t/MyApp.cmd
+// RUN:   %t/macro.swift @%t/MyApp.cmd
+
+//--- nomacro.swift
+func test() {}
 
+//--- macro.swift
 @attached(extension, conformances: P, names: named(requirement))
 macro DelegatedConformance() = #externalMacro(module: "MacroDefinition", type: "DelegatedConformanceViaExtensionMacro")