Merge pull request #73421 from cachemeifyoucan/eng/PR-127116512

[ScanDependencies] Get accurate macro dependency

Author: cachemeifyoucan

include/swift/AST/ModuleDependencies.h

@@ -90,6 +90,12 @@ enum class ModuleDependencyKind : int8_t {
   LastKind = SwiftPlaceholder + 1
 };
 
+/// This is used to idenfity a specific macro plugin dependency.
+struct MacroPluginDependency {
+  std::string LibraryPath;
+  std::string ExecutablePath;
+};
+
 /// This is used to identify a specific module.
 struct ModuleDependencyID {
   std::string ModuleName;
@@ -246,6 +252,9 @@ struct CommonSwiftTextualModuleDependencyDetails {
   /// (Clang) modules on which the bridging header depends.
   std::vector<std::string> bridgingModuleDependencies;
 
+  /// The macro dependencies.
+  llvm::StringMap<MacroPluginDependency> macroDependencies;
+
   /// The Swift frontend invocation arguments to build the Swift module from the
   /// interface.
   std::vector<std::string> buildCommandLine;
@@ -311,6 +320,12 @@ class SwiftInterfaceModuleDependenciesStorage
   void updateCommandLine(const std::vector<std::string> &newCommandLine) {
     textualModuleDetails.buildCommandLine = newCommandLine;
   }
+
+  void addMacroDependency(StringRef macroModuleName, StringRef libraryPath,
+                          StringRef executablePath) {
+    textualModuleDetails.macroDependencies.insert(
+        {macroModuleName, {libraryPath.str(), executablePath.str()}});
+  }
 };
 
 /// Describes the dependencies of a Swift module
@@ -361,6 +376,12 @@ class SwiftSourceModuleDependenciesStorage
   void addTestableImport(ImportPath::Module module) {
     testableImports.insert(module.front().Item.str());
   }
+
+  void addMacroDependency(StringRef macroModuleName, StringRef libraryPath,
+                          StringRef executablePath) {
+    textualModuleDetails.macroDependencies.insert(
+        {macroModuleName, {libraryPath.str(), executablePath.str()}});
+  }
 };
 
 /// Describes the dependencies of a pre-built Swift module (with no
@@ -759,6 +780,10 @@ class ModuleDependencyInfo {
   /// For a Source dependency, register a `Testable` import
   void addTestableImport(ImportPath::Module module);
 
+  /// For a Source dependency, register a macro dependency.
+  void addMacroDependency(StringRef macroModuleName, StringRef libraryPath,
+                          StringRef executablePath);
+
   /// Whether or not a queried module name is a `@Testable` import dependency
   /// of this module. Can only return `true` for Swift source modules.
   bool isTestableImport(StringRef moduleName) const;

lib/AST/ModuleDependencies.cpp

@@ -17,8 +17,9 @@
 #include "swift/AST/Decl.h"
 #include "swift/AST/DiagnosticsFrontend.h"
 #include "swift/AST/DiagnosticsSema.h"
+#include "swift/AST/MacroDefinition.h"
+#include "swift/AST/PluginLoader.h"
 #include "swift/AST/SourceFile.h"
-#include "swift/Basic/Assertions.h"
 #include "swift/Frontend/Frontend.h"
 #include "llvm/CAS/CASProvidingFileSystem.h"
 #include "llvm/CAS/CachingOnDiskFileSystem.h"
@@ -104,6 +105,21 @@ void ModuleDependencyInfo::addTestableImport(ImportPath::Module module) {
   dyn_cast<SwiftSourceModuleDependenciesStorage>(storage.get())->addTestableImport(module);
 }
 
+void ModuleDependencyInfo::addMacroDependency(StringRef macroModuleName,
+                                              StringRef libraryPath,
+                                              StringRef executablePath) {
+  if (auto swiftSourceStorage =
+          dyn_cast<SwiftSourceModuleDependenciesStorage>(storage.get()))
+    swiftSourceStorage->addMacroDependency(macroModuleName, libraryPath,
+                                           executablePath);
+  else if (auto swiftInterfaceStorage =
+               dyn_cast<SwiftInterfaceModuleDependenciesStorage>(storage.get()))
+    swiftInterfaceStorage->addMacroDependency(macroModuleName, libraryPath,
+                                              executablePath);
+  else
+    llvm_unreachable("Unexpected dependency kind");
+}
+
 bool ModuleDependencyInfo::isTestableImport(StringRef moduleName) const {
   if (auto swiftSourceDepStorage = getAsSwiftSourceModule())
     return swiftSourceDepStorage->testableImports.contains(moduleName);
@@ -184,35 +200,45 @@ void ModuleDependencyInfo::addModuleImports(
   SmallVector<Decl *, 32> decls;
   sourceFile.getTopLevelDecls(decls);
   for (auto decl : decls) {
-    auto importDecl = dyn_cast<ImportDecl>(decl);
-    if (!importDecl)
-      continue;
-
-    ImportPath::Builder scratch;
-    auto realPath = importDecl->getRealModulePath(scratch);
-
-    // Explicit 'Builtin' import is not a part of the module's
-    // dependency set, does not exist on the filesystem,
-    // and is resolved within the compiler during compilation.
-    SmallString<64> importedModuleName;
-    realPath.getString(importedModuleName);
-    if (importedModuleName == BUILTIN_NAME)
-      continue;
-
-    // Ignore/diagnose tautological imports akin to import resolution
-    if (!swift::dependencies::checkImportNotTautological(
-            realPath, importDecl->getLoc(), sourceFile,
-            importDecl->isExported()))
-      continue;
-
-    addModuleImport(realPath, &alreadyAddedModules,
-                    sourceManager, importDecl->getLoc());
-
-    // Additionally, keep track of which dependencies of a Source
-    // module are `@Testable`.
-    if (getKind() == swift::ModuleDependencyKind::SwiftSource &&
-        importDecl->isTestable())
-      addTestableImport(realPath);
+    if (auto importDecl = dyn_cast<ImportDecl>(decl)) {
+      ImportPath::Builder scratch;
+      auto realPath = importDecl->getRealModulePath(scratch);
+
+      // Explicit 'Builtin' import is not a part of the module's
+      // dependency set, does not exist on the filesystem,
+      // and is resolved within the compiler during compilation.
+      SmallString<64> importedModuleName;
+      realPath.getString(importedModuleName);
+      if (importedModuleName == BUILTIN_NAME)
+        continue;
+
+      // Ignore/diagnose tautological imports akin to import resolution
+      if (!swift::dependencies::checkImportNotTautological(
+              realPath, importDecl->getLoc(), sourceFile,
+              importDecl->isExported()))
+        continue;
+
+      addModuleImport(realPath, &alreadyAddedModules, sourceManager,
+                      importDecl->getLoc());
+
+      // Additionally, keep track of which dependencies of a Source
+      // module are `@Testable`.
+      if (getKind() == swift::ModuleDependencyKind::SwiftSource &&
+          importDecl->isTestable())
+        addTestableImport(realPath);
+    } else if (auto macroDecl = dyn_cast<MacroDecl>(decl)) {
+      auto macroDef = macroDecl->getDefinition();
+      auto &ctx = macroDecl->getASTContext();
+      if (macroDef.kind != MacroDefinition::Kind::External)
+        continue;
+      auto external = macroDef.getExternalMacro();
+      PluginLoader &loader = ctx.getPluginLoader();
+      auto &entry = loader.lookupPluginByModuleName(external.moduleName);
+      if (entry.libraryPath.empty() && entry.executablePath.empty())
+        continue;
+      addMacroDependency(external.moduleName.str(), entry.libraryPath,
+                         entry.executablePath);
+    }
   }
 
   auto fileName = sourceFile.getFilename();
@@ -609,58 +635,6 @@ void SwiftDependencyTracker::addCommonSearchPathDeps(
   // Add VFSOverlay file.
   for (auto &Overlay: Opts.VFSOverlayFiles)
     FS->status(Overlay);
-
-  // Add plugin dylibs from the toolchain only by look through the plugin search
-  // directory.
-  auto recordFiles = [&](StringRef Path) {
-    std::error_code EC;
-    for (auto I = FS->dir_begin(Path, EC);
-         !EC && I != llvm::vfs::directory_iterator(); I = I.increment(EC)) {
-      if (I->type() != llvm::sys::fs::file_type::regular_file)
-        continue;
-#if defined(_WIN32)
-      constexpr StringRef libPrefix{};
-      constexpr StringRef libSuffix = ".dll";
-#else
-      constexpr StringRef libPrefix = "lib";
-      constexpr StringRef libSuffix = LTDL_SHLIB_EXT;
-#endif
-      StringRef filename = llvm::sys::path::filename(I->path());
-      if (filename.starts_with(libPrefix) && filename.ends_with(libSuffix))
-        FS->status(I->path());
-    }
-  };
-  for (auto &entry : Opts.PluginSearchOpts) {
-    switch (entry.getKind()) {
-
-    // '-load-plugin-library <library path>'.
-    case PluginSearchOption::Kind::LoadPluginLibrary: {
-      auto &val = entry.get<PluginSearchOption::LoadPluginLibrary>();
-      FS->status(val.LibraryPath);
-      break;
-    }
-
-    // '-load-plugin-executable <executable path>#<module name>, ...'.
-    case PluginSearchOption::Kind::LoadPluginExecutable: {
-      // We don't have executable plugin in toolchain.
-      break;
-    }
-
-    // '-plugin-path <library search path>'.
-    case PluginSearchOption::Kind::PluginPath: {
-      auto &val = entry.get<PluginSearchOption::PluginPath>();
-      recordFiles(val.SearchPath);
-      break;
-    }
-
-    // '-external-plugin-path <library search path>#<server path>'.
-    case PluginSearchOption::Kind::ExternalPluginPath: {
-      auto &val = entry.get<PluginSearchOption::ExternalPluginPath>();
-      recordFiles(val.SearchPath);
-      break;
-    }
-    }
-  }
 }
 
 void SwiftDependencyTracker::startTracking() {

lib/DependencyScan/ScanDependencies.cpp

@@ -244,6 +244,7 @@ static llvm::Error resolveExplicitModuleInputs(
     return E;
 
   std::vector<std::string> commandLine = resolvingDepInfo.getCommandline();
+  auto dependencyInfoCopy = resolvingDepInfo;
   for (const auto &depModuleID : dependencies) {
     const auto &depInfo = cache.findKnownDependency(depModuleID);
     switch (depModuleID.Kind) {
@@ -255,6 +256,14 @@ static llvm::Error resolveExplicitModuleInputs(
                        : interfaceDepDetails->moduleCacheKey;
       commandLine.push_back("-swift-module-file=" + depModuleID.ModuleName + "=" +
                             path);
+      // Add the exported macro from interface into current module.
+      llvm::for_each(
+          interfaceDepDetails->textualModuleDetails.macroDependencies,
+          [&](const auto &entry) {
+            dependencyInfoCopy.addMacroDependency(entry.first(),
+                                                  entry.second.LibraryPath,
+                                                  entry.second.ExecutablePath);
+          });
     } break;
     case swift::ModuleDependencyKind::SwiftBinary: {
       auto binaryDepDetails = depInfo.getAsSwiftBinaryModule();
@@ -319,7 +328,6 @@ static llvm::Error resolveExplicitModuleInputs(
   }
 
   // Update the dependency in the cache with the modified command-line.
-  auto dependencyInfoCopy = resolvingDepInfo;
   if (resolvingDepInfo.isSwiftInterfaceModule() ||
       resolvingDepInfo.isClangModule()) {
     if (service.hasPathMapping())
@@ -345,6 +353,10 @@ static llvm::Error resolveExplicitModuleInputs(
       llvm::for_each(
           sourceDep->auxiliaryFiles,
           [&tracker](const std::string &file) { tracker->trackFile(file); });
+      llvm::for_each(sourceDep->textualModuleDetails.macroDependencies,
+                     [&tracker](const auto &entry) {
+                       tracker->trackFile(entry.second.LibraryPath);
+                     });
       auto root = tracker->createTreeFromDependencies();
       if (!root)
         return root.takeError();

lib/Sema/TypeCheckMacros.cpp

@@ -239,7 +239,7 @@ static llvm::Expected<CompilerPlugin *>
 initializePlugin(ASTContext &ctx, CompilerPlugin *plugin, StringRef libraryPath,
                  Identifier moduleName) {
   // Lock the plugin while initializing.
-  // Note that'executablePlugn' can be shared between multiple ASTContext.
+  // Note that 'executablePlugin' can be shared between multiple ASTContext.
   plugin->lock();
   SWIFT_DEFER { plugin->unlock(); };
 

test/CAS/macro_plugin_external.swift

@@ -5,6 +5,7 @@
 
 // RUN: %empty-directory(%t)
 // RUN: %empty-directory(%t/plugins)
+// RUN: split-file %s %t
 //
 //== Build the plugin library
 // RUN: %host-build-swift \
@@ -15,9 +16,17 @@
 // RUN:   %S/../Macros/Inputs/syntax_macro_definitions.swift \
 // RUN:   -g -no-toolchain-stdlib-rpath
 
+/// No macro plugin when macro not used.
 // RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-serialized -module-name MyApp -module-cache-path %t/clang-module-cache -O \
 // RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
-// RUN:   %s -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -external-plugin-path %t/plugins#%swift-plugin-server
+// RUN:   %t/nomacro.swift -o %t/deps1.json -swift-version 5 -cache-compile-job -cas-path %t/cas -external-plugin-path %t/plugins#%swift-plugin-server
+// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps1.json MyApp casFSRootID > %t/no_macro_fs.casid
+// RUN: llvm-cas -cas %t/cas -ls-tree-recursive @%t/no_macro_fs.casid | %FileCheck %s --check-prefix=NO-MACRO
+// NO-MACRO-NOT: MacroDefinition
+
+// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-serialized -module-name MyApp -module-cache-path %t/clang-module-cache -O \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   %t/macro.swift -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -external-plugin-path %t/plugins#%swift-plugin-server
 
 // RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json MyApp casFSRootID > %t/fs.casid
 // RUN: llvm-cas -cas %t/cas -ls-tree-recursive @%t/fs.casid | %FileCheck %s --check-prefix=FS
@@ -37,8 +46,12 @@
 // RUN:   -external-plugin-path %t/plugins/#%swift-plugin-server \
 // RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
 // RUN:   -module-name MyApp -explicit-swift-module-map-file @%t/map.casid \
-// RUN:   %s @%t/MyApp.cmd
+// RUN:   %t/macro.swift @%t/MyApp.cmd
+
+//--- nomacro.swift
+func test() {}
 
+//--- macro.swift
 @attached(extension, conformances: P, names: named(requirement))
 macro DelegatedConformance() = #externalMacro(module: "MacroDefinition", type: "DelegatedConformanceViaExtensionMacro")