[Macro] Fix bugs in macro dependency scanning

Fix few issues from previous implementation from explicit module build
with macros and accurate macro dependency scanning in
swiftlang#73421.

First, there is a crash when propagating the macro dependencies. It
turns out that the current macro plugin implementation doesn't need the
downstream users to know about the plugin search path from the upstream
dependencies.

Secondly, fix a bug that the swiftinterface that has macro usage won't
build because the build command doesn't inherit the plugin search path
option.

Finally, add JSON output for macro dependencies so it is easier to
debug the macro dependencies.

rdar://131214106

Author: cachemeifyoucan

include/swift-c/DependencyScan/DependencyScan.h

@@ -49,6 +49,9 @@ typedef struct swiftscan_dependency_info_s *swiftscan_dependency_info_t;
 /// Opaque container to a link library info.
 typedef struct swiftscan_link_library_info_s *swiftscan_link_library_info_t;
 
+/// Opaque container to a macro dependency.
+typedef struct swiftscan_macro_dependency_s *swiftscan_macro_dependency_t;
+
 /// Opaque container to an overall result of a dependency scan.
 typedef struct swiftscan_dependency_graph_s *swiftscan_dependency_graph_t;
 
@@ -73,6 +76,12 @@ typedef struct {
   size_t count;
 } swiftscan_link_library_set_t;
 
+/// Set of macro dependency
+typedef struct {
+  swiftscan_macro_dependency_t *macro_dependencies;
+  size_t count;
+} swiftscan_macro_dependency_set_t;
+
 typedef enum {
   SWIFTSCAN_DIAGNOSTIC_SEVERITY_ERROR = 0,
   SWIFTSCAN_DIAGNOSTIC_SEVERITY_WARNING = 1,

include/swift/AST/ModuleDependencies.h

@@ -253,7 +253,7 @@ struct CommonSwiftTextualModuleDependencyDetails {
   std::vector<std::string> bridgingModuleDependencies;
 
   /// The macro dependencies.
-  llvm::StringMap<MacroPluginDependency> macroDependencies;
+  std::map<std::string, MacroPluginDependency> macroDependencies;
 
   /// The Swift frontend invocation arguments to build the Swift module from the
   /// interface.
@@ -324,7 +324,7 @@ class SwiftInterfaceModuleDependenciesStorage
   void addMacroDependency(StringRef macroModuleName, StringRef libraryPath,
                           StringRef executablePath) {
     textualModuleDetails.macroDependencies.insert(
-        {macroModuleName, {libraryPath.str(), executablePath.str()}});
+        {macroModuleName.str(), {libraryPath.str(), executablePath.str()}});
   }
 };
 
@@ -380,7 +380,7 @@ class SwiftSourceModuleDependenciesStorage
   void addMacroDependency(StringRef macroModuleName, StringRef libraryPath,
                           StringRef executablePath) {
     textualModuleDetails.macroDependencies.insert(
-        {macroModuleName, {libraryPath.str(), executablePath.str()}});
+        {macroModuleName.str(), {libraryPath.str(), executablePath.str()}});
   }
 };
 
@@ -796,7 +796,7 @@ class ModuleDependencyInfo {
   /// For a Source dependency, register a `Testable` import
   void addTestableImport(ImportPath::Module module);
 
-  /// For a Source dependency, register a macro dependency.
+  /// For a Source/Textual dependency, register a macro dependency.
   void addMacroDependency(StringRef macroModuleName, StringRef libraryPath,
                           StringRef executablePath);
 

include/swift/DependencyScan/DependencyScanImpl.h

@@ -73,6 +73,12 @@ struct swiftscan_link_library_info_s {
   bool forceLoad;
 };
 
+struct swiftscan_macro_dependency_s {
+  swiftscan_string_ref_t moduleName;
+  swiftscan_string_ref_t libraryPath;
+  swiftscan_string_ref_t executablePath;
+};
+
 /// Swift modules to be built from a module interface, may have a bridging
 /// header.
 typedef struct {
@@ -123,6 +129,9 @@ typedef struct {
 
   /// ModuleCacheKey
   swiftscan_string_ref_t module_cache_key;
+
+  /// Macro dependecies.
+  swiftscan_macro_dependency_set_t *macro_dependencies;
 } swiftscan_swift_textual_details_t;
 
 /// Swift modules with only a binary module file.

lib/DependencyScan/DependencyScanJSON.cpp

@@ -243,6 +243,49 @@ void writeLinkLibraries(llvm::raw_ostream &out,
   out << "\n";
 }
 
+static void
+writeMacroDependencies(llvm::raw_ostream &out,
+                       const swiftscan_macro_dependency_set_t *macro_deps,
+                       unsigned indentLevel, bool trailingComma) {
+  if (macro_deps->count == 0)
+    return;
+
+  out.indent(indentLevel * 2);
+  out << "\"macroDependencies\": ";
+  out << "[\n";
+  for (size_t i = 0; i < macro_deps->count; ++i) {
+    const auto &macroInfo = *macro_deps->macro_dependencies[i];
+    out.indent((indentLevel + 1) * 2);
+    out << "{\n";
+    auto entryIndentLevel = ((indentLevel + 2) * 2);
+    out.indent(entryIndentLevel);
+    out << "\"moduleName\": ";
+    writeJSONValue(out, macroInfo.moduleName, indentLevel);
+    out << ",\n";
+    out.indent(entryIndentLevel);
+    out << "\"libraryPath\": ";
+    writeJSONValue(out, macroInfo.libraryPath, entryIndentLevel);
+    out << ",\n";
+    out.indent(entryIndentLevel);
+    out << "\"executablePath\": ";
+    writeJSONValue(out, macroInfo.executablePath, entryIndentLevel);
+    out << "\n";
+    out.indent((indentLevel + 1) * 2);
+    out << "}";
+    if (i != macro_deps->count - 1) {
+      out << ",";
+    }
+    out << "\n";
+  }
+
+  out.indent(indentLevel * 2);
+  out << "]";
+
+  if (trailingComma)
+    out << ",";
+  out << "\n";
+}
+
 static const swiftscan_swift_textual_details_t *
 getAsTextualDependencyModule(swiftscan_module_details_t details) {
   if (details->kind == SWIFTSCAN_DEPENDENCY_INFO_SWIFT_TEXTUAL)
@@ -413,6 +456,8 @@ void writeJSON(llvm::raw_ostream &out,
                              swiftTextualDeps->module_cache_key, 5,
                              /*trailingComma=*/true);
       }
+      writeMacroDependencies(out, swiftTextualDeps->macro_dependencies, 5,
+                             /*trailingComma=*/true);
       writeJSONSingleField(out, "isFramework", swiftTextualDeps->is_framework,
                            5, commaAfterFramework);
       if (swiftTextualDeps->extra_pcm_args->count != 0) {

lib/DependencyScan/ModuleDependencyCacheSerialization.cpp

@@ -305,7 +305,7 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
       if (!moduleCacheKeyID)
         llvm::report_fatal_error("Bad moduleCacheKey");
 
-      // TODO: LinkLibraries
+      // TODO: LinkLibraries, MacroDependencies
       // Form the dependencies storage object
       auto moduleDep = ModuleDependencyInfo::forSwiftInterfaceModule(
           outputModulePath.value(), optionalSwiftInterfaceFile.value(),

lib/DependencyScan/ScanDependencies.cpp

@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "swift-c/DependencyScan/DependencyScan.h"
 #include "swift/Basic/PrettyStackTrace.h"
 
 #include "swift/AST/ASTContext.h"
@@ -256,14 +257,6 @@ static llvm::Error resolveExplicitModuleInputs(
                        : interfaceDepDetails->moduleCacheKey;
       commandLine.push_back("-swift-module-file=" + depModuleID.ModuleName + "=" +
                             path);
-      // Add the exported macro from interface into current module.
-      llvm::for_each(
-          interfaceDepDetails->textualModuleDetails.macroDependencies,
-          [&](const auto &entry) {
-            dependencyInfoCopy.addMacroDependency(entry.first(),
-                                                  entry.second.LibraryPath,
-                                                  entry.second.ExecutablePath);
-          });
     } break;
     case swift::ModuleDependencyKind::SwiftBinary: {
       auto binaryDepDetails = depInfo.getAsSwiftBinaryModule();
@@ -282,7 +275,7 @@ static llvm::Error resolveExplicitModuleInputs(
             {bridgingHeaderDepName, ModuleDependencyKind::Clang});
         const auto bridgingHeaderDepModuleDetails =
           optionalBridgingHeaderDepModuleInfo.getAsClangModule();
-	commandLine.push_back("-Xcc");
+        commandLine.push_back("-Xcc");
         commandLine.push_back(
             "-fmodule-map-file=" +
             remapPath(bridgingHeaderDepModuleDetails->moduleMapFile));
@@ -372,6 +365,10 @@ static llvm::Error resolveExplicitModuleInputs(
       llvm::for_each(
           textualDep->auxiliaryFiles,
           [&tracker](const std::string &file) { tracker->trackFile(file); });
+      llvm::for_each(textualDep->textualModuleDetails.macroDependencies,
+                     [&tracker](const auto &entry) {
+                       tracker->trackFile(entry.second.LibraryPath);
+                     });
       auto root = tracker->createTreeFromDependencies();
       if (!root)
         return root.takeError();
@@ -642,6 +639,28 @@ static swiftscan_diagnostic_set_t *mapCollectedDiagnosticsForOutput(
   return diagnosticOutput;
 }
 
+static swiftscan_macro_dependency_set_t *createMacroDependencySet(
+    const std::map<std::string, MacroPluginDependency> &macroDeps) {
+  swiftscan_macro_dependency_set_t *set = new swiftscan_macro_dependency_set_t;
+  if (macroDeps.empty()) {
+    set->count = 0;
+    set->macro_dependencies = nullptr;
+    return set;
+  }
+  set->count = macroDeps.size();
+  set->macro_dependencies = new swiftscan_macro_dependency_t[set->count];
+  unsigned SI = 0;
+  for (auto &entry : macroDeps) {
+    set->macro_dependencies[SI] = new swiftscan_macro_dependency_s;
+    set->macro_dependencies[SI]->moduleName = create_clone(entry.first.c_str());
+    set->macro_dependencies[SI]->libraryPath =
+        create_clone(entry.second.LibraryPath.c_str());
+    set->macro_dependencies[SI]->executablePath =
+        create_clone(entry.second.ExecutablePath.c_str());
+  }
+  return set;
+}
+
 static swiftscan_dependency_graph_t
 generateFullDependencyGraph(const CompilerInstance &instance,
                             const DependencyScanDiagnosticCollector *diagnosticCollector,
@@ -732,7 +751,9 @@ generateFullDependencyGraph(const CompilerInstance &instance,
                              .CASFileSystemRootID.c_str()),
             create_clone(swiftTextualDeps->textualModuleDetails
                              .CASBridgingHeaderIncludeTreeRootID.c_str()),
-            create_clone(swiftTextualDeps->moduleCacheKey.c_str())};
+            create_clone(swiftTextualDeps->moduleCacheKey.c_str()),
+            createMacroDependencySet(
+                swiftTextualDeps->textualModuleDetails.macroDependencies)};
       } else if (swiftSourceDeps) {
         swiftscan_string_ref_t moduleInterfacePath = create_null();
         swiftscan_string_ref_t bridgingHeaderPath =
@@ -767,7 +788,9 @@ generateFullDependencyGraph(const CompilerInstance &instance,
             /*IncludeTree*/
             create_clone(swiftSourceDeps->textualModuleDetails
                              .CASBridgingHeaderIncludeTreeRootID.c_str()),
-            /*CacheKey*/ create_clone("")};
+            /*CacheKey*/ create_clone(""),
+            createMacroDependencySet(
+                swiftSourceDeps->textualModuleDetails.macroDependencies)};
       } else if (swiftPlaceholderDeps) {
         details->kind = SWIFTSCAN_DEPENDENCY_INFO_SWIFT_PLACEHOLDER;
         details->swift_placeholder_details = {

lib/Frontend/ModuleInterfaceLoader.cpp

@@ -19,6 +19,7 @@
 #include "swift/AST/DiagnosticsSema.h"
 #include "swift/AST/FileSystem.h"
 #include "swift/AST/Module.h"
+#include "swift/AST/SearchPathOptions.h"
 #include "swift/Basic/Assertions.h"
 #include "swift/Basic/Platform.h"
 #include "swift/Basic/StringExtras.h"
@@ -1697,6 +1698,39 @@ void InterfaceSubContextDelegateImpl::inheritOptionsForBuildingInterface(
     genericSubInvocation.setPlatformAvailabilityInheritanceMapPath(*SearchPathOpts.PlatformAvailabilityInheritanceMapPath);
   }
 
+  for (auto &entry : SearchPathOpts.PluginSearchOpts) {
+    switch (entry.getKind()) {
+    case PluginSearchOption::Kind::LoadPluginLibrary: {
+      auto &val = entry.get<PluginSearchOption::LoadPluginLibrary>();
+      GenericArgs.push_back("-load-plugin-library");
+      GenericArgs.push_back(ArgSaver.save(val.LibraryPath));
+      break;
+    }
+    case PluginSearchOption::Kind::LoadPluginExecutable: {
+      auto &val = entry.get<PluginSearchOption::LoadPluginExecutable>();
+      for (auto &moduleName : val.ModuleNames) {
+        GenericArgs.push_back("-load-plugin-executable");
+        GenericArgs.push_back(
+            ArgSaver.save(val.ExecutablePath + "#" + moduleName));
+      }
+      break;
+    }
+    case PluginSearchOption::Kind::PluginPath: {
+      auto &val = entry.get<PluginSearchOption::PluginPath>();
+      GenericArgs.push_back("-plugin-library");
+      GenericArgs.push_back(ArgSaver.save(val.SearchPath));
+      break;
+    }
+    case PluginSearchOption::Kind::ExternalPluginPath: {
+      auto &val = entry.get<PluginSearchOption::ExternalPluginPath>();
+      GenericArgs.push_back("-external-plugin-path");
+      GenericArgs.push_back(
+          ArgSaver.save(val.SearchPath + "#" + val.ServerPath));
+      break;
+    }
+    }
+  }
+
   genericSubInvocation.getFrontendOptions().InputMode
       = FrontendOptions::ParseInputMode::SwiftModuleInterface;
   if (!SearchPathOpts.RuntimeResourcePath.empty()) {

test/CAS/macro_deps.swift

@@ -0,0 +1,104 @@
+// REQUIRES: swift_swift_parser
+
+/// Test loading dependencies that has macros.
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+/// Build macros.
+// RUN: %host-build-swift -swift-version 5 -emit-library -o %t/%target-library-name(MacroOne) -module-name=MacroOne %t/macro-1.swift
+// RUN: %host-build-swift -swift-version 5 -emit-library -o %t/%target-library-name(MacroTwo) -module-name=MacroTwo %t/macro-2.swift
+
+/// Build binary module that depends on textual module that uses macro.
+// RUN: %target-swift-frontend -emit-module -module-cache-path %t/clang-module-cache %t/test.swift -module-name Test -o %t/include/Test.swiftmodule -I %t/include \
+// RUN:   -O -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -swift-version 5 -external-plugin-path %t#%swift-plugin-server
+
+// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-serialized -module-name MyApp -module-cache-path %t/clang-module-cache -O \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -I %t/include \
+// RUN:   %t/main.swift -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -external-plugin-path %t#%swift-plugin-server
+
+// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json Foo macroDependencies | %FileCheck %s --check-prefix=FOO-DEPS
+// FOO-DEPS: MacroOne
+// FOO-DEPS-NOT: MacroTwo
+
+// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json MyApp macroDependencies | %FileCheck %s --check-prefix=APP-DEPS
+// APP-DEPS: MacroTwo
+// APP-DEPS-NOT: MacroOne
+
+/// Build all dependencies.
+// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/SwiftShims.cmd
+// RUN: %swift_frontend_plain @%t/SwiftShims.cmd
+// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json Foo > %t/Foo.cmd
+// RUN: %swift_frontend_plain @%t/Foo.cmd
+
+// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json MyApp > %t/MyApp.cmd
+// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json
+// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid
+
+// RUN: %target-swift-frontend -diagnostic-style=swift \
+// RUN:   -emit-module -o %t/Test.swiftmodule -cache-compile-job -cas-path %t/cas \
+// RUN:   -swift-version 5 -disable-implicit-swift-modules \
+// RUN:   -external-plugin-path %t#%swift-plugin-server \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   -module-name MyApp -explicit-swift-module-map-file @%t/map.casid -O \
+// RUN:   %t/main.swift @%t/MyApp.cmd
+
+//--- macro-1.swift
+import SwiftSyntax
+@_spi(ExperimentalLanguageFeature) import SwiftSyntaxMacros
+
+public struct AssertMacro: ExpressionMacro {
+  public static func expansion(
+    of macro: some FreestandingMacroExpansionSyntax,
+    in context: some MacroExpansionContext
+  ) -> ExprSyntax {
+    guard let argument = macro.arguments.first?.expression else {
+      fatalError("boom")
+    }
+
+    return "assert(\(argument))"
+  }
+}
+
+//--- macro-2.swift
+import SwiftSyntax
+@_spi(ExperimentalLanguageFeature) import SwiftSyntaxMacros
+
+public struct StringifyMacro: ExpressionMacro {
+  public static func expansion(
+    of macro: some FreestandingMacroExpansionSyntax,
+    in context: some MacroExpansionContext
+  ) -> ExprSyntax {
+    guard let argument = macro.arguments.first?.expression else {
+      fatalError("boom")
+    }
+
+    return "(\(argument), \(StringLiteralExprSyntax(content: argument.description)))"
+  }
+}
+
+//--- include/Foo.swiftinterface
+// swift-interface-format-version: 1.0
+// swift-module-flags: -enable-library-evolution -swift-version 5 -O -module-name Foo -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import
+import Swift
+public func foo()
+@freestanding(expression) public macro assert(_: Bool) = #externalMacro(module: "MacroOne", type: "AssertMacro")
+@inlinable
+public func assertFalse() {
+    #assert(false)
+}
+
+//--- test.swift
+import Foo
+@inlinable
+public func test() {
+    #assert(true)
+}
+
+//--- main.swift
+import Test
+@freestanding(expression) macro stringify<T>(_ value: T) -> (T, String) = #externalMacro(module: "MacroTwo", type: "StringifyMacro")
+
+func appTest() {
+    let str = #stringify("test")
+    test()
+}