From 5f71553147fa0515cc357b26779971eb79d5dd07 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Wed, 24 Apr 2024 11:31:48 -0700 Subject: [PATCH 1/3] [ScanDependencies] Get accurate macro dependency Build an accurate macro dependency for swift caching. Specifically, do not include not used macro plugins into the dependency, which might cause false negatives for cache hits. This also builds the foundation for future improvement when dependency scanning will determine the macro plugin to load and swift-frontend do not need to redo the work. rdar://127116512 (cherry picked from commit 4125e8c3cdc4ff0287ed49d24a03c87a4884542a) --- include/swift/AST/ModuleDependencies.h | 25 +++++ lib/AST/ModuleDependencies.cpp | 133 ++++++++++-------------- lib/DependencyScan/ScanDependencies.cpp | 14 ++- test/CAS/macro_plugin_external.swift | 17 ++- 4 files changed, 107 insertions(+), 82 deletions(-) diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h index 7488066d016f5..2db4ff293c1cb 100644 --- a/include/swift/AST/ModuleDependencies.h +++ b/include/swift/AST/ModuleDependencies.h @@ -88,6 +88,12 @@ enum class ModuleDependencyKind : int8_t { LastKind = SwiftPlaceholder + 1 }; +/// This is used to idenfity a specific macro plugin dependency. +struct MacroPluginDependency { + std::string LibraryPath; + std::string ExecutablePath; +}; + /// This is used to identify a specific module. struct ModuleDependencyID { std::string ModuleName; @@ -240,6 +246,9 @@ struct CommonSwiftTextualModuleDependencyDetails { /// (Clang) modules on which the bridging header depends. std::vector bridgingModuleDependencies; + /// The macro dependencies. + llvm::StringMap macroDependencies; + /// The Swift frontend invocation arguments to build the Swift module from the /// interface. std::vector buildCommandLine; @@ -302,6 +311,12 @@ class SwiftInterfaceModuleDependenciesStorage : void updateCommandLine(const std::vector &newCommandLine) { textualModuleDetails.buildCommandLine = newCommandLine; } + + void addMacroDependency(StringRef macroModuleName, StringRef libraryPath, + StringRef executablePath) { + textualModuleDetails.macroDependencies.insert( + {macroModuleName, {libraryPath.str(), executablePath.str()}}); + } }; /// Describes the dependencies of a Swift module @@ -353,6 +368,12 @@ class SwiftSourceModuleDependenciesStorage : void addTestableImport(ImportPath::Module module) { testableImports.insert(module.front().Item.str()); } + + void addMacroDependency(StringRef macroModuleName, StringRef libraryPath, + StringRef executablePath) { + textualModuleDetails.macroDependencies.insert( + {macroModuleName, {libraryPath.str(), executablePath.str()}}); + } }; /// Describes the dependencies of a pre-built Swift module (with no @@ -758,6 +779,10 @@ class ModuleDependencyInfo { /// For a Source dependency, register a `Testable` import void addTestableImport(ImportPath::Module module); + /// For a Source dependency, register a macro dependency. + void addMacroDependency(StringRef macroModuleName, StringRef libraryPath, + StringRef executablePath); + /// Whether or not a queried module name is a `@Testable` import dependency /// of this module. Can only return `true` for Swift source modules. bool isTestableImport(StringRef moduleName) const; diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp index 03284d448fbd6..98ead73df63a4 100644 --- a/lib/AST/ModuleDependencies.cpp +++ b/lib/AST/ModuleDependencies.cpp @@ -17,6 +17,8 @@ #include "swift/AST/Decl.h" #include "swift/AST/DiagnosticsFrontend.h" #include "swift/AST/DiagnosticsSema.h" +#include "swift/AST/MacroDefinition.h" +#include "swift/AST/PluginLoader.h" #include "swift/AST/SourceFile.h" #include "swift/Frontend/Frontend.h" #include "llvm/CAS/CASProvidingFileSystem.h" @@ -103,6 +105,21 @@ void ModuleDependencyInfo::addTestableImport(ImportPath::Module module) { dyn_cast(storage.get())->addTestableImport(module); } +void ModuleDependencyInfo::addMacroDependency(StringRef macroModuleName, + StringRef libraryPath, + StringRef executablePath) { + if (auto swiftSourceStorage = + dyn_cast(storage.get())) + swiftSourceStorage->addMacroDependency(macroModuleName, libraryPath, + executablePath); + else if (auto swiftInterfaceStorage = + dyn_cast(storage.get())) + swiftInterfaceStorage->addMacroDependency(macroModuleName, libraryPath, + executablePath); + else + llvm_unreachable("Unexpected dependency kind"); +} + bool ModuleDependencyInfo::isTestableImport(StringRef moduleName) const { if (auto swiftSourceDepStorage = getAsSwiftSourceModule()) return swiftSourceDepStorage->testableImports.contains(moduleName); @@ -183,35 +200,45 @@ void ModuleDependencyInfo::addModuleImports( SmallVector decls; sourceFile.getTopLevelDecls(decls); for (auto decl : decls) { - auto importDecl = dyn_cast(decl); - if (!importDecl) - continue; - - ImportPath::Builder scratch; - auto realPath = importDecl->getRealModulePath(scratch); - - // Explicit 'Builtin' import is not a part of the module's - // dependency set, does not exist on the filesystem, - // and is resolved within the compiler during compilation. - SmallString<64> importedModuleName; - realPath.getString(importedModuleName); - if (importedModuleName == BUILTIN_NAME) - continue; - - // Ignore/diagnose tautological imports akin to import resolution - if (!swift::dependencies::checkImportNotTautological( - realPath, importDecl->getLoc(), sourceFile, - importDecl->isExported())) - continue; + if (auto importDecl = dyn_cast(decl)) { + ImportPath::Builder scratch; + auto realPath = importDecl->getRealModulePath(scratch); + + // Explicit 'Builtin' import is not a part of the module's + // dependency set, does not exist on the filesystem, + // and is resolved within the compiler during compilation. + SmallString<64> importedModuleName; + realPath.getString(importedModuleName); + if (importedModuleName == BUILTIN_NAME) + continue; - addModuleImport(realPath, &alreadyAddedModules, - sourceManager, importDecl->getLoc()); + // Ignore/diagnose tautological imports akin to import resolution + if (!swift::dependencies::checkImportNotTautological( + realPath, importDecl->getLoc(), sourceFile, + importDecl->isExported())) + continue; - // Additionally, keep track of which dependencies of a Source - // module are `@Testable`. - if (getKind() == swift::ModuleDependencyKind::SwiftSource && - importDecl->isTestable()) - addTestableImport(realPath); + addModuleImport(realPath, &alreadyAddedModules, sourceManager, + importDecl->getLoc()); + + // Additionally, keep track of which dependencies of a Source + // module are `@Testable`. + if (getKind() == swift::ModuleDependencyKind::SwiftSource && + importDecl->isTestable()) + addTestableImport(realPath); + } else if (auto macroDecl = dyn_cast(decl)) { + auto macroDef = macroDecl->getDefinition(); + auto &ctx = macroDecl->getASTContext(); + if (macroDef.kind != MacroDefinition::Kind::External) + continue; + auto external = macroDef.getExternalMacro(); + PluginLoader &loader = ctx.getPluginLoader(); + auto &entry = loader.lookupPluginByModuleName(external.moduleName); + if (entry.libraryPath.empty() && entry.executablePath.empty()) + continue; + addMacroDependency(external.moduleName.str(), entry.libraryPath, + entry.executablePath); + } } auto fileName = sourceFile.getFilename(); @@ -535,58 +562,6 @@ void SwiftDependencyTracker::addCommonSearchPathDeps( // Add VFSOverlay file. for (auto &Overlay: Opts.VFSOverlayFiles) FS->status(Overlay); - - // Add plugin dylibs from the toolchain only by look through the plugin search - // directory. - auto recordFiles = [&](StringRef Path) { - std::error_code EC; - for (auto I = FS->dir_begin(Path, EC); - !EC && I != llvm::vfs::directory_iterator(); I = I.increment(EC)) { - if (I->type() != llvm::sys::fs::file_type::regular_file) - continue; -#if defined(_WIN32) - constexpr StringRef libPrefix{}; - constexpr StringRef libSuffix = ".dll"; -#else - constexpr StringRef libPrefix = "lib"; - constexpr StringRef libSuffix = LTDL_SHLIB_EXT; -#endif - StringRef filename = llvm::sys::path::filename(I->path()); - if (filename.starts_with(libPrefix) && filename.ends_with(libSuffix)) - FS->status(I->path()); - } - }; - for (auto &entry : Opts.PluginSearchOpts) { - switch (entry.getKind()) { - - // '-load-plugin-library '. - case PluginSearchOption::Kind::LoadPluginLibrary: { - auto &val = entry.get(); - FS->status(val.LibraryPath); - break; - } - - // '-load-plugin-executable #, ...'. - case PluginSearchOption::Kind::LoadPluginExecutable: { - // We don't have executable plugin in toolchain. - break; - } - - // '-plugin-path '. - case PluginSearchOption::Kind::PluginPath: { - auto &val = entry.get(); - recordFiles(val.SearchPath); - break; - } - - // '-external-plugin-path #'. - case PluginSearchOption::Kind::ExternalPluginPath: { - auto &val = entry.get(); - recordFiles(val.SearchPath); - break; - } - } - } } void SwiftDependencyTracker::startTracking() { diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index a04adcb933952..33f8da5f641c3 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -243,6 +243,7 @@ static llvm::Error resolveExplicitModuleInputs( return E; std::vector commandLine = resolvingDepInfo.getCommandline(); + auto dependencyInfoCopy = resolvingDepInfo; for (const auto &depModuleID : dependencies) { const auto &depInfo = cache.findKnownDependency(depModuleID); switch (depModuleID.Kind) { @@ -254,6 +255,14 @@ static llvm::Error resolveExplicitModuleInputs( : interfaceDepDetails->moduleCacheKey; commandLine.push_back("-swift-module-file=" + depModuleID.ModuleName + "=" + path); + // Add the exported macro from interface into current module. + llvm::for_each( + interfaceDepDetails->textualModuleDetails.macroDependencies, + [&](const auto &entry) { + dependencyInfoCopy.addMacroDependency(entry.first(), + entry.second.LibraryPath, + entry.second.ExecutablePath); + }); } break; case swift::ModuleDependencyKind::SwiftBinary: { auto binaryDepDetails = depInfo.getAsSwiftBinaryModule(); @@ -318,7 +327,6 @@ static llvm::Error resolveExplicitModuleInputs( } // Update the dependency in the cache with the modified command-line. - auto dependencyInfoCopy = resolvingDepInfo; if (resolvingDepInfo.isSwiftInterfaceModule() || resolvingDepInfo.isClangModule()) { if (service.hasPathMapping()) @@ -344,6 +352,10 @@ static llvm::Error resolveExplicitModuleInputs( llvm::for_each( sourceDep->auxiliaryFiles, [&tracker](const std::string &file) { tracker->trackFile(file); }); + llvm::for_each(sourceDep->textualModuleDetails.macroDependencies, + [&tracker](const auto &entry) { + tracker->trackFile(entry.second.LibraryPath); + }); auto root = tracker->createTreeFromDependencies(); if (!root) return root.takeError(); diff --git a/test/CAS/macro_plugin_external.swift b/test/CAS/macro_plugin_external.swift index 7a904f4d3172a..4c45ea02e4c1e 100644 --- a/test/CAS/macro_plugin_external.swift +++ b/test/CAS/macro_plugin_external.swift @@ -5,6 +5,7 @@ // RUN: %empty-directory(%t) // RUN: %empty-directory(%t/plugins) +// RUN: split-file %s %t // //== Build the plugin library // RUN: %host-build-swift \ @@ -15,9 +16,17 @@ // RUN: %S/../Macros/Inputs/syntax_macro_definitions.swift \ // RUN: -g -no-toolchain-stdlib-rpath +/// No macro plugin when macro not used. // RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-serialized -module-name MyApp -module-cache-path %t/clang-module-cache -O \ // RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ -// RUN: %s -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -external-plugin-path %t/plugins#%swift-plugin-server +// RUN: %t/nomacro.swift -o %t/deps1.json -swift-version 5 -cache-compile-job -cas-path %t/cas -external-plugin-path %t/plugins#%swift-plugin-server +// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps1.json MyApp casFSRootID > %t/no_macro_fs.casid +// RUN: llvm-cas -cas %t/cas -ls-tree-recursive @%t/no_macro_fs.casid | %FileCheck %s --check-prefix=NO-MACRO +// NO-MACRO-NOT: MacroDefinition + +// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-serialized -module-name MyApp -module-cache-path %t/clang-module-cache -O \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: %t/macro.swift -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -external-plugin-path %t/plugins#%swift-plugin-server // RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json MyApp casFSRootID > %t/fs.casid // RUN: llvm-cas -cas %t/cas -ls-tree-recursive @%t/fs.casid | %FileCheck %s --check-prefix=FS @@ -37,8 +46,12 @@ // RUN: -external-plugin-path %t/plugins/#%swift-plugin-server \ // RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ // RUN: -module-name MyApp -explicit-swift-module-map-file @%t/map.casid \ -// RUN: %s @%t/MyApp.cmd +// RUN: %t/macro.swift @%t/MyApp.cmd + +//--- nomacro.swift +func test() {} +//--- macro.swift @attached(extension, conformances: P, names: named(requirement)) macro DelegatedConformance() = #externalMacro(module: "MacroDefinition", type: "DelegatedConformanceViaExtensionMacro") From c29791cf5400cb25050046c77467e96043133886 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 8 Jul 2024 15:08:19 -0700 Subject: [PATCH 2/3] [Macro] Fix bugs in macro dependency scanning Fix few issues from previous implementation from explicit module build with macros and accurate macro dependency scanning in https://github.com/swiftlang/swift/pull/73421. First, there is a crash when propagating the macro dependencies. It turns out that the current macro plugin implementation doesn't need the downstream users to know about the plugin search path from the upstream dependencies. Secondly, fix a bug that the swiftinterface that has macro usage won't build because the build command doesn't inherit the plugin search path option. Finally, add JSON output for macro dependencies so it is easier to debug the macro dependencies. rdar://131214106 (cherry picked from commit da10a02231318eb26022f2cbf4236d3da6701d1e) --- .../swift-c/DependencyScan/DependencyScan.h | 10 ++ include/swift/AST/ModuleDependencies.h | 8 +- .../swift/DependencyScan/DependencyScanImpl.h | 9 ++ lib/DependencyScan/DependencyScanJSON.cpp | 45 ++++++++ lib/DependencyScan/ScanDependencies.cpp | 46 ++++++-- lib/Frontend/ModuleInterfaceLoader.cpp | 34 ++++++ test/CAS/macro_deps.swift | 104 ++++++++++++++++++ tools/libSwiftScan/libSwiftScan.cpp | 12 ++ 8 files changed, 253 insertions(+), 15 deletions(-) create mode 100644 test/CAS/macro_deps.swift diff --git a/include/swift-c/DependencyScan/DependencyScan.h b/include/swift-c/DependencyScan/DependencyScan.h index 8320435945672..843aa31fa792f 100644 --- a/include/swift-c/DependencyScan/DependencyScan.h +++ b/include/swift-c/DependencyScan/DependencyScan.h @@ -46,6 +46,10 @@ typedef struct swiftscan_module_details_s *swiftscan_module_details_t; /// Opaque container to a dependency info of a given module. typedef struct swiftscan_dependency_info_s *swiftscan_dependency_info_t; + +/// Opaque container to a macro dependency. +typedef struct swiftscan_macro_dependency_s *swiftscan_macro_dependency_t; + /// Opaque container to an overall result of a dependency scan. typedef struct swiftscan_dependency_graph_s *swiftscan_dependency_graph_t; @@ -64,6 +68,12 @@ typedef struct { size_t count; } swiftscan_dependency_set_t; +/// Set of macro dependency +typedef struct { + swiftscan_macro_dependency_t *macro_dependencies; + size_t count; +} swiftscan_macro_dependency_set_t; + typedef enum { SWIFTSCAN_DIAGNOSTIC_SEVERITY_ERROR = 0, SWIFTSCAN_DIAGNOSTIC_SEVERITY_WARNING = 1, diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h index 2db4ff293c1cb..7f54c26da0422 100644 --- a/include/swift/AST/ModuleDependencies.h +++ b/include/swift/AST/ModuleDependencies.h @@ -247,7 +247,7 @@ struct CommonSwiftTextualModuleDependencyDetails { std::vector bridgingModuleDependencies; /// The macro dependencies. - llvm::StringMap macroDependencies; + std::map macroDependencies; /// The Swift frontend invocation arguments to build the Swift module from the /// interface. @@ -315,7 +315,7 @@ class SwiftInterfaceModuleDependenciesStorage : void addMacroDependency(StringRef macroModuleName, StringRef libraryPath, StringRef executablePath) { textualModuleDetails.macroDependencies.insert( - {macroModuleName, {libraryPath.str(), executablePath.str()}}); + {macroModuleName.str(), {libraryPath.str(), executablePath.str()}}); } }; @@ -372,7 +372,7 @@ class SwiftSourceModuleDependenciesStorage : void addMacroDependency(StringRef macroModuleName, StringRef libraryPath, StringRef executablePath) { textualModuleDetails.macroDependencies.insert( - {macroModuleName, {libraryPath.str(), executablePath.str()}}); + {macroModuleName.str(), {libraryPath.str(), executablePath.str()}}); } }; @@ -779,7 +779,7 @@ class ModuleDependencyInfo { /// For a Source dependency, register a `Testable` import void addTestableImport(ImportPath::Module module); - /// For a Source dependency, register a macro dependency. + /// For a Source/Textual dependency, register a macro dependency. void addMacroDependency(StringRef macroModuleName, StringRef libraryPath, StringRef executablePath); diff --git a/include/swift/DependencyScan/DependencyScanImpl.h b/include/swift/DependencyScan/DependencyScanImpl.h index a00dff1d6749a..cdaa0c54a56a6 100644 --- a/include/swift/DependencyScan/DependencyScanImpl.h +++ b/include/swift/DependencyScan/DependencyScanImpl.h @@ -64,6 +64,12 @@ struct swiftscan_dependency_info_s { swiftscan_module_details_t details; }; +struct swiftscan_macro_dependency_s { + swiftscan_string_ref_t moduleName; + swiftscan_string_ref_t libraryPath; + swiftscan_string_ref_t executablePath; +}; + /// Swift modules to be built from a module interface, may have a bridging /// header. typedef struct { @@ -111,6 +117,9 @@ typedef struct { /// ModuleCacheKey swiftscan_string_ref_t module_cache_key; + + /// Macro dependecies. + swiftscan_macro_dependency_set_t *macro_dependencies; } swiftscan_swift_textual_details_t; /// Swift modules with only a binary module file. diff --git a/lib/DependencyScan/DependencyScanJSON.cpp b/lib/DependencyScan/DependencyScanJSON.cpp index 133b98b7e7329..1e82c654125b4 100644 --- a/lib/DependencyScan/DependencyScanJSON.cpp +++ b/lib/DependencyScan/DependencyScanJSON.cpp @@ -202,6 +202,49 @@ static void writeDependencies(llvm::raw_ostream &out, out << "\n"; } +static void +writeMacroDependencies(llvm::raw_ostream &out, + const swiftscan_macro_dependency_set_t *macro_deps, + unsigned indentLevel, bool trailingComma) { + if (macro_deps->count == 0) + return; + + out.indent(indentLevel * 2); + out << "\"macroDependencies\": "; + out << "[\n"; + for (size_t i = 0; i < macro_deps->count; ++i) { + const auto ¯oInfo = *macro_deps->macro_dependencies[i]; + out.indent((indentLevel + 1) * 2); + out << "{\n"; + auto entryIndentLevel = ((indentLevel + 2) * 2); + out.indent(entryIndentLevel); + out << "\"moduleName\": "; + writeJSONValue(out, macroInfo.moduleName, indentLevel); + out << ",\n"; + out.indent(entryIndentLevel); + out << "\"libraryPath\": "; + writeJSONValue(out, macroInfo.libraryPath, entryIndentLevel); + out << ",\n"; + out.indent(entryIndentLevel); + out << "\"executablePath\": "; + writeJSONValue(out, macroInfo.executablePath, entryIndentLevel); + out << "\n"; + out.indent((indentLevel + 1) * 2); + out << "}"; + if (i != macro_deps->count - 1) { + out << ","; + } + out << "\n"; + } + + out.indent(indentLevel * 2); + out << "]"; + + if (trailingComma) + out << ","; + out << "\n"; +} + static const swiftscan_swift_textual_details_t * getAsTextualDependencyModule(swiftscan_module_details_t details) { if (details->kind == SWIFTSCAN_DEPENDENCY_INFO_SWIFT_TEXTUAL) @@ -369,6 +412,8 @@ void writeJSON(llvm::raw_ostream &out, swiftTextualDeps->module_cache_key, 5, /*trailingComma=*/true); } + writeMacroDependencies(out, swiftTextualDeps->macro_dependencies, 5, + /*trailingComma=*/true); writeJSONSingleField(out, "isFramework", swiftTextualDeps->is_framework, 5, commaAfterFramework); if (swiftTextualDeps->extra_pcm_args->count != 0) { diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index 33f8da5f641c3..f2991dda7d680 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// +#include "swift-c/DependencyScan/DependencyScan.h" #include "swift/Basic/PrettyStackTrace.h" #include "swift/AST/ASTContext.h" @@ -255,14 +256,6 @@ static llvm::Error resolveExplicitModuleInputs( : interfaceDepDetails->moduleCacheKey; commandLine.push_back("-swift-module-file=" + depModuleID.ModuleName + "=" + path); - // Add the exported macro from interface into current module. - llvm::for_each( - interfaceDepDetails->textualModuleDetails.macroDependencies, - [&](const auto &entry) { - dependencyInfoCopy.addMacroDependency(entry.first(), - entry.second.LibraryPath, - entry.second.ExecutablePath); - }); } break; case swift::ModuleDependencyKind::SwiftBinary: { auto binaryDepDetails = depInfo.getAsSwiftBinaryModule(); @@ -281,7 +274,7 @@ static llvm::Error resolveExplicitModuleInputs( {bridgingHeaderDepName, ModuleDependencyKind::Clang}); const auto bridgingHeaderDepModuleDetails = optionalBridgingHeaderDepModuleInfo.getAsClangModule(); - commandLine.push_back("-Xcc"); + commandLine.push_back("-Xcc"); commandLine.push_back( "-fmodule-map-file=" + remapPath(bridgingHeaderDepModuleDetails->moduleMapFile)); @@ -371,6 +364,10 @@ static llvm::Error resolveExplicitModuleInputs( llvm::for_each( textualDep->auxiliaryFiles, [&tracker](const std::string &file) { tracker->trackFile(file); }); + llvm::for_each(textualDep->textualModuleDetails.macroDependencies, + [&tracker](const auto &entry) { + tracker->trackFile(entry.second.LibraryPath); + }); auto root = tracker->createTreeFromDependencies(); if (!root) return root.takeError(); @@ -591,6 +588,29 @@ static void bridgeDependencyIDs(const ArrayRef dependencies, } } + +static swiftscan_macro_dependency_set_t *createMacroDependencySet( + const std::map ¯oDeps) { + swiftscan_macro_dependency_set_t *set = new swiftscan_macro_dependency_set_t; + if (macroDeps.empty()) { + set->count = 0; + set->macro_dependencies = nullptr; + return set; + } + set->count = macroDeps.size(); + set->macro_dependencies = new swiftscan_macro_dependency_t[set->count]; + unsigned SI = 0; + for (auto &entry : macroDeps) { + set->macro_dependencies[SI] = new swiftscan_macro_dependency_s; + set->macro_dependencies[SI]->moduleName = create_clone(entry.first.c_str()); + set->macro_dependencies[SI]->libraryPath = + create_clone(entry.second.LibraryPath.c_str()); + set->macro_dependencies[SI]->executablePath = + create_clone(entry.second.ExecutablePath.c_str()); + } + return set; +} + static swiftscan_dependency_graph_t generateFullDependencyGraph(const CompilerInstance &instance, const DependencyScanDiagnosticCollector *diagnosticCollector, @@ -680,7 +700,9 @@ generateFullDependencyGraph(const CompilerInstance &instance, .CASFileSystemRootID.c_str()), create_clone(swiftTextualDeps->textualModuleDetails .CASBridgingHeaderIncludeTreeRootID.c_str()), - create_clone(swiftTextualDeps->moduleCacheKey.c_str())}; + create_clone(swiftTextualDeps->moduleCacheKey.c_str()), + createMacroDependencySet( + swiftTextualDeps->textualModuleDetails.macroDependencies)}; } else if (swiftSourceDeps) { swiftscan_string_ref_t moduleInterfacePath = create_null(); swiftscan_string_ref_t bridgingHeaderPath = @@ -714,7 +736,9 @@ generateFullDependencyGraph(const CompilerInstance &instance, /*IncludeTree*/ create_clone(swiftSourceDeps->textualModuleDetails .CASBridgingHeaderIncludeTreeRootID.c_str()), - /*CacheKey*/ create_clone("")}; + /*CacheKey*/ create_clone(""), + createMacroDependencySet( + swiftSourceDeps->textualModuleDetails.macroDependencies)}; } else if (swiftPlaceholderDeps) { details->kind = SWIFTSCAN_DEPENDENCY_INFO_SWIFT_PLACEHOLDER; details->swift_placeholder_details = { diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp index be564ec2395de..5ea55f9ca25a7 100644 --- a/lib/Frontend/ModuleInterfaceLoader.cpp +++ b/lib/Frontend/ModuleInterfaceLoader.cpp @@ -19,6 +19,7 @@ #include "swift/AST/DiagnosticsSema.h" #include "swift/AST/FileSystem.h" #include "swift/AST/Module.h" +#include "swift/AST/SearchPathOptions.h" #include "swift/Basic/Platform.h" #include "swift/Basic/StringExtras.h" #include "swift/Frontend/CachingUtils.h" @@ -1696,6 +1697,39 @@ void InterfaceSubContextDelegateImpl::inheritOptionsForBuildingInterface( genericSubInvocation.setPlatformAvailabilityInheritanceMapPath(*SearchPathOpts.PlatformAvailabilityInheritanceMapPath); } + for (auto &entry : SearchPathOpts.PluginSearchOpts) { + switch (entry.getKind()) { + case PluginSearchOption::Kind::LoadPluginLibrary: { + auto &val = entry.get(); + GenericArgs.push_back("-load-plugin-library"); + GenericArgs.push_back(ArgSaver.save(val.LibraryPath)); + break; + } + case PluginSearchOption::Kind::LoadPluginExecutable: { + auto &val = entry.get(); + for (auto &moduleName : val.ModuleNames) { + GenericArgs.push_back("-load-plugin-executable"); + GenericArgs.push_back( + ArgSaver.save(val.ExecutablePath + "#" + moduleName)); + } + break; + } + case PluginSearchOption::Kind::PluginPath: { + auto &val = entry.get(); + GenericArgs.push_back("-plugin-path"); + GenericArgs.push_back(ArgSaver.save(val.SearchPath)); + break; + } + case PluginSearchOption::Kind::ExternalPluginPath: { + auto &val = entry.get(); + GenericArgs.push_back("-external-plugin-path"); + GenericArgs.push_back( + ArgSaver.save(val.SearchPath + "#" + val.ServerPath)); + break; + } + } + } + genericSubInvocation.getFrontendOptions().InputMode = FrontendOptions::ParseInputMode::SwiftModuleInterface; if (!SearchPathOpts.RuntimeResourcePath.empty()) { diff --git a/test/CAS/macro_deps.swift b/test/CAS/macro_deps.swift new file mode 100644 index 0000000000000..b6c65d2de1327 --- /dev/null +++ b/test/CAS/macro_deps.swift @@ -0,0 +1,104 @@ +// REQUIRES: swift_swift_parser + +/// Test loading dependencies that has macros. +// RUN: %empty-directory(%t) +// RUN: split-file %s %t + +/// Build macros. +// RUN: %host-build-swift -swift-version 5 -emit-library -o %t/%target-library-name(MacroOne) -module-name=MacroOne %t/macro-1.swift +// RUN: %host-build-swift -swift-version 5 -emit-library -o %t/%target-library-name(MacroTwo) -module-name=MacroTwo %t/macro-2.swift + +/// Build binary module that depends on textual module that uses macro. +// RUN: %target-swift-frontend -emit-module -module-cache-path %t/clang-module-cache %t/test.swift -module-name Test -o %t/include/Test.swiftmodule -I %t/include \ +// RUN: -O -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -swift-version 5 -external-plugin-path %t#%swift-plugin-server + +// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-serialized -module-name MyApp -module-cache-path %t/clang-module-cache -O \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -I %t/include \ +// RUN: %t/main.swift -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -external-plugin-path %t#%swift-plugin-server + +// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json Foo macroDependencies | %FileCheck %s --check-prefix=FOO-DEPS +// FOO-DEPS: MacroOne +// FOO-DEPS-NOT: MacroTwo + +// RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json MyApp macroDependencies | %FileCheck %s --check-prefix=APP-DEPS +// APP-DEPS: MacroTwo +// APP-DEPS-NOT: MacroOne + +/// Build all dependencies. +// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/SwiftShims.cmd +// RUN: %swift_frontend_plain @%t/SwiftShims.cmd +// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json Foo > %t/Foo.cmd +// RUN: %swift_frontend_plain @%t/Foo.cmd + +// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json MyApp > %t/MyApp.cmd +// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json +// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid + +// RUN: %target-swift-frontend -diagnostic-style=swift \ +// RUN: -emit-module -o %t/Test.swiftmodule -cache-compile-job -cas-path %t/cas \ +// RUN: -swift-version 5 -disable-implicit-swift-modules \ +// RUN: -external-plugin-path %t#%swift-plugin-server \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: -module-name MyApp -explicit-swift-module-map-file @%t/map.casid -O \ +// RUN: %t/main.swift @%t/MyApp.cmd + +//--- macro-1.swift +import SwiftSyntax +@_spi(ExperimentalLanguageFeature) import SwiftSyntaxMacros + +public struct AssertMacro: ExpressionMacro { + public static func expansion( + of macro: some FreestandingMacroExpansionSyntax, + in context: some MacroExpansionContext + ) -> ExprSyntax { + guard let argument = macro.arguments.first?.expression else { + fatalError("boom") + } + + return "assert(\(argument))" + } +} + +//--- macro-2.swift +import SwiftSyntax +@_spi(ExperimentalLanguageFeature) import SwiftSyntaxMacros + +public struct StringifyMacro: ExpressionMacro { + public static func expansion( + of macro: some FreestandingMacroExpansionSyntax, + in context: some MacroExpansionContext + ) -> ExprSyntax { + guard let argument = macro.arguments.first?.expression else { + fatalError("boom") + } + + return "(\(argument), \(StringLiteralExprSyntax(content: argument.description)))" + } +} + +//--- include/Foo.swiftinterface +// swift-interface-format-version: 1.0 +// swift-module-flags: -enable-library-evolution -swift-version 5 -O -module-name Foo -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import +import Swift +public func foo() +@freestanding(expression) public macro assert(_: Bool) = #externalMacro(module: "MacroOne", type: "AssertMacro") +@inlinable +public func assertFalse() { + #assert(false) +} + +//--- test.swift +import Foo +@inlinable +public func test() { + #assert(true) +} + +//--- main.swift +import Test +@freestanding(expression) macro stringify(_ value: T) -> (T, String) = #externalMacro(module: "MacroTwo", type: "StringifyMacro") + +func appTest() { + let str = #stringify("test") + test() +} diff --git a/tools/libSwiftScan/libSwiftScan.cpp b/tools/libSwiftScan/libSwiftScan.cpp index 4c42d4ac1c86d..3df62ff95865b 100644 --- a/tools/libSwiftScan/libSwiftScan.cpp +++ b/tools/libSwiftScan/libSwiftScan.cpp @@ -29,6 +29,16 @@ using namespace swift::dependencies; DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningTool, swiftscan_scanner_t) //=== Private Cleanup Functions -------------------------------------------===// +void swiftscan_macro_dependency_dispose( + swiftscan_macro_dependency_set_t *macro) { + for (unsigned i = 0; i < macro->count; ++i) { + swiftscan_string_dispose(macro->macro_dependencies[i]->moduleName); + swiftscan_string_dispose(macro->macro_dependencies[i]->libraryPath); + swiftscan_string_dispose(macro->macro_dependencies[i]->executablePath); + delete macro->macro_dependencies[i]; + } + delete macro; +} void swiftscan_dependency_info_details_dispose( swiftscan_module_details_t details) { @@ -58,6 +68,8 @@ void swiftscan_dependency_info_details_dispose( details_impl->swift_textual_details.bridging_header_include_tree); swiftscan_string_dispose( details_impl->swift_textual_details.module_cache_key); + swiftscan_macro_dependency_dispose( + details_impl->swift_textual_details.macro_dependencies); break; case SWIFTSCAN_DEPENDENCY_INFO_SWIFT_BINARY: swiftscan_string_dispose( From a845dac6ce2dc1f1112ce058f2eb966aa2313b7a Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 15 Jul 2024 12:38:27 -0700 Subject: [PATCH 3/3] [ScanDependencies] Fix a memory leak in dependency graph Fix a memory leak from https://github.com/swiftlang/swift/pull/75134. (cherry picked from commit ed936853dccbae9792feb25aa422b61657cefb14) --- lib/DependencyScan/DependencyScanJSON.cpp | 2 +- lib/DependencyScan/DependencyScanningTool.cpp | 3 ++- lib/DependencyScan/ScanDependencies.cpp | 8 +++----- tools/libSwiftScan/libSwiftScan.cpp | 4 ++++ 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/DependencyScan/DependencyScanJSON.cpp b/lib/DependencyScan/DependencyScanJSON.cpp index 1e82c654125b4..782e5d8ce7b07 100644 --- a/lib/DependencyScan/DependencyScanJSON.cpp +++ b/lib/DependencyScan/DependencyScanJSON.cpp @@ -206,7 +206,7 @@ static void writeMacroDependencies(llvm::raw_ostream &out, const swiftscan_macro_dependency_set_t *macro_deps, unsigned indentLevel, bool trailingComma) { - if (macro_deps->count == 0) + if (!macro_deps) return; out.indent(indentLevel * 2); diff --git a/lib/DependencyScan/DependencyScanningTool.cpp b/lib/DependencyScan/DependencyScanningTool.cpp index c5e93e0848b7b..1f72065fd098b 100644 --- a/lib/DependencyScan/DependencyScanningTool.cpp +++ b/lib/DependencyScan/DependencyScanningTool.cpp @@ -232,7 +232,8 @@ static swiftscan_dependency_graph_t generateHollowDiagnosticOutput( false, c_string_utils::create_null(), c_string_utils::create_null(), - c_string_utils::create_null()}; + c_string_utils::create_null(), + nullptr}; hollowMainModuleInfo->details = hollowDetails; // Populate the diagnostic info diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index f2991dda7d680..b6a793974e35c 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -591,12 +591,10 @@ static void bridgeDependencyIDs(const ArrayRef dependencies, static swiftscan_macro_dependency_set_t *createMacroDependencySet( const std::map ¯oDeps) { + if (macroDeps.empty()) + return nullptr; + swiftscan_macro_dependency_set_t *set = new swiftscan_macro_dependency_set_t; - if (macroDeps.empty()) { - set->count = 0; - set->macro_dependencies = nullptr; - return set; - } set->count = macroDeps.size(); set->macro_dependencies = new swiftscan_macro_dependency_t[set->count]; unsigned SI = 0; diff --git a/tools/libSwiftScan/libSwiftScan.cpp b/tools/libSwiftScan/libSwiftScan.cpp index 3df62ff95865b..babb6641e0623 100644 --- a/tools/libSwiftScan/libSwiftScan.cpp +++ b/tools/libSwiftScan/libSwiftScan.cpp @@ -31,12 +31,16 @@ DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningTool, swiftscan_scanner_t) //=== Private Cleanup Functions -------------------------------------------===// void swiftscan_macro_dependency_dispose( swiftscan_macro_dependency_set_t *macro) { + if (!macro) + return; + for (unsigned i = 0; i < macro->count; ++i) { swiftscan_string_dispose(macro->macro_dependencies[i]->moduleName); swiftscan_string_dispose(macro->macro_dependencies[i]->libraryPath); swiftscan_string_dispose(macro->macro_dependencies[i]->executablePath); delete macro->macro_dependencies[i]; } + delete[] macro->macro_dependencies; delete macro; }