Place build.db under the scratch directory

The build database should be shared between different triple builds to
re-generate the build manifest correctly.
Also this change splits llbuild target names for each triple to avoid cache
invalidation when switching triple.

Author: kateinoigakukun

Sources/Build/BuildManifest/LLBuildManifestBuilder.swift

@@ -322,7 +322,7 @@ extension ResolvedModule {
     }
 
     public func getLLBuildTargetName(buildParameters: BuildParameters) -> String {
-        "\(self.name)-\(buildParameters.buildConfig)\(buildParameters.suffix(triple: self.buildTriple)).module"
+        "\(self.name)-\(buildParameters.triple.tripleString)-\(buildParameters.buildConfig)\(buildParameters.suffix(triple: self.buildTriple)).module"
     }
 
     public func getLLBuildResourcesCmdName(buildParameters: BuildParameters) -> String {

Sources/Build/BuildOperation.swift

@@ -52,6 +52,9 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
     /// the plugin configuration for build plugins
     let pluginConfiguration: PluginConfiguration?
 
+    /// The path to scratch space (.build) directory.
+    let scratchDirectory: AbsolutePath
+
     /// The llbuild build delegate reference.
     private var buildSystemDelegate: BuildOperationBuildSystemDelegateHandler?
 
@@ -114,6 +117,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         cacheBuildManifest: Bool,
         packageGraphLoader: @escaping () throws -> ModulesGraph,
         pluginConfiguration: PluginConfiguration? = .none,
+        scratchDirectory: AbsolutePath,
         additionalFileRules: [FileRuleDescription],
         pkgConfigDirectories: [AbsolutePath],
         dependenciesByRootPackageIdentity: [PackageIdentity: [PackageIdentity]],
@@ -136,6 +140,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         self.packageGraphLoader = packageGraphLoader
         self.additionalFileRules = additionalFileRules
         self.pluginConfiguration = pluginConfiguration
+        self.scratchDirectory = scratchDirectory
         self.pkgConfigDirectories = pkgConfigDirectories
         self.dependenciesByRootPackageIdentity = dependenciesByRootPackageIdentity
         self.rootPackageIdentityByTargetName = (try? Dictionary<String, PackageIdentity>(throwingUniqueKeysWithValues: targetsByRootPackageIdentity.lazy.flatMap { e in e.value.map { ($0, e.key) } })) ?? [:]
@@ -554,6 +559,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
                 toolsBuildParameters: pluginsBuildParameters,
                 cacheBuildManifest: false,
                 packageGraphLoader: { buildToolsGraph },
+                scratchDirectory: pluginsBuildParameters.dataPath,
                 additionalFileRules: self.additionalFileRules,
                 pkgConfigDirectories: self.pkgConfigDirectories,
                 dependenciesByRootPackageIdentity: [:],
@@ -724,7 +730,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         )
         self.buildSystemDelegate = buildSystemDelegate
 
-        let databasePath = self.productsBuildParameters.dataPath.appending("build.db").pathString
+        let databasePath = self.scratchDirectory.appending("build.db").pathString
         let buildSystem = SPMLLBuild.BuildSystem(
             buildFile: self.productsBuildParameters.llbuildManifest.pathString,
             databaseFile: databasePath,

Sources/CoreCommands/BuildSystemSupport.swift

@@ -49,6 +49,7 @@ private struct NativeBuildSystemFactory: BuildSystemFactory {
                 workDirectory: try self.swiftCommandState.getActiveWorkspace().location.pluginWorkingDirectory,
                 disableSandbox: self.swiftCommandState.shouldDisableSandbox
             ),
+            scratchDirectory: self.swiftCommandState.scratchDirectory,
             additionalFileRules: FileRuleDescription.swiftpmFileTypes,
             pkgConfigDirectories: self.swiftCommandState.options.locations.pkgConfigDirectories,
             dependenciesByRootPackageIdentity: rootPackageInfo.dependencies,

Sources/swift-bootstrap/main.swift

@@ -319,6 +319,7 @@ struct SwiftBootstrapBuildTool: ParsableCommand {
                     toolsBuildParameters: buildParameters,
                     cacheBuildManifest: false,
                     packageGraphLoader: packageGraphLoader,
+                    scratchDirectory: scratchDirectory,
                     additionalFileRules: [],
                     pkgConfigDirectories: [],
                     dependenciesByRootPackageIdentity: [:],

Tests/BuildTests/BuildOperationTests.swift

@@ -14,34 +14,60 @@
 @testable import PackageModel
 
 import Basics
+import LLBuildManifest
+@_spi(DontAdoptOutsideOfSwiftPMExposedForBenchmarksAndTestsOnly)
+import PackageGraph
+import SPMBuildCore
 import SPMTestSupport
 import XCTest
 
 import class TSCBasic.BufferedOutputByteStream
 import class TSCBasic.InMemoryFileSystem
 
+private func mockBuildOperation(
+    buildParameters: BuildParameters,
+    cacheBuildManifest: Bool = false,
+    packageGraphLoader: @escaping () -> ModulesGraph = { fatalError() },
+    scratchDirectory: AbsolutePath,
+    fs: any Basics.FileSystem,
+    observabilityScope: ObservabilityScope
+) -> BuildOperation {
+    return BuildOperation(
+        productsBuildParameters: buildParameters,
+        toolsBuildParameters: buildParameters,
+        cacheBuildManifest: cacheBuildManifest,
+        packageGraphLoader: packageGraphLoader,
+        scratchDirectory: scratchDirectory,
+        additionalFileRules: [],
+        pkgConfigDirectories: [],
+        dependenciesByRootPackageIdentity: [:],
+        targetsByRootPackageIdentity: [:],
+        outputStream: BufferedOutputByteStream(),
+        logLevel: .info,
+        fileSystem: fs,
+        observabilityScope: observabilityScope
+    )
+}
+
 final class BuildOperationTests: XCTestCase {
     func testDetectUnexpressedDependencies() throws {
-        let buildParameters = mockBuildParameters(shouldDisableLocalRpath: false)
+        let scratchDirectory = AbsolutePath("/path/to/build")
+        let triple = hostTriple
+        let buildParameters = mockBuildParameters(
+            buildPath: scratchDirectory.appending(triple.tripleString),
+            shouldDisableLocalRpath: false,
+            triple: triple
+        )
 
         let fs = InMemoryFileSystem(files: [
             "\(buildParameters.dataPath)/debug/Lunch.build/Lunch.d" : "/Best.framework"
         ])
 
         let observability = ObservabilitySystem.makeForTesting()
-        let buildOp = BuildOperation(
-            productsBuildParameters: buildParameters,
-            toolsBuildParameters: buildParameters,
-            cacheBuildManifest: false,
-            packageGraphLoader: { fatalError() },
-            additionalFileRules: [],
-            pkgConfigDirectories: [],
-            dependenciesByRootPackageIdentity: [:],
-            targetsByRootPackageIdentity: [:],
-            outputStream: BufferedOutputByteStream(),
-            logLevel: .info,
-            fileSystem: fs,
-            observabilityScope: observability.topScope
+        let buildOp = mockBuildOperation(
+            buildParameters: buildParameters,
+            scratchDirectory: scratchDirectory,
+            fs: fs, observabilityScope: observability.topScope
         )
         buildOp.detectUnexpressedDependencies(
             availableLibraries: [
@@ -65,4 +91,85 @@ final class BuildOperationTests: XCTestCase {
             ["target 'Lunch' has an unexpressed depedency on 'foo'"]
         )
     }
+
+    func testDetectProductTripleChange() throws {
+        let observability = ObservabilitySystem.makeForTesting()
+        let fs = InMemoryFileSystem(
+            emptyFiles: "/Pkg/Sources/ATarget/foo.swift"
+        )
+        let packageGraph = try loadModulesGraph(
+            fileSystem: fs,
+            manifests: [
+                .createRootManifest(
+                    displayName: "SwitchTriple",
+                    path: "/Pkg",
+                    targets: [
+                        TargetDescription(name: "ATarget"),
+                    ]
+                ),
+            ],
+            observabilityScope: observability.topScope
+        )
+        try withTemporaryDirectory { tmpDir in
+            let scratchDirectory = tmpDir.appending(".build")
+            let fs = localFileSystem
+            let triples = try [Triple("x86_64-unknown-linux-gnu"), Triple("wasm32-unknown-wasi")]
+            var llbuildManifestByTriple: [String: String] = [:]
+
+            // Perform initial builds for each triple
+            for triple in triples {
+                let buildParameters = mockBuildParameters(
+                    buildPath: scratchDirectory.appending(triple.tripleString),
+                    config: .debug,
+                    triple: triple
+                )
+                let buildOp = mockBuildOperation(
+                    buildParameters: buildParameters,
+                    cacheBuildManifest: false,
+                    packageGraphLoader: { packageGraph },
+                    scratchDirectory: scratchDirectory,
+                    fs: fs, observabilityScope: observability.topScope
+                )
+                // Generate initial llbuild manifest
+                let _ = try buildOp.getBuildDescription()
+                // Record the initial llbuild manifest as expected one
+                llbuildManifestByTriple[triple.tripleString] = try fs.readFileContents(buildParameters.llbuildManifest)
+            }
+
+            XCTAssertTrue(fs.exists(scratchDirectory.appending("debug.yaml")))
+            // FIXME: There should be a build database with manifest cache after the initial build.
+            // The initial build usually triggered with `cacheBuildManifest=false` because llbuild
+            // manifest file and description.json are not found. However, with `cacheBuildManifest=false`,
+            // `BuildOperation` does not trigger "PackageStructure" build, thus the initial build does
+            // not record the manifest cache. So "getBuildDescription" doesn't create build.db for the
+            // initial planning and the second build always need full-planning.
+            //
+            // XCTAssertTrue(fs.exists(scratchDirectory.appending("build.db")))
+
+            // Perform incremental build several times and switch triple for each time
+            for _ in 0..<4 {
+                for triple in triples {
+                    let buildParameters = mockBuildParameters(
+                        buildPath: scratchDirectory.appending(triple.tripleString),
+                        config: .debug,
+                        triple: triple
+                    )
+                    let buildOp = mockBuildOperation(
+                        buildParameters: buildParameters,
+                        cacheBuildManifest: true,
+                        packageGraphLoader: { packageGraph },
+                        scratchDirectory: scratchDirectory,
+                        fs: fs, observabilityScope: observability.topScope
+                    )
+                    // Generate llbuild manifest
+                    let _ = try buildOp.getBuildDescription()
+
+                    // Ensure that llbuild manifest is updated to the expected one
+                    let actualManifest: String = try fs.readFileContents(buildParameters.llbuildManifest)
+                    let expectedManifest = try XCTUnwrap(llbuildManifestByTriple[triple.tripleString])
+                    XCTAssertEqual(actualManifest, expectedManifest)
+                }
+            }
+        }
+    }
 }

Tests/BuildTests/LLBuildManifestBuilderTests.swift

@@ -218,6 +218,6 @@ final class LLBuildManifestBuilderTests: XCTestCase {
         let builder = LLBuildManifestBuilder(plan, fileSystem: fs, observabilityScope: scope)
         let manifest = try builder.generateManifest(at: "/manifest")
 
-        XCTAssertNotNil(manifest.commands["C.SwiftSyntax-debug-tool.module"])
+        XCTAssertNotNil(manifest.commands["C.SwiftSyntax-aarch64-unknown-linux-gnu-debug-tool.module"])
     }
 }