From d2396e20f7d87caa50991e243d50050b72086556 Mon Sep 17 00:00:00 2001 From: Paul LeMarquand Date: Thu, 21 Aug 2025 12:54:01 -0400 Subject: [PATCH 01/10] Add LLDB debugger support to swift-test command Introduces interactive debugging capabilities to `swift test` via a `--debugger` flag enabling developers to failing tests directly within LLDB. This adds debugging parity with `swift run --debugger` which allows users to debug their executables directly. When launching lldb this implenentation uses `execv()` to replace the `swift-test` process with LLDB. This approach avoids process hierarchy complications and ensures LLDB has full terminal control for interactive debugging features. When there is only one type of tests the debugger creates a single LLDB target configured specifically for that framework. For XCTest, this means targeting the test bundle with xctest as the executable, while Swift Testing targets the test binary directly with appropriate command-line arguments. When both testing frameworks have tests available we create multiple LLDB targets within a single debugging session. A Python script automatically switches targets as the executable exits, which lets users debug both types of tests spread across two executables in the same session. The Python script also maintains breakpoint persistence across target switches, allowing you to set a breakpoint for either test type no matter the active target. Finally, we add a `failbreak` command alias that sets breakpoint(s) that break on test failure, allowing users to automatically stop on failed tests. Issue: #8129 --- .../Miscellaneous/TestDebugging/Package.swift | 10 + .../Sources/TestDebugging/TestDebugging.swift | 23 + .../TestDebuggingTests.swift | 34 ++ Sources/Commands/SwiftTestCommand.swift | 248 +++++++- .../Commands/Utilities/TestingSupport.swift | 559 ++++++++++++++++++ Tests/CommandsTests/PackageCommandTests.swift | 4 +- Tests/CommandsTests/TestCommandTests.swift | 293 ++++++++- 7 files changed, 1139 insertions(+), 32 deletions(-) create mode 100644 Fixtures/Miscellaneous/TestDebugging/Package.swift create mode 100644 Fixtures/Miscellaneous/TestDebugging/Sources/TestDebugging/TestDebugging.swift create mode 100644 Fixtures/Miscellaneous/TestDebugging/Tests/TestDebuggingTests/TestDebuggingTests.swift diff --git a/Fixtures/Miscellaneous/TestDebugging/Package.swift b/Fixtures/Miscellaneous/TestDebugging/Package.swift new file mode 100644 index 00000000000..96687b0b207 --- /dev/null +++ b/Fixtures/Miscellaneous/TestDebugging/Package.swift @@ -0,0 +1,10 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "TestDebugging", + targets: [ + .target(name: "TestDebugging"), + .testTarget(name: "TestDebuggingTests", dependencies: ["TestDebugging"]), + ] +) \ No newline at end of file diff --git a/Fixtures/Miscellaneous/TestDebugging/Sources/TestDebugging/TestDebugging.swift b/Fixtures/Miscellaneous/TestDebugging/Sources/TestDebugging/TestDebugging.swift new file mode 100644 index 00000000000..8a3b22fdebd --- /dev/null +++ b/Fixtures/Miscellaneous/TestDebugging/Sources/TestDebugging/TestDebugging.swift @@ -0,0 +1,23 @@ +public struct Calculator { + public init() {} + + public func add(_ a: Int, _ b: Int) -> Int { + return a + b + } + + public func subtract(_ a: Int, _ b: Int) -> Int { + return a - b + } + + public func multiply(_ a: Int, _ b: Int) -> Int { + return a * b + } + + public func divide(_ a: Int, _ b: Int) -> Int { + return a / b + } + + public func purposelyFail() -> Bool { + return false + } +} \ No newline at end of file diff --git a/Fixtures/Miscellaneous/TestDebugging/Tests/TestDebuggingTests/TestDebuggingTests.swift b/Fixtures/Miscellaneous/TestDebugging/Tests/TestDebuggingTests/TestDebuggingTests.swift new file mode 100644 index 00000000000..c5b3aca2fd3 --- /dev/null +++ b/Fixtures/Miscellaneous/TestDebugging/Tests/TestDebuggingTests/TestDebuggingTests.swift @@ -0,0 +1,34 @@ +import XCTest +import Testing +@testable import TestDebugging + +// MARK: - XCTest Suite +final class XCTestCalculatorTests: XCTestCase { + + func testAdditionPasses() { + let calculator = Calculator() + let result = calculator.add(2, 3) + XCTAssertEqual(result, 5, "Addition should return 5 for 2 + 3") + } + + func testSubtractionFails() { + let calculator = Calculator() + let result = calculator.subtract(5, 3) + XCTAssertEqual(result, 3, "This test is designed to fail - subtraction 5 - 3 should equal 2, not 3") + } +} + +// MARK: - Swift Testing Suite +@Test("Calculator Addition Works Correctly") +func calculatorAdditionPasses() { + let calculator = Calculator() + let result = calculator.add(4, 6) + #expect(result == 10, "Addition should return 10 for 4 + 6") +} + +@Test("Calculator Boolean Check Fails") +func calculatorBooleanFails() { + let calculator = Calculator() + let result = calculator.purposelyFail() + #expect(result == true, "This test is designed to fail - purposelyFail() should return false, not true") +} \ No newline at end of file diff --git a/Sources/Commands/SwiftTestCommand.swift b/Sources/Commands/SwiftTestCommand.swift index 669f04dab3b..7c54f0caf31 100644 --- a/Sources/Commands/SwiftTestCommand.swift +++ b/Sources/Commands/SwiftTestCommand.swift @@ -216,6 +216,11 @@ struct TestCommandOptions: ParsableArguments { help: "Enable code coverage.") var enableCodeCoverage: Bool = false + /// Launch tests under LLDB debugger. + @Flag(name: .customLong("debugger"), + help: "Launch tests under LLDB debugger.") + var shouldLaunchInLLDB: Bool = false + /// Configure test output. @Option(help: ArgumentHelp("", visibility: .hidden)) public var testOutput: TestOutput = .default @@ -280,8 +285,17 @@ public struct SwiftTestCommand: AsyncSwiftCommand { var results = [TestRunner.Result]() + if options.shouldLaunchInLLDB { + let result = try await runTestProductsWithLLDB( + testProducts, + productsBuildParameters: buildParameters, + swiftCommandState: swiftCommandState + ) + results.append(result) + } + // Run XCTest. - if options.testLibraryOptions.isEnabled(.xctest, swiftCommandState: swiftCommandState) { + if !options.shouldLaunchInLLDB && options.testLibraryOptions.isEnabled(.xctest, swiftCommandState: swiftCommandState) { // Validate XCTest is available on Darwin-based systems. If it's not available and we're hitting this code // path, that means the developer must have explicitly passed --enable-xctest (or the toolchain is // corrupt, I suppose.) @@ -351,7 +365,7 @@ public struct SwiftTestCommand: AsyncSwiftCommand { } // Run Swift Testing (parallel or not, it has a single entry point.) - if options.testLibraryOptions.isEnabled(.swiftTesting, swiftCommandState: swiftCommandState) { + if !options.shouldLaunchInLLDB && options.testLibraryOptions.isEnabled(.swiftTesting, swiftCommandState: swiftCommandState) { lazy var testEntryPointPath = testProducts.lazy.compactMap(\.testEntryPointPath).first if options.testLibraryOptions.isExplicitlyEnabled(.swiftTesting, swiftCommandState: swiftCommandState) || testEntryPointPath == nil { results.append( @@ -474,6 +488,182 @@ public struct SwiftTestCommand: AsyncSwiftCommand { } } + /// Runs test products under LLDB debugger for interactive debugging. + /// + /// This method handles debugging for enabled testing libraries: + /// 1. If both XCTest and Swift Testing are enabled, prompts user to choose or runs both in separate sessions + /// 2. Validates that exactly one test product is available for debugging + /// 3. Creates a DebugTestRunner and launches LLDB with the test binary + /// + /// - Parameters: + /// - testProducts: The built test products + /// - productsBuildParameters: Build parameters for the products + /// - swiftCommandState: The Swift command state + /// - Returns: The test result (typically .success since LLDB takes over) + private func runTestProductsWithLLDB( + _ testProducts: [BuiltTestProduct], + productsBuildParameters: BuildParameters, + swiftCommandState: SwiftCommandState + ) async throws -> TestRunner.Result { + // Validate that we have exactly one test product for debugging + guard testProducts.count == 1 else { + if testProducts.isEmpty { + throw StringError("No test products found for debugging") + } else { + let productNames = testProducts.map { $0.productName }.joined(separator: ", ") + throw StringError("Multiple test products found (\(productNames)). Specify a single target with --filter when using --debugger") + } + } + + let testProduct = testProducts[0] + let toolchain = try swiftCommandState.getTargetToolchain() + + // Determine which testing libraries are enabled + let xctestEnabled = options.testLibraryOptions.isEnabled(.xctest, swiftCommandState: swiftCommandState) + let swiftTestingEnabled = options.testLibraryOptions.isEnabled(.swiftTesting, swiftCommandState: swiftCommandState) && + (options.testLibraryOptions.isExplicitlyEnabled(.swiftTesting, swiftCommandState: swiftCommandState) || + testProduct.testEntryPointPath == nil) + + // Create a list of testing libraries to run in sequence, checking for actual tests + var librariesToRun: [TestingLibrary] = [] + var skippedLibraries: [(TestingLibrary, String)] = [] + + // Only add XCTest if it's enabled AND has tests to run + if xctestEnabled { + // Always check for XCTest tests by getting test suites + let testSuites = try TestingSupport.getTestSuites( + in: testProducts, + swiftCommandState: swiftCommandState, + enableCodeCoverage: options.enableCodeCoverage, + shouldSkipBuilding: options.sharedOptions.shouldSkipBuilding, + experimentalTestOutput: options.enableExperimentalTestOutput, + sanitizers: globalOptions.build.sanitizers + ) + let filteredTests = try testSuites + .filteredTests(specifier: options.testCaseSpecifier) + .skippedTests(specifier: options.skippedTests(fileSystem: swiftCommandState.fileSystem)) + + if !filteredTests.isEmpty { + librariesToRun.append(.xctest) + } else { + skippedLibraries.append((.xctest, "no XCTest tests found")) + } + } + + if swiftTestingEnabled { + librariesToRun.append(.swiftTesting) + } + + // Ensure we have at least one library to run + guard !librariesToRun.isEmpty else { + if !skippedLibraries.isEmpty { + let skippedMessages = skippedLibraries.map { library, reason in + let libraryName = library == .xctest ? "XCTest" : "Swift Testing" + return "\(libraryName): \(reason)" + } + throw StringError("No testing libraries have tests to debug. Skipped: \(skippedMessages.joined(separator: ", "))") + } + throw StringError("No testing libraries are enabled for debugging") + } + + try await runTestLibrariesWithLLDB( + testProduct: testProduct, + target: DebuggableTestSession( + targets: librariesToRun.map { + DebuggableTestSession.Target( + library: $0, + additionalArgs: try additionalLLDBArguments(for: $0, testProducts: testProducts, swiftCommandState: swiftCommandState), + bundlePath: testBundlePath(for: $0, testProduct: testProduct) + ) + } + ), + testProducts: testProducts, + productsBuildParameters: productsBuildParameters, + swiftCommandState: swiftCommandState, + toolchain: toolchain + ) + + // Clean up Python script file after all sessions complete + // (Breakpoint file cleanup is handled by DebugTestRunner based on SessionState) + if librariesToRun.count > 1 { + let tempDir = try swiftCommandState.fileSystem.tempDirectory + let pythonScriptFile = tempDir.appending("save_breakpoints.py") + + if swiftCommandState.fileSystem.exists(pythonScriptFile) { + try? swiftCommandState.fileSystem.removeFileTree(pythonScriptFile) + } + } + + return .success + } + + private func additionalLLDBArguments(for library: TestingLibrary, testProducts: [BuiltTestProduct], swiftCommandState: SwiftCommandState) throws -> [String] { + // Determine test binary path and arguments based on the testing library + switch library { + case .xctest: + let (xctestArgs, _) = try xctestArgs(for: testProducts, swiftCommandState: swiftCommandState) + return xctestArgs + + case .swiftTesting: + let commandLineArguments = CommandLine.arguments.dropFirst() + var swiftTestingArgs = ["--testing-library", "swift-testing", "--enable-swift-testing"] + + if let separatorIndex = commandLineArguments.firstIndex(of: "--") { + // Only pass arguments after the "--" separator + swiftTestingArgs += Array(commandLineArguments.dropFirst(separatorIndex + 1)) + } + return swiftTestingArgs + } + } + + private func testBundlePath(for library: TestingLibrary, testProduct: BuiltTestProduct) -> AbsolutePath { + switch library { + case .xctest: + testProduct.bundlePath + case .swiftTesting: + testProduct.binaryPath + } + } + + /// Runs a single testing library under LLDB debugger. + /// + /// - Parameters: + /// - testProduct: The test product to debug + /// - library: The testing library to run + /// - testProducts: All built test products (for XCTest args generation) + /// - productsBuildParameters: Build parameters for the products + /// - swiftCommandState: The Swift command state + /// - toolchain: The toolchain to use + /// - sessionState: The debugging session state for breakpoint persistence + private func runTestLibrariesWithLLDB( + testProduct: BuiltTestProduct, + target: DebuggableTestSession, + testProducts: [BuiltTestProduct], + productsBuildParameters: BuildParameters, + swiftCommandState: SwiftCommandState, + toolchain: UserToolchain + ) async throws { + // Create and launch the debug test runner + let debugRunner = DebugTestRunner( + target: target, + buildParameters: productsBuildParameters, + toolchain: toolchain, + testEnv: try TestingSupport.constructTestEnvironment( + toolchain: toolchain, + destinationBuildParameters: productsBuildParameters, + sanitizers: globalOptions.build.sanitizers, + library: .xctest // TODO + ), + cancellator: swiftCommandState.cancellator, + fileSystem: swiftCommandState.fileSystem, + observabilityScope: swiftCommandState.observabilityScope, + verbose: globalOptions.logging.verbose + ) + + // Launch LLDB using AsyncProcess with proper input/output forwarding + try debugRunner.run() + } + private func runTestProducts( _ testProducts: [BuiltTestProduct], additionalArguments: [String], @@ -481,20 +671,17 @@ public struct SwiftTestCommand: AsyncSwiftCommand { swiftCommandState: SwiftCommandState, library: TestingLibrary ) async throws -> TestRunner.Result { - // Pass through all arguments from the command line to Swift Testing. + // Pass through arguments that come after "--" to Swift Testing. var additionalArguments = additionalArguments if library == .swiftTesting { - // Reconstruct the arguments list. If an xUnit path was specified, remove it. - var commandLineArguments = [String]() - var originalCommandLineArguments = CommandLine.arguments.dropFirst().makeIterator() - while let arg = originalCommandLineArguments.next() { - if arg == "--xunit-output" { - _ = originalCommandLineArguments.next() - } else { - commandLineArguments.append(arg) - } + // Only pass arguments that come after the "--" separator to Swift Testing + let allCommandLineArguments = CommandLine.arguments.dropFirst() + + if let separatorIndex = allCommandLineArguments.firstIndex(of: "--") { + // Only pass arguments after the "--" separator + let testArguments = Array(allCommandLineArguments.dropFirst(separatorIndex + 1)) + additionalArguments += testArguments } - additionalArguments += commandLineArguments if var xunitPath = options.xUnitOutput { if options.testLibraryOptions.isEnabled(.xctest, swiftCommandState: swiftCommandState) { @@ -667,6 +854,11 @@ public struct SwiftTestCommand: AsyncSwiftCommand { /// /// - Throws: if a command argument is invalid private func validateArguments(swiftCommandState: SwiftCommandState) throws { + // Validation for --debugger first, since it affects other validations. + if options.shouldLaunchInLLDB { + try validateLLDBCompatibility(swiftCommandState: swiftCommandState) + } + // Validation for --num-workers. if let workers = options.numberOfWorkers { // The --num-worker option should be called with --parallel. Since @@ -690,6 +882,36 @@ public struct SwiftTestCommand: AsyncSwiftCommand { } } + /// Validates that --debugger is compatible with other provided arguments + /// + /// - Throws: if --debugger is used with incompatible flags + private func validateLLDBCompatibility(swiftCommandState: SwiftCommandState) throws { + // --debugger cannot be used with release configuration + let configuration = options.globalOptions.build.configuration ?? swiftCommandState.preferredBuildConfiguration + if configuration == .release { + throw StringError("--debugger cannot be used with release configuration (debugging requires debug symbols)") + } + + // --debugger cannot be used with parallel testing + if options.shouldRunInParallel { + throw StringError("--debugger cannot be used with --parallel (debugging requires sequential execution)") + } + + // --debugger cannot be used with --num-workers (which requires --parallel anyway) + if options.numberOfWorkers != nil { + throw StringError("--debugger cannot be used with --num-workers (debugging requires sequential execution)") + } + + // --debugger cannot be used with information-only modes that exit early + if options._deprecated_shouldListTests { + throw StringError("--debugger cannot be used with --list-tests (use 'swift test list' for listing tests)") + } + + if options.shouldPrintCodeCovPath { + throw StringError("--debugger cannot be used with --show-codecov-path (debugging session cannot show paths)") + } + } + public init() {} } diff --git a/Sources/Commands/Utilities/TestingSupport.swift b/Sources/Commands/Utilities/TestingSupport.swift index 81b32961376..0ca23e610bc 100644 --- a/Sources/Commands/Utilities/TestingSupport.swift +++ b/Sources/Commands/Utilities/TestingSupport.swift @@ -12,16 +12,41 @@ import Basics import CoreCommands +import Foundation import PackageModel import SPMBuildCore import TSCUtility import Workspace +#if canImport(WinSDK) +import WinSDK +#elseif canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + import struct TSCBasic.FileSystemError import class Basics.AsyncProcess import var TSCBasic.stderrStream import var TSCBasic.stdoutStream import func TSCBasic.withTemporaryFile +import func TSCBasic.exec + +struct DebuggableTestSession { + struct Target { + let library: TestingLibrary + let additionalArgs: [String] + let bundlePath: AbsolutePath + } + + let targets: [Target] + + /// Whether this is part of a multi-session sequence + var isMultiSession: Bool { + targets.count > 1 + } +} /// Internal helper functionality for the SwiftTestTool command and for the /// plugin support. @@ -276,6 +301,540 @@ enum TestingSupport { } } +/// A class to run tests under LLDB debugger. +final class DebugTestRunner { + private let target: DebuggableTestSession + private let buildParameters: BuildParameters + private let toolchain: UserToolchain + private let testEnv: Environment + private let cancellator: Cancellator + private let fileSystem: FileSystem + private let observabilityScope: ObservabilityScope + private let verbose: Bool + + /// Creates an instance of debug test runner. + init( + target: DebuggableTestSession, + buildParameters: BuildParameters, + toolchain: UserToolchain, + testEnv: Environment, + cancellator: Cancellator, + fileSystem: FileSystem, + observabilityScope: ObservabilityScope, + verbose: Bool = false + ) { + self.target = target + self.buildParameters = buildParameters + self.toolchain = toolchain + self.testEnv = testEnv + self.cancellator = cancellator + self.fileSystem = fileSystem + self.observabilityScope = observabilityScope + self.verbose = verbose + } + + /// Launches the test binary under LLDB for interactive debugging. + /// + /// This method: + /// 1. Discovers LLDB using the toolchain + /// 2. Configures the environment for debugging + /// 3. Launches LLDB with the proper test runner as target + /// 4. Provides interactive debugging experience through appropriate process management + /// + /// **Implementation approach varies by testing library:** + /// - **XCTest**: Uses PTY (pseudo-terminal) via `runInPty()` to support LLDB's full-screen + /// terminal features while maintaining parent process control for sequential execution + /// - **Swift Testing**: Uses `exec()` to replace the current process (works because Swift Testing + /// is always the last library in the sequence, avoiding the need for sequential execution) + /// + /// The PTY approach is necessary for XCTest because LLDB requires advanced terminal features + /// (ANSI escape sequences, raw input mode, terminal sizing) that simple stdin/stdout redirection + /// cannot provide, while still allowing the parent process to show completion messages and + /// run multiple testing libraries sequentially. + /// + /// **Test Mode**: When running Swift Package Manager's own tests (detected via environment variables), + /// this method uses `AsyncProcess` instead of `exec()` to launch LLDB as a subprocess without stdin. + /// This allows the parent test process to capture LLDB's output for validation while ensuring LLDB + /// exits immediately due to lack of interactive input. + /// + /// - Throws: Various errors if LLDB cannot be found or launched + func run() throws { + let lldbPath: AbsolutePath + do { + lldbPath = try toolchain.getLLDB() + } catch { + observabilityScope.emit(error: "LLDB not found in toolchain: \(error)") + throw error + } + + let lldbArgs = try prepareLLDBArguments(for: target) + observabilityScope.emit(info: "LLDB will run: \(lldbPath.pathString) \(lldbArgs.joined(separator: " "))") + + // Set environment variables from testEnv on the current process + // so they are inherited by the exec'd LLDB process. Exec will replace + // this process. + for (key, value) in testEnv { + try Environment.set(key: key, value: value) + } + + // Check if we're running Swift Package Manager's own tests + let isRunningTests = Environment.current["SWIFTPM_TESTS_LLDB"] != nil + + if isRunningTests { + // When running tests, use AsyncProcess to launch LLDB as a subprocess + // This allows the test to capture output while LLDB exits due to no stdin + try runLLDBForTesting(lldbPath: lldbPath, args: lldbArgs) + } else { + // Normal interactive mode - use exec to replace the current process with LLDB + // This avoids PTY issues that interfere with LLDB's command line editing + try exec(path: lldbPath.pathString, args: [lldbPath.pathString] + lldbArgs) + } + } + + /// Launches LLDB as a subprocess for testing purposes. + /// + /// This method is used when running Swift Package Manager's own tests to validate + /// debugger functionality. It launches LLDB without stdin attached, which causes + /// LLDB to execute its startup commands and then exit, allowing the test to capture + /// and validate the output. + /// + /// - Parameters: + /// - lldbPath: Path to the LLDB executable + /// - args: Command line arguments for LLDB + /// - Throws: Process execution errors + private func runLLDBForTesting(lldbPath: AbsolutePath, args: [String]) throws { + let process = AsyncProcess( + arguments: [lldbPath.pathString] + args, + environment: testEnv, + outputRedirection: .collect + ) + + try process.launch() + let result = try process.waitUntilExit() + + // Print the output so tests can capture it + if let stdout = try? result.utf8Output() { + print(stdout, terminator: "") + } + if let stderr = try? result.utf8stderrOutput() { + print(stderr, terminator: "") + } + + // Exit with the same code as LLDB to indicate success/failure + switch result.exitStatus { + case .terminated(let code): + if code != 0 { + throw AsyncProcessResult.Error.nonZeroExit(result) + } + default: + throw AsyncProcessResult.Error.nonZeroExit(result) + } + } + + /// Returns the path to the Python script file. + private func pythonScriptFilePath() throws -> AbsolutePath { + let tempDir = try fileSystem.tempDirectory + return tempDir.appending("target_switcher.py") + } + + /// Prepares LLDB arguments for debugging based on the testing library. + /// + /// This method creates a temporary LLDB command file with the necessary setup commands + /// for debugging tests, including target creation, argument configuration, and symbol loading. + /// + /// - Parameter library: The testing library being used (XCTest or Swift Testing) + /// - Returns: Array of LLDB command line arguments + /// - Throws: Various errors if required tools are not found or file operations fail + private func prepareLLDBArguments(for target: DebuggableTestSession) throws -> [String] { + let tempDir = try fileSystem.tempDirectory + let lldbCommandFile = tempDir.appending("lldb-commands.txt") + + var lldbCommands: [String] = [] + if target.isMultiSession { + try setupMultipleTargets(&lldbCommands) + } else if let library = target.targets.first { + try setupSingleTarget(&lldbCommands, for: library) + } else { + throw StringError("No testing libraries found for debugging") + } + + // Clear the screen of all the previous commands to unclutter the users initial state. + // Skip clearing in verbose mode so startup commands remain visible + if !verbose { + lldbCommands.append("script print(\"\\033[H\\033[J\", end=\"\")") + } + + let commandScript = lldbCommands.joined(separator: "\n") + try fileSystem.writeFileContents(lldbCommandFile, string: commandScript) + + // Return script file arguments without batch mode to allow interactive debugging + return ["-s", lldbCommandFile.pathString] + } + + /// Sets up multiple targets when both XCTest and Swift Testing are available + private func setupMultipleTargets(_ lldbCommands: inout [String]) throws { + var hasSwiftTesting = false + var hasXCTest = false + + for testingLibrary in target.targets { + let (executable, args) = try getExecutableAndArgs(for: testingLibrary) + lldbCommands.append("target create \(executable.pathString)") + lldbCommands.append("settings clear target.run-args") + + for arg in args { + lldbCommands.append("settings append target.run-args \"\(arg)\"") + } + + let modulePath = getModulePath(for: testingLibrary) + lldbCommands.append("target modules add \"\(modulePath.pathString)\"") + + if testingLibrary.library == .swiftTesting { + hasSwiftTesting = true + } else if testingLibrary.library == .xctest { + hasXCTest = true + } + } + + setupCommandAliases(&lldbCommands, hasSwiftTesting: hasSwiftTesting, hasXCTest: hasXCTest) + + // Create the target switching Python script + let scriptPath = try createTargetSwitchingScript() + lldbCommands.append("command script import \"\(scriptPath.pathString)\"") + + // Select the first target and launch with pause on main + lldbCommands.append("target select 0") + } + + /// Sets up a single target when only one testing library is available + private func setupSingleTarget(_ lldbCommands: inout [String], for target: DebuggableTestSession.Target) throws { + let (executable, args) = try getExecutableAndArgs(for: target) + // Create target + lldbCommands.append("target create \(executable.pathString)") + lldbCommands.append("settings clear target.run-args") + + // Add arguments + for arg in args { + lldbCommands.append("settings append target.run-args \"\(arg)\"") + } + + // Load symbols for the test bundle + let modulePath = getModulePath(for: target) + lldbCommands.append("target modules add \"\(modulePath.pathString)\"") + + setupCommandAliases(&lldbCommands, hasSwiftTesting: target.library == .swiftTesting, hasXCTest: target.library == .xctest) + } + + private func setupCommandAliases(_ lldbCommands: inout [String], hasSwiftTesting: Bool, hasXCTest: Bool) { + #if os(macOS) + let swiftTestingFailureBreakpoint = "-s Testing -n \"failureBreakpoint()\"" + let xctestFailureBreakpoint = "-n \"_XCTFailureBreakpoint\"" + #elseif os(Windows) + let swiftTestingFailureBreakpoint = "-s Testing.dll -n \"failureBreakpoint()\"" + let xctestFailureBreakpoint = "-s XCTest.dll -n \"XCTest.XCTestCase.recordFailure\"" + #else + let swiftTestingFailureBreakpoint = "-s libTesting.so -n \"Testing.failureBreakpoint\"" + let xctestFailureBreakpoint = "-s libXCTest.so -n \"XCTest.XCTestCase.recordFailure\"" + #endif + + // Add clear screen alias + lldbCommands.append("command alias clear script print(\"\\033[H\\033[J\", end=\"\")") + + // Add failure breakpoint commands based on available libraries + if hasSwiftTesting && hasXCTest { + lldbCommands.append("command alias failbreak script lldb.debugger.HandleCommand('breakpoint set \(swiftTestingFailureBreakpoint)'); lldb.debugger.HandleCommand('breakpoint set \(xctestFailureBreakpoint)')") + } else if hasSwiftTesting { + lldbCommands.append("command alias failbreak breakpoint set \(swiftTestingFailureBreakpoint)") + } else if hasXCTest { + lldbCommands.append("command alias failbreak breakpoint set \(xctestFailureBreakpoint)") + } + } + + /// Gets the executable path and arguments for a given testing library + private func getExecutableAndArgs(for target: DebuggableTestSession.Target) throws -> (AbsolutePath, [String]) { + switch target.library { + case .xctest: + #if os(macOS) + guard let xctestPath = toolchain.xctestPath else { + throw StringError("XCTest not found in toolchain") + } + return (xctestPath, [target.bundlePath.pathString] + target.additionalArgs) + #else + return (target.bundlePath, target.additionalArgs) + #endif + case .swiftTesting: + #if os(macOS) + let executable = try toolchain.getSwiftTestingHelper() + let args = ["--test-bundle-path", target.bundlePath.pathString] + target.additionalArgs + #else + let executable = target.bundlePath + let args = target.additionalArgs + #endif + return (executable, args) + } + } + + /// Gets the module path for symbol loading + private func getModulePath(for target: DebuggableTestSession.Target) -> AbsolutePath { + var modulePath = target.bundlePath + if target.library == .xctest && buildParameters.triple.isDarwin() { + if let name = target.bundlePath.components.last?.replacing(".xctest", with: "") { + if let relativePath = try? RelativePath(validating: "Contents/MacOS/\(name)") { + modulePath = target.bundlePath.appending(relativePath) + } + } + } + return modulePath + } + + /// Creates a Python script that handles automatic target switching + private func createTargetSwitchingScript() throws -> AbsolutePath { + let scriptPath = try pythonScriptFilePath() + + let pythonScript = """ +# target_switcher.py +import lldb +import threading +import time +import sys + +current_target_index = 0 +max_targets = 0 +debugger_ref = None +known_breakpoints = set() +sequence_active = True # Start active by default + +def sync_breakpoints_to_target(source_target, dest_target): + \"\"\"Synchronize breakpoints from source target to destination target.\"\"\" + if not source_target or not dest_target: + return + + def breakpoint_exists_in_target_by_spec(target, file_name, line_number, function_name): + \"\"\"Check if a breakpoint already exists in the target by specification.\"\"\" + for i in range(target.GetNumBreakpoints()): + existing_bp = target.GetBreakpointAtIndex(i) + if not existing_bp.IsValid(): + continue + + # Check function name breakpoints + if function_name: + # Get the breakpoint's function name specifications + names = lldb.SBStringList() + existing_bp.GetNames(names) + + # Check names from GetNames() + for j in range(names.GetSize()): + if names.GetStringAtIndex(j) == function_name: + return True + + # If no names found, check the description for pending breakpoints + if names.GetSize() == 0: + bp_desc = str(existing_bp).strip() + import re + match = re.search(r"name = '([^']+)'", bp_desc) + if match and match.group(1) == function_name: + return True + + # Check file/line breakpoints (only if resolved) + if file_name and line_number: + for j in range(existing_bp.GetNumLocations()): + location = existing_bp.GetLocationAtIndex(j) + if location.IsValid(): + addr = location.GetAddress() + line_entry = addr.GetLineEntry() + if line_entry.IsValid(): + existing_file_spec = line_entry.GetFileSpec() + existing_line_number = line_entry.GetLine() + if (existing_file_spec.GetFilename() == file_name and + existing_line_number == line_number): + return True + return False + + # Get all breakpoints from source target + for i in range(source_target.GetNumBreakpoints()): + bp = source_target.GetBreakpointAtIndex(i) + if not bp.IsValid(): + continue + + # Handle breakpoints by their specifications, not just resolved locations + # First check if this is a function name breakpoint + names = lldb.SBStringList() + bp.GetNames(names) + + # For pending breakpoints, GetNames() might be empty, so also check the description + bp_desc = str(bp).strip() + + # Extract function name from description if names is empty + function_names_to_sync = [] + if names.GetSize() > 0: + # Use the names from GetNames() + for j in range(names.GetSize()): + function_name = names.GetStringAtIndex(j) + if function_name: + function_names_to_sync.append(function_name) + else: + # Parse function name from description for pending breakpoints + # Description format: "1: name = 'failureBreakpoint()', module = Testing, locations = 0 (pending)" + import re + match = re.search(r"name = '([^']+)'", bp_desc) + if match: + function_name = match.group(1) + function_names_to_sync.append(function_name) + + # Sync the function name breakpoints + for function_name in function_names_to_sync: + if not breakpoint_exists_in_target_by_spec(dest_target, None, None, function_name): + new_bp = dest_target.BreakpointCreateByName(function_name) + if new_bp.IsValid(): + new_bp.SetEnabled(bp.IsEnabled()) + new_bp.SetCondition(bp.GetCondition()) + new_bp.SetIgnoreCount(bp.GetIgnoreCount()) + + # Handle resolved location-based breakpoints (file/line) + # Only process if the breakpoint has resolved locations + if bp.GetNumLocations() > 0: + for j in range(bp.GetNumLocations()): + location = bp.GetLocationAtIndex(j) + if not location.IsValid(): + continue + + addr = location.GetAddress() + line_entry = addr.GetLineEntry() + + if line_entry.IsValid(): + file_spec = line_entry.GetFileSpec() + line_number = line_entry.GetLine() + file_name = file_spec.GetFilename() + + # Check if this breakpoint already exists in destination target + if breakpoint_exists_in_target_by_spec(dest_target, file_name, line_number, None): + continue + + # Create the same breakpoint in the destination target + new_bp = dest_target.BreakpointCreateByLocation(file_spec, line_number) + if new_bp.IsValid(): + # Copy breakpoint properties + new_bp.SetEnabled(bp.IsEnabled()) + new_bp.SetCondition(bp.GetCondition()) + new_bp.SetIgnoreCount(bp.GetIgnoreCount()) + +def sync_breakpoints_to_all_targets(): + \"\"\"Synchronize breakpoints from current target to all other targets.\"\"\" + global debugger_ref, max_targets + + if not debugger_ref or max_targets <= 1: + return + + current_target = debugger_ref.GetSelectedTarget() + if not current_target: + return + + # Sync to all other targets + for i in range(max_targets): + target = debugger_ref.GetTargetAtIndex(i) + if target and target != current_target: + sync_breakpoints_to_target(current_target, target) + +def monitor_breakpoints(): + \"\"\"Monitor breakpoint changes and sync them across targets.\"\"\" + global debugger_ref, known_breakpoints, max_targets + + if max_targets <= 1: + return + + last_breakpoint_count = 0 + + while True: # Keep running forever, not just while current_target_index < max_targets + if debugger_ref: + current_target = debugger_ref.GetSelectedTarget() + if current_target: + current_bp_count = current_target.GetNumBreakpoints() + + # If breakpoint count changed, sync to all targets + if current_bp_count != last_breakpoint_count: + sync_breakpoints_to_all_targets() + last_breakpoint_count = current_bp_count + + time.sleep(0.5) # Check every 500ms + +def check_process_status(): + \"\"\"Periodically check if the current process has exited.\"\"\" + global current_target_index, max_targets, debugger_ref, sequence_active + + while True: # Keep running forever, don't exit + if debugger_ref: + target = debugger_ref.GetSelectedTarget() + if target: + process = target.GetProcess() + if process and process.GetState() == lldb.eStateExited: + # Process has exited + if sequence_active and current_target_index < max_targets: + # We're in an active sequence, trigger switch + current_target_index += 1 + + if current_target_index < max_targets: + # Switch to next target and launch immediately + print("\\n") + debugger_ref.HandleCommand(f'target select {current_target_index}') + print(" ") + + # Get target name for user feedback + new_target = debugger_ref.GetSelectedTarget() + target_name = new_target.GetExecutable().GetFilename() if new_target else "Unknown" + + # Launch the next target immediately with pause on main + debugger_ref.HandleCommand('process launch') # -m to pause on main + else: + # Reset to first target and deactivate sequence until user runs again + current_target_index = 0 + sequence_active = False # Pause automatic switching + + print("\\n") + debugger_ref.HandleCommand('target select 0') + print("\\nAll testing targets completed.") + print("Type 'run' to restart the entire test sequence from the beginning.\\n") + + # Clear the current line and move cursor to start + sys.stdout.write("\\033[2K\\r") + # Reprint a fake prompt + sys.stdout.write("(lldb) ") + sys.stdout.flush() + elif process and process.GetState() in [lldb.eStateRunning, lldb.eStateLaunching]: + # Process is running - if sequence was inactive, reactivate it + if not sequence_active: + sequence_active = True + # Find which target is currently selected to set the correct index + selected_target = debugger_ref.GetSelectedTarget() + if selected_target: + for i in range(max_targets): + if debugger_ref.GetTargetAtIndex(i) == selected_target: + current_target_index = i + break + + time.sleep(0.1) # Check every second + +def __lldb_init_module(debugger, internal_dict): + global max_targets, debugger_ref + + debugger_ref = debugger + + # Count the number of targets + max_targets = debugger.GetNumTargets() + + if max_targets > 1: + # Start the process status checker + status_thread = threading.Thread(target=check_process_status, daemon=True) + status_thread.start() + + # Start the breakpoint monitor + bp_thread = threading.Thread(target=monitor_breakpoints, daemon=True) + bp_thread.start() +""" + + try fileSystem.writeFileContents(scriptPath, string: pythonScript) + return scriptPath + } +} + extension SwiftCommandState { func buildParametersForTest( enableCodeCoverage: Bool, diff --git a/Tests/CommandsTests/PackageCommandTests.swift b/Tests/CommandsTests/PackageCommandTests.swift index 2e9f61dbfe1..64472f632c6 100644 --- a/Tests/CommandsTests/PackageCommandTests.swift +++ b/Tests/CommandsTests/PackageCommandTests.swift @@ -44,8 +44,8 @@ fileprivate func execute( try localFileSystem.writeFileContents(packagePath.appending("Package.swift"), string: manifest) } - // don't ignore local packages when caching - environment["SWIFTPM_TESTS_PACKAGECACHE"] = "1" + environment["SWIFTPM_TESTS_LLDB"] = "1" + return try await executeSwiftPackage( packagePath, configuration: configuration, diff --git a/Tests/CommandsTests/TestCommandTests.swift b/Tests/CommandsTests/TestCommandTests.swift index 452526c9b12..8b5c4c30ee0 100644 --- a/Tests/CommandsTests/TestCommandTests.swift +++ b/Tests/CommandsTests/TestCommandTests.swift @@ -21,6 +21,22 @@ import _InternalTestSupport import TSCTestSupport import Testing +fileprivate func execute( + _ args: [String], + packagePath: AbsolutePath? = nil, + configuration: BuildConfiguration = .debug, + buildSystem: BuildSystemProvider.Kind, + throwIfCommandFails: Bool = true +) async throws -> (stdout: String, stderr: String) { + try await executeSwiftTest( + packagePath, + configuration: configuration, + extraArgs: args, + throwIfCommandFails: throwIfCommandFails, + buildSystem: buildSystem, + ) +} + @Suite( .serialized, // to limit the number of swift executable running. .tags( @@ -30,22 +46,6 @@ import Testing ) struct TestCommandTests { - private func execute( - _ args: [String], - packagePath: AbsolutePath? = nil, - configuration: BuildConfiguration = .debug, - buildSystem: BuildSystemProvider.Kind, - throwIfCommandFails: Bool = true - ) async throws -> (stdout: String, stderr: String) { - try await executeSwiftTest( - packagePath, - configuration: configuration, - extraArgs: args, - throwIfCommandFails: throwIfCommandFails, - buildSystem: buildSystem, - ) - } - @Test( arguments: SupportedBuildSystemOnAllPlatforms, BuildConfiguration.allCases, ) @@ -1138,7 +1138,7 @@ struct TestCommandTests { try await fixture(name: "Miscellaneous/Errors/FatalErrorInSingleXCTest/TypeLibrary") { fixturePath in // WHEN swift-test is executed let error = await #expect(throws: SwiftPMError.self) { - try await self.execute( + try await execute( [], packagePath: fixturePath, configuration: configuration, @@ -1173,4 +1173,263 @@ struct TestCommandTests { } } + // MARK: - LLDB Flag Validation Tests + + @Suite + struct LLDBTests { + @Test(arguments: SupportedBuildSystemOnAllPlatforms) + func lldbWithParallelThrowsError(buildSystem: BuildSystemProvider.Kind) async throws { + let configuration = BuildConfiguration.debug + try await fixture(name: "Miscellaneous/EchoExecutable") { fixturePath in + let error = await #expect(throws: SwiftPMError.self) { + try await execute( + ["--debugger", "--parallel"], + packagePath: fixturePath, + configuration: configuration, + buildSystem: buildSystem + ) + } + + guard case let SwiftPMError.executionFailure(_, stdout, stderr) = try #require(error) else { + Issue.record("Incorrect error was raised.") + return + } + + #expect( + stderr.contains("error: --debugger cannot be used with --parallel (debugging requires sequential execution)"), + "got stdout: \(stdout), stderr: \(stderr)", + ) + } + } + + @Test(arguments: SupportedBuildSystemOnAllPlatforms) + func lldbWithNumWorkersThrowsError(buildSystem: BuildSystemProvider.Kind) async throws { + let configuration = BuildConfiguration.debug + try await fixture(name: "Miscellaneous/EchoExecutable") { fixturePath in + let error = await #expect(throws: SwiftPMError.self) { + try await execute( + ["--debugger", "--parallel", "--num-workers", "2"], + packagePath: fixturePath, + configuration: configuration, + buildSystem: buildSystem, + ) + } + guard case let SwiftPMError.executionFailure(_, stdout, stderr) = try #require(error) else { + Issue.record("Incorrect error was raised.") + return + } + + // Should hit the --parallel error first since validation is done in order + #expect( + stderr.contains("error: --debugger cannot be used with --parallel (debugging requires sequential execution)"), + "got stdout: \(stdout), stderr: \(stderr)", + ) + } + } + + @Test(arguments: SupportedBuildSystemOnAllPlatforms) + func lldbWithNumWorkersOnlyThrowsError(buildSystem: BuildSystemProvider.Kind) async throws { + let configuration = BuildConfiguration.debug + try await fixture(name: "Miscellaneous/EchoExecutable") { fixturePath in + let error = await #expect(throws: SwiftPMError.self) { + try await execute( + ["--debugger", "--num-workers", "2"], + packagePath: fixturePath, + configuration: configuration, + buildSystem: buildSystem, + ) + } + guard case let SwiftPMError.executionFailure(_, stdout, stderr) = try #require(error) else { + Issue.record("Incorrect error was raised.") + return + } + + #expect( + stderr.contains("error: --debugger cannot be used with --num-workers (debugging requires sequential execution)"), + "got stdout: \(stdout), stderr: \(stderr)", + ) + } + } + + @Test(arguments: SupportedBuildSystemOnAllPlatforms) + func lldbWithListTestsThrowsError(buildSystem: BuildSystemProvider.Kind) async throws { + let configuration = BuildConfiguration.debug + try await fixture(name: "Miscellaneous/EchoExecutable") { fixturePath in + let error = await #expect(throws: SwiftPMError.self) { + try await execute( + ["--debugger", "--list-tests"], + packagePath: fixturePath, + configuration: configuration, + buildSystem: buildSystem, + ) + } + guard case let SwiftPMError.executionFailure(_, stdout, stderr) = try #require(error) else { + Issue.record("Incorrect error was raised.") + return + } + + #expect( + stderr.contains("error: --debugger cannot be used with --list-tests (use 'swift test list' for listing tests)"), + "got stdout: \(stdout), stderr: \(stderr)", + ) + } + } + + @Test(arguments: SupportedBuildSystemOnAllPlatforms) + func lldbWithShowCodecovPathThrowsError(buildSystem: BuildSystemProvider.Kind) async throws { + let configuration = BuildConfiguration.debug + try await fixture(name: "Miscellaneous/EchoExecutable") { fixturePath in + let error = await #expect(throws: SwiftPMError.self) { + try await execute( + ["--debugger", "--show-codecov-path"], + packagePath: fixturePath, + configuration: configuration, + buildSystem: buildSystem, + ) + } + guard case let SwiftPMError.executionFailure(_, stdout, stderr) = try #require(error) else { + Issue.record("Incorrect error was raised.") + return + } + + #expect( + stderr.contains("error: --debugger cannot be used with --show-codecov-path (debugging session cannot show paths)"), + "got stdout: \(stdout), stderr: \(stderr)", + ) + } + } + + @Test(arguments: SupportedBuildSystemOnAllPlatforms) + func lldbWithReleaseConfigurationThrowsError(buildSystem: BuildSystemProvider.Kind) async throws { + try await fixture(name: "Miscellaneous/EchoExecutable") { fixturePath in + let error = await #expect(throws: SwiftPMError.self) { + try await execute( + ["--debugger", "-c", "release"], + packagePath: fixturePath, + configuration: .release, + buildSystem: buildSystem, + ) + } + guard case let SwiftPMError.executionFailure(_, stdout, stderr) = try #require(error) else { + Issue.record("Incorrect error was raised.") + return + } + + #expect( + stderr.contains("error: --debugger cannot be used with release configuration (debugging requires debug symbols)"), + "got stdout: \(stdout), stderr: \(stderr)", + ) + } + } + + @Test(arguments: SupportedBuildSystemOnAllPlatforms) + func lldbWithCompatibleFlagsDoesNotThrowValidationError(buildSystem: BuildSystemProvider.Kind) async throws { + let configuration = BuildConfiguration.debug + try await fixture(name: "Miscellaneous/EchoExecutable") { fixturePath in + let (stdout, stderr) = try await execute( + ["--debugger", "--filter", ".*", "--skip", "sometest", "--enable-testable-imports"], + packagePath: fixturePath, + configuration: configuration, + buildSystem: buildSystem, + ) + + #expect( + !stderr.contains("error: --debugger cannot be used with"), + "got stdout: \(stdout), stderr: \(stderr)", + ) + } + } + + @Test(arguments: SupportedBuildSystemOnAllPlatforms) + func debuggerFlagWithXCTestSuite(buildSystem: BuildSystemProvider.Kind) async throws { + let configuration = BuildConfiguration.debug + try await fixture(name: "Miscellaneous/TestDebugging") { fixturePath in + let (stdout, stderr) = try await execute( + ["--debugger", "--disable-swift-testing", "--verbose"], + packagePath: fixturePath, + configuration: configuration, + buildSystem: buildSystem, + ) + + #expect( + !stderr.contains("error: --debugger cannot be used with"), + "got stdout: \(stdout), stderr: \(stderr)", + ) + + #expect( + stdout.contains("target create") && stdout.contains("xctest"), + "Expected LLDB to target xctest binary, got stdout: \(stdout), stderr: \(stderr)", + ) + + #expect( + stdout.contains("failbreak breakpoint set -n \"_XCTFailureBreakpoint\""), + "Expected XCTest failure breakpoint setup, got stdout: \(stdout), stderr: \(stderr)", + ) + } + } + + @Test(arguments: SupportedBuildSystemOnAllPlatforms) + func debuggerFlagWithSwiftTestingSuite(buildSystem: BuildSystemProvider.Kind) async throws { + let configuration = BuildConfiguration.debug + try await fixture(name: "Miscellaneous/TestDebugging") { fixturePath in + let (stdout, stderr) = try await execute( + ["--debugger", "--disable-xctest", "--verbose"], + packagePath: fixturePath, + configuration: configuration, + buildSystem: buildSystem, + ) + + #expect( + !stderr.contains("error: --debugger cannot be used with"), + "got stdout: \(stdout), stderr: \(stderr)", + ) + + #expect( + stdout.contains("target create") && stdout.contains("swiftpm-testing-helper"), + "Expected LLDB to target swiftpm-testing-helper binary, got stdout: \(stdout), stderr: \(stderr)", + ) + + #expect( + stdout.contains("failbreak breakpoint set -s Testing -n \"failureBreakpoint()\""), + "Expected Swift Testing failure breakpoint setup, got stdout: \(stdout), stderr: \(stderr)", + ) + } + } + + @Test(arguments: SupportedBuildSystemOnAllPlatforms) + func debuggerFlagWithBothTestingSuites(buildSystem: BuildSystemProvider.Kind) async throws { + let configuration = BuildConfiguration.debug + try await fixture(name: "Miscellaneous/TestDebugging") { fixturePath in + let (stdout, stderr) = try await execute( + ["--debugger", "--verbose"], + packagePath: fixturePath, + configuration: configuration, + buildSystem: buildSystem, + ) + + #expect( + !stderr.contains("error: --debugger cannot be used with"), + "got stdout: \(stdout), stderr: \(stderr)", + ) + + #expect( + stdout.contains("target create"), + "Expected LLDB to create targets, got stdout: \(stdout), stderr: \(stderr)", + ) + + #expect( + stdout.contains("failbreak") && ( + stdout.contains("_XCTFailureBreakpoint") || + stdout.contains("failureBreakpoint()") + ), + "Expected combined failure breakpoint setup, got stdout: \(stdout), stderr: \(stderr)", + ) + + #expect( + stdout.contains("command script import"), + "Expected Python script import for multi-target switching, got stdout: \(stdout), stderr: \(stderr)", + ) + } + } + } } From 03655c8d833489582689ccc00154cb950b15b600 Mon Sep 17 00:00:00 2001 From: Paul LeMarquand Date: Wed, 3 Sep 2025 08:53:53 -0400 Subject: [PATCH 02/10] Update argument help string --- Sources/Commands/SwiftTestCommand.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Commands/SwiftTestCommand.swift b/Sources/Commands/SwiftTestCommand.swift index 7c54f0caf31..7967b61c1b8 100644 --- a/Sources/Commands/SwiftTestCommand.swift +++ b/Sources/Commands/SwiftTestCommand.swift @@ -218,7 +218,7 @@ struct TestCommandOptions: ParsableArguments { /// Launch tests under LLDB debugger. @Flag(name: .customLong("debugger"), - help: "Launch tests under LLDB debugger.") + help: "Launch the tests in a debugger session.") var shouldLaunchInLLDB: Bool = false /// Configure test output. From 2e4e87748fa35efac3094eb2de7c92fb036efd1d Mon Sep 17 00:00:00 2001 From: Paul LeMarquand Date: Wed, 3 Sep 2025 09:38:53 -0400 Subject: [PATCH 03/10] Fixup tests on Linux/Windows --- Tests/CommandsTests/TestCommandTests.swift | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Tests/CommandsTests/TestCommandTests.swift b/Tests/CommandsTests/TestCommandTests.swift index 8b5c4c30ee0..88bee1b25ee 100644 --- a/Tests/CommandsTests/TestCommandTests.swift +++ b/Tests/CommandsTests/TestCommandTests.swift @@ -1362,8 +1362,8 @@ struct TestCommandTests { ) #expect( - stdout.contains("failbreak breakpoint set -n \"_XCTFailureBreakpoint\""), - "Expected XCTest failure breakpoint setup, got stdout: \(stdout), stderr: \(stderr)", + stdout.contains("failbreak breakpoint set"), + "Expected a failure breakpoint to be setup, got stdout: \(stdout), stderr: \(stderr)", ) } } @@ -1390,7 +1390,7 @@ struct TestCommandTests { ) #expect( - stdout.contains("failbreak breakpoint set -s Testing -n \"failureBreakpoint()\""), + stdout.contains("failbreak breakpoint set"), "Expected Swift Testing failure breakpoint setup, got stdout: \(stdout), stderr: \(stderr)", ) } @@ -1418,10 +1418,7 @@ struct TestCommandTests { ) #expect( - stdout.contains("failbreak") && ( - stdout.contains("_XCTFailureBreakpoint") || - stdout.contains("failureBreakpoint()") - ), + getNumberOfMatches(of: "breakpoint set", in: stdout) == 2, "Expected combined failure breakpoint setup, got stdout: \(stdout), stderr: \(stderr)", ) From 5077640ebef68a0f235ae4bd935a9eb4ec37dede Mon Sep 17 00:00:00 2001 From: Paul LeMarquand Date: Wed, 3 Sep 2025 15:27:59 -0400 Subject: [PATCH 04/10] Refactor validation tests to avoid command invocation --- Tests/CommandsTests/TestCommandTests.swift | 271 +++++++++++---------- 1 file changed, 142 insertions(+), 129 deletions(-) diff --git a/Tests/CommandsTests/TestCommandTests.swift b/Tests/CommandsTests/TestCommandTests.swift index 88bee1b25ee..47257a0195d 100644 --- a/Tests/CommandsTests/TestCommandTests.swift +++ b/Tests/CommandsTests/TestCommandTests.swift @@ -10,10 +10,11 @@ // //===----------------------------------------------------------------------===// -import Foundation +@testable import Commands +@testable import CoreCommands +import Foundation import Basics -import Commands import struct SPMBuildCore.BuildSystemProvider import enum PackageModel.BuildConfiguration import PackageModel @@ -21,6 +22,10 @@ import _InternalTestSupport import TSCTestSupport import Testing +import struct ArgumentParser.ExitCode +import protocol ArgumentParser.AsyncParsableCommand +import class TSCBasic.BufferedOutputByteStream + fileprivate func execute( _ args: [String], packagePath: AbsolutePath? = nil, @@ -1179,165 +1184,117 @@ struct TestCommandTests { struct LLDBTests { @Test(arguments: SupportedBuildSystemOnAllPlatforms) func lldbWithParallelThrowsError(buildSystem: BuildSystemProvider.Kind) async throws { - let configuration = BuildConfiguration.debug - try await fixture(name: "Miscellaneous/EchoExecutable") { fixturePath in - let error = await #expect(throws: SwiftPMError.self) { - try await execute( - ["--debugger", "--parallel"], - packagePath: fixturePath, - configuration: configuration, - buildSystem: buildSystem - ) - } + let args = args(["--debugger", "--parallel"], for: buildSystem) + let command = try #require(SwiftTestCommand.parseAsRoot(args) as? SwiftTestCommand) + let (state, outputStream) = try commandState() - guard case let SwiftPMError.executionFailure(_, stdout, stderr) = try #require(error) else { - Issue.record("Incorrect error was raised.") - return - } - - #expect( - stderr.contains("error: --debugger cannot be used with --parallel (debugging requires sequential execution)"), - "got stdout: \(stdout), stderr: \(stderr)", - ) + let error = await #expect(throws: ExitCode.self) { + try await command.run(state) } + + #expect(error == ExitCode.failure, "Expected ExitCode.failure, got \(String(describing: error))") + + let errorDescription = outputStream.bytes.description + #expect( + errorDescription.contains("--debugger cannot be used with --parallel"), + "Expected error about incompatible flags, got: \(errorDescription)" + ) } @Test(arguments: SupportedBuildSystemOnAllPlatforms) func lldbWithNumWorkersThrowsError(buildSystem: BuildSystemProvider.Kind) async throws { - let configuration = BuildConfiguration.debug - try await fixture(name: "Miscellaneous/EchoExecutable") { fixturePath in - let error = await #expect(throws: SwiftPMError.self) { - try await execute( - ["--debugger", "--parallel", "--num-workers", "2"], - packagePath: fixturePath, - configuration: configuration, - buildSystem: buildSystem, - ) - } - guard case let SwiftPMError.executionFailure(_, stdout, stderr) = try #require(error) else { - Issue.record("Incorrect error was raised.") - return - } + let args = args(["--debugger", "--parallel", "--num-workers", "2"], for: buildSystem) + let command = try #require(SwiftTestCommand.parseAsRoot(args) as? SwiftTestCommand) + let (state, outputStream) = try commandState() - // Should hit the --parallel error first since validation is done in order - #expect( - stderr.contains("error: --debugger cannot be used with --parallel (debugging requires sequential execution)"), - "got stdout: \(stdout), stderr: \(stderr)", - ) + let error = await #expect(throws: ExitCode.self) { + try await command.run(state) } + + #expect(error == ExitCode.failure, "Expected ExitCode.failure, got \(String(describing: error))") + + let errorDescription = outputStream.bytes.description + // Should hit the --parallel error first since validation is done in order + #expect( + errorDescription.contains("--debugger cannot be used with --parallel"), + "Expected error about incompatible flags, got: \(errorDescription)" + ) } @Test(arguments: SupportedBuildSystemOnAllPlatforms) func lldbWithNumWorkersOnlyThrowsError(buildSystem: BuildSystemProvider.Kind) async throws { - let configuration = BuildConfiguration.debug - try await fixture(name: "Miscellaneous/EchoExecutable") { fixturePath in - let error = await #expect(throws: SwiftPMError.self) { - try await execute( - ["--debugger", "--num-workers", "2"], - packagePath: fixturePath, - configuration: configuration, - buildSystem: buildSystem, - ) - } - guard case let SwiftPMError.executionFailure(_, stdout, stderr) = try #require(error) else { - Issue.record("Incorrect error was raised.") - return - } + let args = args(["--debugger", "--num-workers", "2"], for: buildSystem) + let command = try #require(SwiftTestCommand.parseAsRoot(args) as? SwiftTestCommand) + let (state, outputStream) = try commandState() - #expect( - stderr.contains("error: --debugger cannot be used with --num-workers (debugging requires sequential execution)"), - "got stdout: \(stdout), stderr: \(stderr)", - ) + let error = await #expect(throws: ExitCode.self) { + try await command.run(state) } + + #expect(error == ExitCode.failure, "Expected ExitCode.failure, got \(String(describing: error))") + + let errorDescription = outputStream.bytes.description + #expect( + errorDescription.contains("--debugger cannot be used with --num-workers"), + "Expected error about incompatible flags, got: \(errorDescription)" + ) } @Test(arguments: SupportedBuildSystemOnAllPlatforms) func lldbWithListTestsThrowsError(buildSystem: BuildSystemProvider.Kind) async throws { - let configuration = BuildConfiguration.debug - try await fixture(name: "Miscellaneous/EchoExecutable") { fixturePath in - let error = await #expect(throws: SwiftPMError.self) { - try await execute( - ["--debugger", "--list-tests"], - packagePath: fixturePath, - configuration: configuration, - buildSystem: buildSystem, - ) - } - guard case let SwiftPMError.executionFailure(_, stdout, stderr) = try #require(error) else { - Issue.record("Incorrect error was raised.") - return - } + let args = args(["--debugger", "--list-tests"], for: buildSystem) + let command = try #require(SwiftTestCommand.parseAsRoot(args) as? SwiftTestCommand) + let (state, outputStream) = try commandState() - #expect( - stderr.contains("error: --debugger cannot be used with --list-tests (use 'swift test list' for listing tests)"), - "got stdout: \(stdout), stderr: \(stderr)", - ) + let error = await #expect(throws: ExitCode.self) { + try await command.run(state) } + + #expect(error == ExitCode.failure, "Expected ExitCode.failure, got \(String(describing: error))") + + let errorDescription = outputStream.bytes.description + #expect( + errorDescription.contains("--debugger cannot be used with --list-tests"), + "Expected error about incompatible flags, got: \(errorDescription)" + ) } @Test(arguments: SupportedBuildSystemOnAllPlatforms) func lldbWithShowCodecovPathThrowsError(buildSystem: BuildSystemProvider.Kind) async throws { - let configuration = BuildConfiguration.debug - try await fixture(name: "Miscellaneous/EchoExecutable") { fixturePath in - let error = await #expect(throws: SwiftPMError.self) { - try await execute( - ["--debugger", "--show-codecov-path"], - packagePath: fixturePath, - configuration: configuration, - buildSystem: buildSystem, - ) - } - guard case let SwiftPMError.executionFailure(_, stdout, stderr) = try #require(error) else { - Issue.record("Incorrect error was raised.") - return - } + let args = args(["--debugger", "--show-codecov-path"], for: buildSystem) + let command = try #require(SwiftTestCommand.parseAsRoot(args) as? SwiftTestCommand) + let (state, outputStream) = try commandState() - #expect( - stderr.contains("error: --debugger cannot be used with --show-codecov-path (debugging session cannot show paths)"), - "got stdout: \(stdout), stderr: \(stderr)", - ) + let error = await #expect(throws: ExitCode.self) { + try await command.run(state) } + + #expect(error == ExitCode.failure, "Expected ExitCode.failure, got \(String(describing: error))") + + let errorDescription = outputStream.bytes.description + #expect( + errorDescription.contains("--debugger cannot be used with --show-codecov-path"), + "Expected error about incompatible flags, got: \(errorDescription)" + ) } @Test(arguments: SupportedBuildSystemOnAllPlatforms) func lldbWithReleaseConfigurationThrowsError(buildSystem: BuildSystemProvider.Kind) async throws { - try await fixture(name: "Miscellaneous/EchoExecutable") { fixturePath in - let error = await #expect(throws: SwiftPMError.self) { - try await execute( - ["--debugger", "-c", "release"], - packagePath: fixturePath, - configuration: .release, - buildSystem: buildSystem, - ) - } - guard case let SwiftPMError.executionFailure(_, stdout, stderr) = try #require(error) else { - Issue.record("Incorrect error was raised.") - return - } + let args = args(["--debugger"], for: buildSystem, buildConfiguration: .release) + let command = try #require(SwiftTestCommand.parseAsRoot(args) as? SwiftTestCommand) + let (state, outputStream) = try commandState() - #expect( - stderr.contains("error: --debugger cannot be used with release configuration (debugging requires debug symbols)"), - "got stdout: \(stdout), stderr: \(stderr)", - ) + let error = await #expect(throws: ExitCode.self) { + try await command.run(state) } - } - @Test(arguments: SupportedBuildSystemOnAllPlatforms) - func lldbWithCompatibleFlagsDoesNotThrowValidationError(buildSystem: BuildSystemProvider.Kind) async throws { - let configuration = BuildConfiguration.debug - try await fixture(name: "Miscellaneous/EchoExecutable") { fixturePath in - let (stdout, stderr) = try await execute( - ["--debugger", "--filter", ".*", "--skip", "sometest", "--enable-testable-imports"], - packagePath: fixturePath, - configuration: configuration, - buildSystem: buildSystem, - ) + #expect(error == ExitCode.failure, "Expected ExitCode.failure, got \(String(describing: error))") - #expect( - !stderr.contains("error: --debugger cannot be used with"), - "got stdout: \(stdout), stderr: \(stderr)", - ) - } + let errorDescription = outputStream.bytes.description + #expect( + errorDescription.contains("--debugger cannot be used with release configuration"), + "Expected error about incompatible flags, got: \(errorDescription)" + ) } @Test(arguments: SupportedBuildSystemOnAllPlatforms) @@ -1356,8 +1313,14 @@ struct TestCommandTests { "got stdout: \(stdout), stderr: \(stderr)", ) + #if os(macOS) + let targetName = "xctest" + #else + let targetName = buildSystem == .swiftbuild ? "test-runner" : "xctest" + #endif + #expect( - stdout.contains("target create") && stdout.contains("xctest"), + stdout.contains("target create") && stdout.contains(targetName), "Expected LLDB to target xctest binary, got stdout: \(stdout), stderr: \(stderr)", ) @@ -1384,8 +1347,14 @@ struct TestCommandTests { "got stdout: \(stdout), stderr: \(stderr)", ) + #if os(macOS) + let targetName = "swiftpm-testing-helper" + #else + let targetName = "TestDebuggingTests-test-runner" + #endif + #expect( - stdout.contains("target create") && stdout.contains("swiftpm-testing-helper"), + stdout.contains("target create") && stdout.contains(targetName), "Expected LLDB to target swiftpm-testing-helper binary, got stdout: \(stdout), stderr: \(stderr)", ) @@ -1428,5 +1397,49 @@ struct TestCommandTests { ) } } + + func args(_ args: [String], for buildSystem: BuildSystemProvider.Kind, buildConfiguration: BuildConfiguration = .debug) -> [String] { + return args + buildConfiguration.args + getBuildSystemArgs(for: buildSystem) + } + + func commandState() throws -> (SwiftCommandState, BufferedOutputByteStream) { + let outputStream = BufferedOutputByteStream() + + let state = try SwiftCommandState( + outputStream: outputStream, + options: try GlobalOptions.parse([]), + toolWorkspaceConfiguration: .init(shouldInstallSignalHandlers: false), + workspaceDelegateProvider: { + CommandWorkspaceDelegate( + observabilityScope: $0, + outputHandler: $1, + progressHandler: $2, + inputHandler: $3 + ) + }, + workspaceLoaderProvider: { + XcodeWorkspaceLoader( + fileSystem: $0, + observabilityScope: $1 + ) + }, + createPackagePath: false + ) + return (state, outputStream) + } } } + +fileprivate extension BuildConfiguration { + var args: [String] { + var args = ["--configuration"] + switch self { + case .debug: + args.append("debug") + case .release: + args.append("release") + } + return args + } +} + From c6d51465f10608976ad84befc96757ff1e64b6ae Mon Sep 17 00:00:00 2001 From: Paul LeMarquand Date: Thu, 4 Sep 2025 10:48:46 -0400 Subject: [PATCH 05/10] Mark some tests with known issues --- Tests/CommandsTests/TestCommandTests.swift | 187 ++++++++++++--------- 1 file changed, 110 insertions(+), 77 deletions(-) diff --git a/Tests/CommandsTests/TestCommandTests.swift b/Tests/CommandsTests/TestCommandTests.swift index 47257a0195d..2bcc50709a5 100644 --- a/Tests/CommandsTests/TestCommandTests.swift +++ b/Tests/CommandsTests/TestCommandTests.swift @@ -1297,104 +1297,137 @@ struct TestCommandTests { ) } - @Test(arguments: SupportedBuildSystemOnAllPlatforms) + @Test( + .bug(id: 0, "SWBINTTODO: MacOS: Could not find or use auto-linked library 'Testing': library 'Testing' not found"), + arguments: SupportedBuildSystemOnAllPlatforms, + ) func debuggerFlagWithXCTestSuite(buildSystem: BuildSystemProvider.Kind) async throws { - let configuration = BuildConfiguration.debug - try await fixture(name: "Miscellaneous/TestDebugging") { fixturePath in - let (stdout, stderr) = try await execute( - ["--debugger", "--disable-swift-testing", "--verbose"], - packagePath: fixturePath, - configuration: configuration, - buildSystem: buildSystem, - ) + try await withKnownIssue( + """ + MacOS: Could not find or use auto-linked library 'Testing': library 'Testing' not found + """ + ) { + let configuration = BuildConfiguration.debug + try await fixture(name: "Miscellaneous/TestDebugging") { fixturePath in + let (stdout, stderr) = try await execute( + ["--debugger", "--disable-swift-testing", "--verbose"], + packagePath: fixturePath, + configuration: configuration, + buildSystem: buildSystem, + ) - #expect( - !stderr.contains("error: --debugger cannot be used with"), - "got stdout: \(stdout), stderr: \(stderr)", - ) + #expect( + !stderr.contains("error: --debugger cannot be used with"), + "got stdout: \(stdout), stderr: \(stderr)", + ) - #if os(macOS) - let targetName = "xctest" - #else - let targetName = buildSystem == .swiftbuild ? "test-runner" : "xctest" - #endif + #if os(macOS) + let targetName = "xctest" + #else + let targetName = buildSystem == .swiftbuild ? "test-runner" : "xctest" + #endif - #expect( - stdout.contains("target create") && stdout.contains(targetName), - "Expected LLDB to target xctest binary, got stdout: \(stdout), stderr: \(stderr)", - ) + #expect( + stdout.contains("target create") && stdout.contains(targetName), + "Expected LLDB to target xctest binary, got stdout: \(stdout), stderr: \(stderr)", + ) - #expect( - stdout.contains("failbreak breakpoint set"), - "Expected a failure breakpoint to be setup, got stdout: \(stdout), stderr: \(stderr)", - ) + #expect( + stdout.contains("failbreak breakpoint set"), + "Expected a failure breakpoint to be setup, got stdout: \(stdout), stderr: \(stderr)", + ) + } + } when: { + buildSystem == .swiftbuild && ProcessInfo.hostOperatingSystem == .macOS && CiEnvironment.runningInSelfHostedPipeline } } - @Test(arguments: SupportedBuildSystemOnAllPlatforms) + @Test( + .bug(id: 0, "SWBINTTODO: MacOS: Could not find or use auto-linked library 'Testing': library 'Testing' not found"), + arguments: SupportedBuildSystemOnAllPlatforms + ) func debuggerFlagWithSwiftTestingSuite(buildSystem: BuildSystemProvider.Kind) async throws { - let configuration = BuildConfiguration.debug - try await fixture(name: "Miscellaneous/TestDebugging") { fixturePath in - let (stdout, stderr) = try await execute( - ["--debugger", "--disable-xctest", "--verbose"], - packagePath: fixturePath, - configuration: configuration, - buildSystem: buildSystem, - ) + try await withKnownIssue( + """ + MacOS: Could not find or use auto-linked library 'Testing': library 'Testing' not found + """ + ) { + let configuration = BuildConfiguration.debug + try await fixture(name: "Miscellaneous/TestDebugging") { fixturePath in + let (stdout, stderr) = try await execute( + ["--debugger", "--disable-xctest", "--verbose"], + packagePath: fixturePath, + configuration: configuration, + buildSystem: buildSystem, + ) - #expect( - !stderr.contains("error: --debugger cannot be used with"), - "got stdout: \(stdout), stderr: \(stderr)", - ) + #expect( + !stderr.contains("error: --debugger cannot be used with"), + "got stdout: \(stdout), stderr: \(stderr)", + ) - #if os(macOS) - let targetName = "swiftpm-testing-helper" - #else - let targetName = "TestDebuggingTests-test-runner" - #endif + #if os(macOS) + let targetName = "swiftpm-testing-helper" + #else + let targetName = buildSystem == .native ? "TestDebuggingPackageTests.xctest" : "TestDebuggingTests-test-runner" + #endif - #expect( - stdout.contains("target create") && stdout.contains(targetName), - "Expected LLDB to target swiftpm-testing-helper binary, got stdout: \(stdout), stderr: \(stderr)", - ) + #expect( + stdout.contains("target create") && stdout.contains(targetName), + "Expected LLDB to target swiftpm-testing-helper binary, got stdout: \(stdout), stderr: \(stderr)", + ) - #expect( - stdout.contains("failbreak breakpoint set"), - "Expected Swift Testing failure breakpoint setup, got stdout: \(stdout), stderr: \(stderr)", - ) + #expect( + stdout.contains("failbreak breakpoint set"), + "Expected Swift Testing failure breakpoint setup, got stdout: \(stdout), stderr: \(stderr)", + ) + } + } when: { + buildSystem == .swiftbuild && ProcessInfo.hostOperatingSystem == .macOS && CiEnvironment.runningInSelfHostedPipeline } } - @Test(arguments: SupportedBuildSystemOnAllPlatforms) + @Test( + .bug(id: 0, "SWBINTTODO: MacOS: Could not find or use auto-linked library 'Testing': library 'Testing' not found"), + arguments: SupportedBuildSystemOnAllPlatforms + ) func debuggerFlagWithBothTestingSuites(buildSystem: BuildSystemProvider.Kind) async throws { - let configuration = BuildConfiguration.debug - try await fixture(name: "Miscellaneous/TestDebugging") { fixturePath in - let (stdout, stderr) = try await execute( - ["--debugger", "--verbose"], - packagePath: fixturePath, - configuration: configuration, - buildSystem: buildSystem, - ) + try await withKnownIssue( + """ + MacOS: Could not find or use auto-linked library 'Testing': library 'Testing' not found + """ + ) { + let configuration = BuildConfiguration.debug + try await fixture(name: "Miscellaneous/TestDebugging") { fixturePath in + let (stdout, stderr) = try await execute( + ["--debugger", "--verbose"], + packagePath: fixturePath, + configuration: configuration, + buildSystem: buildSystem, + ) - #expect( - !stderr.contains("error: --debugger cannot be used with"), - "got stdout: \(stdout), stderr: \(stderr)", - ) + #expect( + !stderr.contains("error: --debugger cannot be used with"), + "got stdout: \(stdout), stderr: \(stderr)", + ) - #expect( - stdout.contains("target create"), - "Expected LLDB to create targets, got stdout: \(stdout), stderr: \(stderr)", - ) + #expect( + stdout.contains("target create"), + "Expected LLDB to create targets, got stdout: \(stdout), stderr: \(stderr)", + ) - #expect( - getNumberOfMatches(of: "breakpoint set", in: stdout) == 2, - "Expected combined failure breakpoint setup, got stdout: \(stdout), stderr: \(stderr)", - ) + #expect( + getNumberOfMatches(of: "breakpoint set", in: stdout) == 2, + "Expected combined failure breakpoint setup, got stdout: \(stdout), stderr: \(stderr)", + ) - #expect( - stdout.contains("command script import"), - "Expected Python script import for multi-target switching, got stdout: \(stdout), stderr: \(stderr)", - ) + #expect( + stdout.contains("command script import"), + "Expected Python script import for multi-target switching, got stdout: \(stdout), stderr: \(stderr)", + ) + } + } when: { + buildSystem == .swiftbuild && ProcessInfo.hostOperatingSystem == .macOS && CiEnvironment.runningInSelfHostedPipeline } } From d056ae8580444b13200527fffd7f2dc6addee580 Mon Sep 17 00:00:00 2001 From: Paul LeMarquand Date: Thu, 4 Sep 2025 15:37:00 -0400 Subject: [PATCH 06/10] Test command output can be written asynchronously --- .../SwiftTesting+Helpers.swift | 33 ++++++++++++++++ Tests/CommandsTests/PackageCommandTests.swift | 4 +- Tests/CommandsTests/TestCommandTests.swift | 38 ++++++++++--------- 3 files changed, 55 insertions(+), 20 deletions(-) diff --git a/Sources/_InternalTestSupport/SwiftTesting+Helpers.swift b/Sources/_InternalTestSupport/SwiftTesting+Helpers.swift index 69e8b06fee0..cd3af6c3f2b 100644 --- a/Sources/_InternalTestSupport/SwiftTesting+Helpers.swift +++ b/Sources/_InternalTestSupport/SwiftTesting+Helpers.swift @@ -10,6 +10,8 @@ import Basics import Testing +import Foundation +import class TSCBasic.BufferedOutputByteStream public func expectFileExists( at path: AbsolutePath, @@ -116,3 +118,34 @@ public func expectAsyncThrowsError( errorHandler(error) } } + +/// Checks if an output stream contains a specific string, with retry logic for asynchronous writes. +/// - Parameters: +/// - outputStream: The output stream to check +/// - needle: The string to search for in the output stream +/// - timeout: Maximum time to wait for the string to appear (default: 3 seconds) +/// - retryInterval: Time to wait between checks (default: 50 milliseconds) +/// - Returns: True if the string was found within the timeout period +public func waitForOutputStreamToContain( + _ outputStream: BufferedOutputByteStream, + _ needle: String, + timeout: TimeInterval = 3.0, + retryInterval: TimeInterval = 0.05 +) async throws -> Bool { + let description = outputStream.bytes.description + if description.contains(needle) { + return true + } + + let startTime = Date() + while Date().timeIntervalSince(startTime) < timeout { + let description = outputStream.bytes.description + if description.contains(needle) { + return true + } + + try await Task.sleep(nanoseconds: UInt64(retryInterval * 1_000_000_000)) + } + + return outputStream.bytes.description.contains(needle) +} \ No newline at end of file diff --git a/Tests/CommandsTests/PackageCommandTests.swift b/Tests/CommandsTests/PackageCommandTests.swift index 64472f632c6..2e9f61dbfe1 100644 --- a/Tests/CommandsTests/PackageCommandTests.swift +++ b/Tests/CommandsTests/PackageCommandTests.swift @@ -44,8 +44,8 @@ fileprivate func execute( try localFileSystem.writeFileContents(packagePath.appending("Package.swift"), string: manifest) } - environment["SWIFTPM_TESTS_LLDB"] = "1" - + // don't ignore local packages when caching + environment["SWIFTPM_TESTS_PACKAGECACHE"] = "1" return try await executeSwiftPackage( packagePath, configuration: configuration, diff --git a/Tests/CommandsTests/TestCommandTests.swift b/Tests/CommandsTests/TestCommandTests.swift index 2bcc50709a5..51810a41d82 100644 --- a/Tests/CommandsTests/TestCommandTests.swift +++ b/Tests/CommandsTests/TestCommandTests.swift @@ -1194,10 +1194,12 @@ struct TestCommandTests { #expect(error == ExitCode.failure, "Expected ExitCode.failure, got \(String(describing: error))") - let errorDescription = outputStream.bytes.description + // The output stream is written to asynchronously on a DispatchQueue and can + // receive output after the command has thrown. + let found = try await waitForOutputStreamToContain(outputStream, "--debugger cannot be used with --parallel") #expect( - errorDescription.contains("--debugger cannot be used with --parallel"), - "Expected error about incompatible flags, got: \(errorDescription)" + found, + "Expected error about incompatible flags, got: \(outputStream.bytes.description)" ) } @@ -1213,11 +1215,11 @@ struct TestCommandTests { #expect(error == ExitCode.failure, "Expected ExitCode.failure, got \(String(describing: error))") - let errorDescription = outputStream.bytes.description // Should hit the --parallel error first since validation is done in order + let found = try await waitForOutputStreamToContain(outputStream, "--debugger cannot be used with --parallel") #expect( - errorDescription.contains("--debugger cannot be used with --parallel"), - "Expected error about incompatible flags, got: \(errorDescription)" + found, + "Expected error about incompatible flags, got: \(outputStream.bytes.description)" ) } @@ -1233,10 +1235,10 @@ struct TestCommandTests { #expect(error == ExitCode.failure, "Expected ExitCode.failure, got \(String(describing: error))") - let errorDescription = outputStream.bytes.description + let found = try await waitForOutputStreamToContain(outputStream, "--debugger cannot be used with --num-workers") #expect( - errorDescription.contains("--debugger cannot be used with --num-workers"), - "Expected error about incompatible flags, got: \(errorDescription)" + found, + "Expected error about incompatible flags, got: \(outputStream.bytes.description)" ) } @@ -1252,10 +1254,10 @@ struct TestCommandTests { #expect(error == ExitCode.failure, "Expected ExitCode.failure, got \(String(describing: error))") - let errorDescription = outputStream.bytes.description + let found = try await waitForOutputStreamToContain(outputStream, "--debugger cannot be used with --list-tests") #expect( - errorDescription.contains("--debugger cannot be used with --list-tests"), - "Expected error about incompatible flags, got: \(errorDescription)" + found, + "Expected error about incompatible flags, got: \(outputStream.bytes.description)" ) } @@ -1271,10 +1273,10 @@ struct TestCommandTests { #expect(error == ExitCode.failure, "Expected ExitCode.failure, got \(String(describing: error))") - let errorDescription = outputStream.bytes.description + let found = try await waitForOutputStreamToContain(outputStream, "--debugger cannot be used with --show-codecov-path") #expect( - errorDescription.contains("--debugger cannot be used with --show-codecov-path"), - "Expected error about incompatible flags, got: \(errorDescription)" + found, + "Expected error about incompatible flags, got: \(outputStream.bytes.description)" ) } @@ -1290,10 +1292,10 @@ struct TestCommandTests { #expect(error == ExitCode.failure, "Expected ExitCode.failure, got \(String(describing: error))") - let errorDescription = outputStream.bytes.description + let found = try await waitForOutputStreamToContain(outputStream, "--debugger cannot be used with release configuration") #expect( - errorDescription.contains("--debugger cannot be used with release configuration"), - "Expected error about incompatible flags, got: \(errorDescription)" + found, + "Expected error about incompatible flags, got: \(outputStream.bytes.description)" ) } From 8246b8b155070e4f97f1db540b3a0d5372592eaa Mon Sep 17 00:00:00 2001 From: Paul LeMarquand Date: Fri, 5 Sep 2025 09:15:34 -0400 Subject: [PATCH 07/10] Revert now unnecessary swift testing argument handling --- Sources/Commands/SwiftTestCommand.swift | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Sources/Commands/SwiftTestCommand.swift b/Sources/Commands/SwiftTestCommand.swift index 7967b61c1b8..279b283bb88 100644 --- a/Sources/Commands/SwiftTestCommand.swift +++ b/Sources/Commands/SwiftTestCommand.swift @@ -671,17 +671,20 @@ public struct SwiftTestCommand: AsyncSwiftCommand { swiftCommandState: SwiftCommandState, library: TestingLibrary ) async throws -> TestRunner.Result { - // Pass through arguments that come after "--" to Swift Testing. + // Pass through all arguments from the command line to Swift Testing. var additionalArguments = additionalArguments if library == .swiftTesting { - // Only pass arguments that come after the "--" separator to Swift Testing - let allCommandLineArguments = CommandLine.arguments.dropFirst() - - if let separatorIndex = allCommandLineArguments.firstIndex(of: "--") { - // Only pass arguments after the "--" separator - let testArguments = Array(allCommandLineArguments.dropFirst(separatorIndex + 1)) - additionalArguments += testArguments + // Reconstruct the arguments list. If an xUnit path was specified, remove it. + var commandLineArguments = [String]() + var originalCommandLineArguments = CommandLine.arguments.dropFirst().makeIterator() + while let arg = originalCommandLineArguments.next() { + if arg == "--xunit-output" { + _ = originalCommandLineArguments.next() + } else { + commandLineArguments.append(arg) + } } + additionalArguments += commandLineArguments if var xunitPath = options.xUnitOutput { if options.testLibraryOptions.isEnabled(.xctest, swiftCommandState: swiftCommandState) { From 1f6e90ff51eb52eadbb8373a80c0b0f7a3b750fe Mon Sep 17 00:00:00 2001 From: Paul LeMarquand Date: Tue, 9 Sep 2025 09:40:05 -0400 Subject: [PATCH 08/10] Disable some tests on Windows --- Tests/CommandsTests/TestCommandTests.swift | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Tests/CommandsTests/TestCommandTests.swift b/Tests/CommandsTests/TestCommandTests.swift index 51810a41d82..0c760bdbc01 100644 --- a/Tests/CommandsTests/TestCommandTests.swift +++ b/Tests/CommandsTests/TestCommandTests.swift @@ -1306,7 +1306,8 @@ struct TestCommandTests { func debuggerFlagWithXCTestSuite(buildSystem: BuildSystemProvider.Kind) async throws { try await withKnownIssue( """ - MacOS: Could not find or use auto-linked library 'Testing': library 'Testing' not found + MacOS, .swiftbuild: Could not find or use auto-linked library 'Testing': library 'Testing' not found + Windows: Missing LLDB DLLs w/ ARM64 toolchain """ ) { let configuration = BuildConfiguration.debug @@ -1340,7 +1341,8 @@ struct TestCommandTests { ) } } when: { - buildSystem == .swiftbuild && ProcessInfo.hostOperatingSystem == .macOS && CiEnvironment.runningInSelfHostedPipeline + (buildSystem == .swiftbuild && ProcessInfo.hostOperatingSystem == .macOS && CiEnvironment.runningInSelfHostedPipeline) + || (ProcessInfo.hostOperatingSystem == .windows && CiEnvironment.runningInSelfHostedPipeline) } } @@ -1351,7 +1353,8 @@ struct TestCommandTests { func debuggerFlagWithSwiftTestingSuite(buildSystem: BuildSystemProvider.Kind) async throws { try await withKnownIssue( """ - MacOS: Could not find or use auto-linked library 'Testing': library 'Testing' not found + MacOS, .swiftbuild: Could not find or use auto-linked library 'Testing': library 'Testing' not found + Windows: Missing LLDB DLLs w/ ARM64 toolchain """ ) { let configuration = BuildConfiguration.debug @@ -1385,7 +1388,8 @@ struct TestCommandTests { ) } } when: { - buildSystem == .swiftbuild && ProcessInfo.hostOperatingSystem == .macOS && CiEnvironment.runningInSelfHostedPipeline + (buildSystem == .swiftbuild && ProcessInfo.hostOperatingSystem == .macOS && CiEnvironment.runningInSelfHostedPipeline) + || (ProcessInfo.hostOperatingSystem == .windows && CiEnvironment.runningInSelfHostedPipeline) } } @@ -1396,7 +1400,8 @@ struct TestCommandTests { func debuggerFlagWithBothTestingSuites(buildSystem: BuildSystemProvider.Kind) async throws { try await withKnownIssue( """ - MacOS: Could not find or use auto-linked library 'Testing': library 'Testing' not found + MacOS, .swiftbuild: Could not find or use auto-linked library 'Testing': library 'Testing' not found + Windows: Missing LLDB DLLs w/ ARM64 toolchain """ ) { let configuration = BuildConfiguration.debug @@ -1429,7 +1434,8 @@ struct TestCommandTests { ) } } when: { - buildSystem == .swiftbuild && ProcessInfo.hostOperatingSystem == .macOS && CiEnvironment.runningInSelfHostedPipeline + (buildSystem == .swiftbuild && ProcessInfo.hostOperatingSystem == .macOS && CiEnvironment.runningInSelfHostedPipeline) + || (ProcessInfo.hostOperatingSystem == .windows && CiEnvironment.runningInSelfHostedPipeline) } } From 028f414a77c5b09fd8b34338aac425ab8840b004 Mon Sep 17 00:00:00 2001 From: Paul LeMarquand Date: Wed, 10 Sep 2025 15:31:17 -0400 Subject: [PATCH 09/10] Consolidate BuildConfiguration.buildArgs into one spot --- Sources/_InternalTestSupport/misc.swift | 28 +++++++++++++--------- Tests/CommandsTests/TestCommandTests.swift | 15 +----------- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/Sources/_InternalTestSupport/misc.swift b/Sources/_InternalTestSupport/misc.swift index e04298e5f47..2e6d7e7ac69 100644 --- a/Sources/_InternalTestSupport/misc.swift +++ b/Sources/_InternalTestSupport/misc.swift @@ -319,7 +319,7 @@ fileprivate func setup( #else try Process.checkNonZeroExit(args: "cp", "-R", "-H", srcDir.pathString, dstDir.pathString) #endif - + // Ensure we get a clean test fixture. try localFileSystem.removeFileTree(dstDir.appending(component: ".build")) try localFileSystem.removeFileTree(dstDir.appending(component: ".swiftpm")) @@ -522,14 +522,7 @@ private func swiftArgs( Xswiftc: [String], buildSystem: BuildSystemProvider.Kind? ) -> [String] { - var args = ["--configuration"] - switch configuration { - case .debug: - args.append("debug") - case .release: - args.append("release") - } - + var args = configuration.buildArgs args += Xcc.flatMap { ["-Xcc", $0] } args += Xld.flatMap { ["-Xlinker", $0] } args += Xswiftc.flatMap { ["-Xswiftc", $0] } @@ -538,7 +531,7 @@ private func swiftArgs( return args } -@available(*, +@available(*, deprecated, renamed: "loadModulesGraph", message: "Rename for consistency: the type of this functions return value is named `ModulesGraph`." @@ -571,6 +564,19 @@ public func loadPackageGraph( ) } +extension BuildConfiguration { + public var buildArgs: [String] { + var args = ["--configuration"] + switch self { + case .debug: + args.append("debug") + case .release: + args.append("release") + } + return args + } +} + public let emptyZipFile = ByteString([0x80, 0x75, 0x05, 0x06] + [UInt8](repeating: 0x00, count: 18)) extension FileSystem { @@ -682,7 +688,7 @@ public func getNumberOfMatches(of match: String, in value: String) -> Int { } public extension String { - var withSwiftLineEnding: String { + var withSwiftLineEnding: String { return replacingOccurrences(of: "\r\n", with: "\n") } } diff --git a/Tests/CommandsTests/TestCommandTests.swift b/Tests/CommandsTests/TestCommandTests.swift index 0c760bdbc01..0255f4e31a8 100644 --- a/Tests/CommandsTests/TestCommandTests.swift +++ b/Tests/CommandsTests/TestCommandTests.swift @@ -1440,7 +1440,7 @@ struct TestCommandTests { } func args(_ args: [String], for buildSystem: BuildSystemProvider.Kind, buildConfiguration: BuildConfiguration = .debug) -> [String] { - return args + buildConfiguration.args + getBuildSystemArgs(for: buildSystem) + return args + buildConfiguration.buildArgs + getBuildSystemArgs(for: buildSystem) } func commandState() throws -> (SwiftCommandState, BufferedOutputByteStream) { @@ -1471,16 +1471,3 @@ struct TestCommandTests { } } -fileprivate extension BuildConfiguration { - var args: [String] { - var args = ["--configuration"] - switch self { - case .debug: - args.append("debug") - case .release: - args.append("release") - } - return args - } -} - From 89e4856f072e1c1b99b938d61c3f79937e1d37a1 Mon Sep 17 00:00:00 2001 From: Paul LeMarquand Date: Thu, 11 Sep 2025 20:16:27 -0400 Subject: [PATCH 10/10] Remove clear alias to be added later to both test and run cmds --- Sources/Commands/Utilities/TestingSupport.swift | 3 --- 1 file changed, 3 deletions(-) diff --git a/Sources/Commands/Utilities/TestingSupport.swift b/Sources/Commands/Utilities/TestingSupport.swift index 0ca23e610bc..85a18f7374e 100644 --- a/Sources/Commands/Utilities/TestingSupport.swift +++ b/Sources/Commands/Utilities/TestingSupport.swift @@ -536,9 +536,6 @@ final class DebugTestRunner { let xctestFailureBreakpoint = "-s libXCTest.so -n \"XCTest.XCTestCase.recordFailure\"" #endif - // Add clear screen alias - lldbCommands.append("command alias clear script print(\"\\033[H\\033[J\", end=\"\")") - // Add failure breakpoint commands based on available libraries if hasSwiftTesting && hasXCTest { lldbCommands.append("command alias failbreak script lldb.debugger.HandleCommand('breakpoint set \(swiftTestingFailureBreakpoint)'); lldb.debugger.HandleCommand('breakpoint set \(xctestFailureBreakpoint)')")