diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index 8e0416ba0..976aa7725 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -191,6 +191,9 @@ public struct __CommandLineArguments_v0: Sendable { /// The value of the `--parallel` or `--no-parallel` argument. public var parallel: Bool? + /// The value of the `--symbolicate-backtraces` argument. + public var symbolicateBacktraces: String? + /// The value of the `--verbose` argument. public var verbose: Bool? @@ -280,6 +283,7 @@ extension __CommandLineArguments_v0: Codable { enum CodingKeys: String, CodingKey { case listTests case parallel + case symbolicateBacktraces case verbose case veryVerbose case quiet @@ -366,6 +370,11 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum result.parallel = false } + // Whether or not to symbolicate backtraces in the event stream. + if let symbolicateBacktracesIndex = args.firstIndex(of: "--symbolicate-backtraces"), !isLastArgument(at: symbolicateBacktracesIndex) { + result.symbolicateBacktraces = args[args.index(after: symbolicateBacktracesIndex)] + } + // Verbosity if let verbosityIndex = args.firstIndex(of: "--verbosity"), !isLastArgument(at: verbosityIndex), let verbosity = Int(args[args.index(after: verbosityIndex)]) { @@ -425,6 +434,19 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr // Parallelization (on by default) configuration.isParallelizationEnabled = args.parallel ?? true + // Whether or not to symbolicate backtraces in the event stream. + if let symbolicateBacktraces = args.symbolicateBacktraces { + switch symbolicateBacktraces.lowercased() { + case "mangled", "on", "true": + configuration.backtraceSymbolicationMode = .mangled + case "demangled": + configuration.backtraceSymbolicationMode = .demangled + default: + throw _EntryPointError.invalidArgument("--symbolicate-backtraces", value: symbolicateBacktraces) + + } + } + #if !SWT_NO_FILE_IO // XML output if let xunitOutputPath = args.xunitOutput { diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedBacktrace.swift b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedBacktrace.swift new file mode 100644 index 000000000..8f681c20a --- /dev/null +++ b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedBacktrace.swift @@ -0,0 +1,42 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +extension ABIv0 { + /// A type implementing the JSON encoding of ``Backtrace`` for the ABI entry + /// point and event stream output. + /// + /// This type is not part of the public interface of the testing library. It + /// assists in converting values to JSON; clients that consume this JSON are + /// expected to write their own decoders. + struct EncodedBacktrace: Sendable { + /// The frames in the backtrace. + var symbolicatedAddresses: [Backtrace.SymbolicatedAddress] + + init(encoding backtrace: borrowing Backtrace, in eventContext: borrowing Event.Context) { + if let symbolicationMode = eventContext.configuration?.backtraceSymbolicationMode { + symbolicatedAddresses = backtrace.symbolicate(symbolicationMode) + } else { + symbolicatedAddresses = backtrace.addresses.map { Backtrace.SymbolicatedAddress(address: $0) } + } + } + } +} + +// MARK: - Codable + +extension ABIv0.EncodedBacktrace: Codable { + func encode(to encoder: any Encoder) throws { + try symbolicatedAddresses.encode(to: encoder) + } + + init(from decoder: any Decoder) throws { + self.symbolicatedAddresses = try [Backtrace.SymbolicatedAddress](from: decoder) + } +} diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedEvent.swift b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedEvent.swift index 951244af3..65cd78234 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedEvent.swift +++ b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedEvent.swift @@ -70,7 +70,7 @@ extension ABIv0 { kind = .testCaseStarted case let .issueRecorded(recordedIssue): kind = .issueRecorded - issue = EncodedIssue(encoding: recordedIssue) + issue = EncodedIssue(encoding: recordedIssue, in: eventContext) case .testCaseEnded: if eventContext.test?.isParameterized == false { return nil diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift index 7399fd830..419af3d73 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift +++ b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift @@ -23,12 +23,14 @@ extension ABIv0 { var sourceLocation: SourceLocation? /// The backtrace where this issue occurred, if available. - var _backtrace: [Backtrace.Address]? + var _backtrace: EncodedBacktrace? - init(encoding issue: borrowing Issue) { + init(encoding issue: borrowing Issue, in eventContext: borrowing Event.Context) { isKnown = issue.isKnown sourceLocation = issue.sourceLocation - _backtrace = issue.sourceContext.backtrace.map(\.addresses) + if let backtrace = issue.sourceContext.backtrace { + _backtrace = EncodedBacktrace(encoding: backtrace, in: eventContext) + } } } } diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index 69b851645..2afb51367 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -13,6 +13,7 @@ add_library(Testing ABI/v0/ABIv0.Record.swift ABI/v0/ABIv0.Record+Streaming.swift ABI/v0/ABIv0.swift + ABI/v0/Encoded/ABIv0.EncodedBacktrace.swift ABI/v0/Encoded/ABIv0.EncodedEvent.swift ABI/v0/Encoded/ABIv0.EncodedInstant.swift ABI/v0/Encoded/ABIv0.EncodedIssue.swift @@ -50,6 +51,7 @@ add_library(Testing Running/Runner.swift Running/SkipInfo.swift SourceAttribution/Backtrace.swift + SourceAttribution/Backtrace+Symbolication.swift SourceAttribution/CustomTestStringConvertible.swift SourceAttribution/Expression.swift SourceAttribution/Expression+Macro.swift diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index 80cd65a49..c359d4ee0 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -20,6 +20,16 @@ public struct Configuration: Sendable { /// Whether or not to parallelize the execution of tests and test cases. public var isParallelizationEnabled = true + /// How to symbolicate backtraces captured during a test run. + /// + /// If the value of this property is not `nil`, symbolication will be + /// performed automatically when a backtrace is encoded into an event stream. + /// + /// The value of this property does not affect event handlers implemented in + /// Swift in-process. When handling a backtrace in Swift, use its + /// ``Backtrace/symbolicate(_:)`` function to symbolicate it. + public var backtraceSymbolicationMode: Backtrace.SymbolicationMode? + /// A type describing whether or not, and how, to iterate a test plan /// repeatedly. /// diff --git a/Sources/Testing/SourceAttribution/Backtrace+Symbolication.swift b/Sources/Testing/SourceAttribution/Backtrace+Symbolication.swift new file mode 100644 index 000000000..21b39dfed --- /dev/null +++ b/Sources/Testing/SourceAttribution/Backtrace+Symbolication.swift @@ -0,0 +1,167 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +private import _TestingInternals + +/// A type representing a backtrace or stack trace. +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) +extension Backtrace { + /// An enumeration describing the symbolication mode to use when handling + /// events containing backtraces. + public enum SymbolicationMode: Sendable { + /// The backtrace should be symbolicated, but no demangling should be + /// performed. + case mangled + + /// The backtrace should be symbolicated and Swift symbols should be + /// demangled if possible. + /// + /// "Foreign" symbol names such as those produced by C++ are not demangled. + case demangled + } + + /// A type representing an instance of ``Backtrace/Address`` that has been + /// symbolicated by a call to ``Backtrace/symbolicate(_:)``. + public struct SymbolicatedAddress: Sendable { + /// The (unsymbolicated) address from the backtrace. + public var address: Address + + /// The offset of ``address`` from the start of the corresponding function, + /// if available. + /// + /// If ``address`` could not be resolved to a symbol, the value of this + /// property is `nil`. + public var offset: UInt64? + + /// The name of the symbol at ``address``, if available. + /// + /// If ``address`` could not be resolved to a symbol, the value of this + /// property is `nil`. + public var symbolName: String? + } + + /// Symbolicate the addresses in this backtrace. + /// + /// - Parameters: + /// - mode: How to symbolicate the addresses in the backtrace. + /// + /// - Returns: An array of strings representing the names of symbols in + /// `addresses`. + /// + /// If an address in `addresses` cannot be symbolicated, the corresponding + /// instance of ``SymbolicatedAddress`` in the resulting array has a `nil` + /// value for its ``Backtrace/SymbolicatedAddress/symbolName`` property. + public func symbolicate(_ mode: SymbolicationMode) -> [SymbolicatedAddress] { + var result = addresses.map { SymbolicatedAddress(address: $0) } + +#if SWT_TARGET_OS_APPLE + for (i, address) in addresses.enumerated() { + var info = Dl_info() + if 0 != dladdr(UnsafePointer(bitPattern: UInt(clamping: address)), &info) { + let offset = address - Address(clamping: UInt(bitPattern: info.dli_saddr)) + let symbolName = info.dli_sname.flatMap(String.init(validatingCString:)) + result[i] = SymbolicatedAddress(address: address, offset: offset, symbolName: symbolName) + } + } +#elseif os(Linux) + // Although Linux has dladdr(), it does not have symbol names from ELF + // binaries by default. The standard library's backtracing functionality has + // implemented sufficient ELF/DWARF parsing to be able to symbolicate Linux + // backtraces. TODO: adopt the standard library's Backtrace on Linux + // Note that this means on Linux we don't have demangling capability (since + // we don't have the mangled symbol names in the first place) so this code + // does not check the mode argument. +#elseif os(Windows) + _withDbgHelpLibrary { hProcess in + guard let hProcess else { + return + } + for (i, address) in addresses.enumerated() { + withUnsafeTemporaryAllocation(of: SYMBOL_INFO_PACKAGEW.self, capacity: 1) { symbolInfo in + let symbolInfo = symbolInfo.baseAddress! + symbolInfo.pointee.si.SizeOfStruct = ULONG(MemoryLayout.stride) + symbolInfo.pointee.si.MaxNameLen = ULONG(MAX_SYM_NAME) + var displacement = DWORD64(0) + if SymFromAddrW(hProcess, DWORD64(clamping: address), &displacement, symbolInfo.pointer(to: \.si)!) { + let symbolName = String.decodeCString(symbolInfo.pointer(to: \.si.Name)!, as: UTF16.self)?.result + result[i] = SymbolicatedAddress(address: address, offset: displacement, symbolName: symbolName) + } + } + } + } +#elseif os(WASI) + // WASI does not currently support backtracing let alone symbolication. +#else +#warning("Platform-specific implementation missing: backtrace symbolication unavailable") +#endif + + if mode != .mangled { + result = result.map { symbolicatedAddress in + var symbolicatedAddress = symbolicatedAddress + if let demangledName = symbolicatedAddress.symbolName.flatMap(_demangle) { + symbolicatedAddress.symbolName = demangledName + } + return symbolicatedAddress + } + } + + return result + } +} + +// MARK: - Codable + +extension Backtrace.SymbolicatedAddress: Codable {} + +// MARK: - Swift runtime wrappers + +/// Demangle a symbol name. +/// +/// - Parameters: +/// - mangledSymbolName: The symbol name to demangle. +/// +/// - Returns: The demangled form of `mangledSymbolName` according to the +/// Swift standard library or the platform's C++ standard library, or `nil` +/// if the symbol name could not be demangled. +private func _demangle(_ mangledSymbolName: String) -> String? { + mangledSymbolName.withCString { mangledSymbolName in + guard let demangledSymbolName = swift_demangle(mangledSymbolName, strlen(mangledSymbolName), nil, nil, 0) else { + return nil + } + defer { + free(demangledSymbolName) + } + return String(validatingCString: demangledSymbolName) + } +} + +#if os(Windows) +/// Configure the environment to allow calling into the Debug Help library. +/// +/// - Parameters: +/// - body: A function to invoke. A process handle valid for use with Debug +/// Help functions is passed in, or `nullptr` if the Debug Help library +/// could not be initialized. +/// - context: An arbitrary pointer to pass to `body`. +/// +/// On Windows, the Debug Help library (DbgHelp.lib) is not thread-safe. All +/// calls into it from the Swift runtime and stdlib should route through this +/// function. +private func _withDbgHelpLibrary(_ body: (HANDLE?) -> Void) { + withoutActuallyEscaping(body) { body in + withUnsafePointer(to: body) { context in + _swift_win32_withDbgHelpLibrary({ hProcess, context in + let body = context!.load(as: ((HANDLE?) -> Void).self) + body(hProcess) + }, .init(mutating: context)) + } + } +} +#endif diff --git a/Sources/_TestingInternals/include/Demangle.h b/Sources/_TestingInternals/include/Demangle.h new file mode 100644 index 000000000..aa5f9fb1d --- /dev/null +++ b/Sources/_TestingInternals/include/Demangle.h @@ -0,0 +1,60 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if !defined(SWT_DEMANGLE_H) +#define SWT_DEMANGLE_H + +#include "Defines.h" +#include "Includes.h" + +SWT_ASSUME_NONNULL_BEGIN + +/// Demangle a Swift symbol name. +/// +/// - Parameters: +/// - mangledName: A pointer to the mangled symbol name to demangle. +/// - mangledNameLength: The length of `mangledName` in bytes, not including +/// any trailing null byte. +/// - outputBuffer: Unused by the testing library. Pass `nullptr`. +/// - outputBufferSize: Unused by the testing library. Pass `nullptr`. +/// - flags: Unused by the testing library. Pass `0`. +/// +/// - Returns: The demangled form of `mangledName`, or `nullptr` if demangling +/// failed. The caller is responsible for freeing this string with `free()` +/// when done. +SWT_IMPORT_FROM_STDLIB char *_Nullable swift_demangle( + const char *mangledName, + size_t mangledNameLength, + char *_Nullable outputBuffer, + size_t *_Nullable outputBufferSize, + uint32_t flags +); + +#if defined(_WIN32) +/// Configure the environment to allow calling into the Debug Help library. +/// +/// - Parameters: +/// - body: A function to invoke. A process handle valid for use with Debug +/// Help functions is passed in, or `nullptr` if the Debug Help library +/// could not be initialized. +/// - context: An arbitrary pointer to pass to `body`. +/// +/// On Windows, the Debug Help library (DbgHelp.lib) is not thread-safe. All +/// calls into it from the Swift runtime and stdlib should route through this +/// function. +SWT_IMPORT_FROM_STDLIB void _swift_win32_withDbgHelpLibrary( + void (* body)(HANDLE _Nullable hProcess, void *_Null_unspecified context), + void *_Null_unspecified context +); +#endif + +SWT_ASSUME_NONNULL_END + +#endif diff --git a/Sources/_TestingInternals/include/Discovery.h b/Sources/_TestingInternals/include/Discovery.h index 56dd4db0e..f15ad1b69 100644 --- a/Sources/_TestingInternals/include/Discovery.h +++ b/Sources/_TestingInternals/include/Discovery.h @@ -12,8 +12,7 @@ #define SWT_DISCOVERY_H #include "Defines.h" - -#include +#include "Includes.h" SWT_ASSUME_NONNULL_BEGIN diff --git a/Sources/_TestingInternals/include/Includes.h b/Sources/_TestingInternals/include/Includes.h index fe0c67adf..420a8c745 100644 --- a/Sources/_TestingInternals/include/Includes.h +++ b/Sources/_TestingInternals/include/Includes.h @@ -27,6 +27,7 @@ /// - Note: Avoid including headers that aren't actually used. #include +#include /// Guard against including `signal.h` on WASI. The `signal.h` header file /// itself is available in wasi-libc, but it's just a stub that doesn't actually /// do anything. And also including it requires a special macro definition @@ -34,6 +35,8 @@ #if __has_include() && !defined(__wasi__) #include #endif +#include +#include #include #include #include diff --git a/Tests/TestingTests/BacktraceTests.swift b/Tests/TestingTests/BacktraceTests.swift index ac0fc98f5..88e829a9f 100644 --- a/Tests/TestingTests/BacktraceTests.swift +++ b/Tests/TestingTests/BacktraceTests.swift @@ -149,4 +149,14 @@ struct BacktraceTests { #expect(original == copy) } #endif + + @Test("Symbolication", arguments: [Backtrace.SymbolicationMode.mangled, .demangled]) + func symbolication(mode: Backtrace.SymbolicationMode) { + let backtrace = Backtrace.current() + let symbolNames = backtrace.symbolicate(mode) + #expect(backtrace.addresses.count == symbolNames.count) + if testsWithSignificantIOAreEnabled { + print(symbolNames.map(String.init(describingForTest:)).joined(separator: "\n")) + } + } } diff --git a/Tests/TestingTests/SwiftPMTests.swift b/Tests/TestingTests/SwiftPMTests.swift index 8e13967f5..51764fa1a 100644 --- a/Tests/TestingTests/SwiftPMTests.swift +++ b/Tests/TestingTests/SwiftPMTests.swift @@ -43,6 +43,22 @@ struct SwiftPMTests { #expect(!configuration.isParallelizationEnabled) } + @Test("--symbolicate-backtraces argument", + arguments: [ + (String?.none, Backtrace.SymbolicationMode?.none), + ("mangled", .mangled), ("on", .mangled), ("true", .mangled), + ("demangled", .demangled), + ] + ) + func symbolicateBacktraces(argumentValue: String?, expectedMode: Backtrace.SymbolicationMode?) throws { + let configuration = if let argumentValue { + try configurationForEntryPoint(withArguments: ["PATH", "--symbolicate-backtraces", argumentValue]) + } else { + try configurationForEntryPoint(withArguments: ["PATH"]) + } + #expect(configuration.backtraceSymbolicationMode == expectedMode) + } + @Test("No --filter or --skip argument") func defaultFiltering() async throws { let configuration = try configurationForEntryPoint(withArguments: ["PATH"])