Optionally symbolicate backtraces.

This PR adds the ability to symbolicate backtraces on Darwin and Windows. A few
different modes are provided: mangled, demangled, and "precise demangled" (which
includes symbol addresses and instruction pointer offsets.) Tools such as the
Swift VS Code plugin will be able to adopt this new feature along with VS Code's
new call stacks API (microsoft/vscode#222126).

Note that on Linux, it is not currently possible to symbolicate backtraces
meaningfully. The standard library's `Backtrace` type has the ability to do this
for us, but we'll need some tweaks to its interface before we can adopt it.

Note also that Apple's internal Core Symbolication framework is not used; we may
be able to add support for it in a future PR (or Apple may opt to use it in
their internal fork of Swift Testing.)

There is no way to emit backtraces to the command line right now. I considered
having `--very-verbose` imply backtraces, but it's something I'm going to
reserve for a future PR after discussion with the community.

Author: grynspan

Sources/Testing/ABI/EntryPoints/EntryPoint.swift

@@ -191,6 +191,9 @@ public struct __CommandLineArguments_v0: Sendable {
   /// The value of the `--parallel` or `--no-parallel` argument.
   public var parallel: Bool?
 
+  /// The value of the `--symbolicate-backtraces` argument.
+  public var symbolicateBacktraces: String?
+
   /// The value of the `--verbose` argument.
   public var verbose: Bool?
 
@@ -280,6 +283,7 @@ extension __CommandLineArguments_v0: Codable {
   enum CodingKeys: String, CodingKey {
     case listTests
     case parallel
+    case symbolicateBacktraces
     case verbose
     case veryVerbose
     case quiet
@@ -366,6 +370,11 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum
     result.parallel = false
   }
 
+  // Whether or not to symbolicate backtraces in the event stream.
+  if let symbolicateBacktracesIndex = args.firstIndex(of: "--symbolicate-backtraces"), !isLastArgument(at: symbolicateBacktracesIndex) {
+    result.symbolicateBacktraces = args[args.index(after: symbolicateBacktracesIndex)]
+  }
+
   // Verbosity
   if let verbosityIndex = args.firstIndex(of: "--verbosity"), !isLastArgument(at: verbosityIndex),
      let verbosity = Int(args[args.index(after: verbosityIndex)]) {
@@ -425,6 +434,21 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr
   // Parallelization (on by default)
   configuration.isParallelizationEnabled = args.parallel ?? true
 
+  // Whether or not to symbolicate backtraces in the event stream.
+  if let symbolicateBacktraces = args.symbolicateBacktraces {
+    switch symbolicateBacktraces.lowercased() {
+    case "mangled", "on", "true":
+      configuration.backtraceSymbolicationMode = .mangled
+    case "demangled":
+      configuration.backtraceSymbolicationMode = .demangled
+    case "precise-demangled":
+      configuration.backtraceSymbolicationMode = .preciseDemangled
+    default:
+      throw _EntryPointError.invalidArgument("--symbolicate-backtraces", value: symbolicateBacktraces)
+
+    }
+  }
+
 #if !SWT_NO_FILE_IO
   // XML output
   if let xunitOutputPath = args.xunitOutput {

Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedBacktrace.swift

@@ -0,0 +1,53 @@
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+//
+
+extension ABIv0 {
+  /// A type implementing the JSON encoding of ``Backtrace`` for the ABI entry
+  /// point and event stream output.
+  ///
+  /// This type is not part of the public interface of the testing library. It
+  /// assists in converting values to JSON; clients that consume this JSON are
+  /// expected to write their own decoders.
+  struct EncodedBacktrace: Sendable {
+    /// A type describing a frame in the backtrace.
+    struct Frame: Sendable {
+      /// The address of the frame.
+      var address: Backtrace.Address
+
+      /// The name of the frame, possibly demangled, if available.
+      var symbolName: String?
+    }
+
+    /// The frames in the backtrace.
+    var frames: [Frame]
+
+    init(encoding backtrace: borrowing Backtrace, in eventContext: borrowing Event.Context) {
+      if let symbolicationMode = eventContext.configuration?.backtraceSymbolicationMode {
+        frames = zip(backtrace.addresses, backtrace.symbolicate(symbolicationMode)).map(Frame.init)
+      } else {
+        frames = backtrace.addresses.map { Frame(address: $0) }
+      }
+    }
+  }
+}
+
+// MARK: - Codable
+
+extension ABIv0.EncodedBacktrace: Codable {
+  func encode(to encoder: any Encoder) throws {
+    try frames.encode(to: encoder)
+  }
+
+  init(from decoder: any Decoder) throws {
+    self.frames = try [Frame](from: decoder)
+  }
+}
+
+extension ABIv0.EncodedBacktrace.Frame: Codable {}

Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedEvent.swift

@@ -70,7 +70,7 @@ extension ABIv0 {
         kind = .testCaseStarted
       case let .issueRecorded(recordedIssue):
         kind = .issueRecorded
-        issue = EncodedIssue(encoding: recordedIssue)
+        issue = EncodedIssue(encoding: recordedIssue, in: eventContext)
       case .testCaseEnded:
         if eventContext.test?.isParameterized == false {
           return nil

Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift

@@ -23,12 +23,14 @@ extension ABIv0 {
     var sourceLocation: SourceLocation?
 
     /// The backtrace where this issue occurred, if available.
-    var _backtrace: [Backtrace.Address]?
+    var _backtrace: EncodedBacktrace?
 
-    init(encoding issue: borrowing Issue) {
+    init(encoding issue: borrowing Issue, in eventContext: borrowing Event.Context) {
       isKnown = issue.isKnown
       sourceLocation = issue.sourceLocation
-      _backtrace = issue.sourceContext.backtrace.map(\.addresses)
+      if let backtrace = issue.sourceContext.backtrace {
+        _backtrace = EncodedBacktrace(encoding: backtrace, in: eventContext)
+      }
     }
   }
 }

Sources/Testing/CMakeLists.txt

@@ -13,6 +13,7 @@ add_library(Testing
   ABI/v0/ABIv0.Record.swift
   ABI/v0/ABIv0.Record+Streaming.swift
   ABI/v0/ABIv0.swift
+  ABI/v0/Encoded/ABIv0.EncodedBacktrace.swift
   ABI/v0/Encoded/ABIv0.EncodedEvent.swift
   ABI/v0/Encoded/ABIv0.EncodedInstant.swift
   ABI/v0/Encoded/ABIv0.EncodedIssue.swift
@@ -50,6 +51,7 @@ add_library(Testing
   Running/Runner.swift
   Running/SkipInfo.swift
   SourceAttribution/Backtrace.swift
+  SourceAttribution/Backtrace+Symbolication.swift
   SourceAttribution/CustomTestStringConvertible.swift
   SourceAttribution/Expression.swift
   SourceAttribution/Expression+Macro.swift

Sources/Testing/Events/Event.swift

@@ -203,7 +203,7 @@ public struct Event: Sendable {
     // Create both the event and its associated context here at same point, to
     // ensure their task local-derived values are the same.
     let event = Event(kind, testID: test?.id, testCaseID: testCase?.id, instant: instant)
-    let context = Event.Context(test: test, testCase: testCase)
+    let context = Event.Context(test: test, testCase: testCase, configuration: nil)
     event._post(in: context, configuration: configuration)
   }
 }
@@ -239,16 +239,24 @@ extension Event {
     /// functions), the value of this property is `nil`.
     public var testCase: Test.Case?
 
+    /// The configuration handling the corresponding event, if any.
+    ///
+    /// The value of this property is a copy of the configuration that owns the
+    /// currently-running event handler; to avoid reference cycles, the
+    /// ``Configuration/eventHandler`` property of this instance is cleared.
+    public var configuration: Configuration?
+
     /// Initialize a new instance of this type.
     ///
     /// - Parameters:
     ///   - test: The test for which this instance's associated event occurred,
     ///     if any.
     ///   - testCase: The test case for which this instance's associated event
     ///     occurred, if any.
-    init(test: Test? = .current, testCase: Test.Case? = .current) {
+    init(test: Test?, testCase: Test.Case?, configuration: Configuration?) {
       self.test = test
       self.testCase = testCase
+      self.configuration = configuration
     }
   }
 

Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift

@@ -230,7 +230,7 @@ extension Event.HumanReadableOutputRecorder {
       "«unknown»"
     }
     let instant = event.instant
-    let iterationCount = Configuration.current?.repetitionPolicy.maximumIterationCount
+    let iterationCount = eventContext.configuration?.repetitionPolicy.maximumIterationCount
 
     // First, make any updates to the context/state associated with this
     // recorder.

Sources/Testing/Running/Configuration+EventHandling.swift

@@ -20,6 +20,9 @@ extension Configuration {
   /// `eventHandler` but this method may also be used as a customization point
   /// to change how the event is passed to the event handler.
   func handleEvent(_ event: borrowing Event, in context: borrowing Event.Context) {
-    eventHandler(event, context)
+    var contextCopy = copy context
+    contextCopy.configuration = self
+    contextCopy.configuration?.eventHandler = { _, _ in }
+    eventHandler(event, contextCopy)
   }
 }

Sources/Testing/Running/Configuration.swift

@@ -20,6 +20,16 @@ public struct Configuration: Sendable {
   /// Whether or not to parallelize the execution of tests and test cases.
   public var isParallelizationEnabled = true
 
+  /// How to symbolicate backtraces captured during a test run.
+  ///
+  /// If the value of this property is not `nil`, symbolication will be
+  /// performed automatically when a backtrace is encoded into an event stream.
+  ///
+  /// The value of this property does not affect event handlers implemented in
+  /// Swift in-process. When handling a backtrace in Swift, use its
+  /// ``Backtrace/symbolicate(_:)`` function to symbolicate it.
+  public var backtraceSymbolicationMode: Backtrace.SymbolicationMode?
+
   /// A type describing whether or not, and how, to iterate a test plan
   /// repeatedly.
   ///

Sources/Testing/Running/Runner.RuntimeState.swift

@@ -222,3 +222,12 @@ extension Test.Case {
     return try await Runner.RuntimeState.$current.withValue(runtimeState, operation: body)
   }
 }
+
+// MARK: - Event.Context convenience initializer
+
+extension Event.Context {
+  init() {
+    let state = Runner.RuntimeState.current
+    self.init(test: state?.test, testCase: state?.testCase, configuration: state?.configuration)
+  }
+}