Add option to generate an output mirror file that contains the exact data sent from SourceKit-LSP to the client

This could help us debug low-level issues of SourceKit-LSP to client communication such as #1890.

Author: ahoppen

Documentation/Configuration File.md

@@ -44,6 +44,7 @@ The structure of the file is currently not guaranteed to be stable. Options may
   - `level: "debug"|"info"|"default"|"error"|"fault"`: The level from which one onwards log messages should be written.
   - `privacyLevel: "public"|"private"|"sensitive"`: Whether potentially sensitive information should be redacted. Default is `public`, which redacts potentially sensitive information.
   - `inputMirrorDirectory: string`: Write all input received by SourceKit-LSP on stdin to a file in this directory. Useful to record and replay an entire SourceKit-LSP session.
+  - `outputMirrorDirectory: string`: Write all data sent from SourceKit-LSP to the client to a file in this directory. Useful to record the raw communication between SourceKit-LSP and the client on a low level.
 - `defaultWorkspaceType: "buildServer"|"compilationDatabase"|"swiftPM"`: Default workspace type. Overrides workspace type selection logic.
 - `generatedFilesPath: string`: Directory in which generated interfaces and macro expansions should be stored.
 - `backgroundIndexing: boolean`: Whether background indexing is enabled.

Sources/LanguageServerProtocolJSONRPC/JSONRPCConnection.swift

@@ -61,6 +61,12 @@ public final class JSONRPCConnection: Connection {
   private let sendIO: DispatchIO
   private let messageRegistry: MessageRegistry
 
+  /// If non-nil, all input received by this `JSONRPCConnection` will be written to the file handle
+  let inputMirrorFile: FileHandle?
+
+  /// If non-nil, all output created by this `JSONRPCConnection` will be written to the file handle
+  let outputMirrorFile: FileHandle?
+
   enum State {
     case created, running, closed
   }
@@ -120,9 +126,13 @@ public final class JSONRPCConnection: Connection {
     name: String,
     protocol messageRegistry: MessageRegistry,
     inFD: FileHandle,
-    outFD: FileHandle
+    outFD: FileHandle,
+    inputMirrorFile: FileHandle? = nil,
+    outputMirrorFile: FileHandle? = nil
   ) {
     self.name = name
+    self.inputMirrorFile = inputMirrorFile
+    self.outputMirrorFile = outputMirrorFile
     self.receiveHandler = nil
     #if os(Linux) || os(Android)
     // We receive a `SIGPIPE` if we write to a pipe that points to a crashed process. This in particular happens if the
@@ -268,13 +278,10 @@ public final class JSONRPCConnection: Connection {
   /// Start processing `inFD` and send messages to `receiveHandler`.
   ///
   /// - parameter receiveHandler: The message handler to invoke for requests received on the `inFD`.
-  /// - parameter mirrorFile: If non-nil, all input received by this `JSONRPCConnection` will be written to the file
-  ///   handle
   ///
   /// - Important: `start` must be called before sending any data over the `JSONRPCConnection`.
   public func start(
     receiveHandler: MessageHandler,
-    mirrorFile: FileHandle? = nil,
     closeHandler: @escaping @Sendable () async -> Void = {}
   ) {
     queue.sync {
@@ -303,8 +310,8 @@ public final class JSONRPCConnection: Connection {
           return
         }
 
-        orLog("Writing data mirror file") {
-          try mirrorFile?.write(contentsOf: data)
+        orLog("Writing input mirror file") {
+          try self.inputMirrorFile?.write(contentsOf: data)
         }
 
         // Parse and handle any messages in `buffer + data`, leaving any remaining unparsed bytes in `buffer`.
@@ -538,6 +545,9 @@ public final class JSONRPCConnection: Connection {
     dispatchPrecondition(condition: .onQueue(queue))
     guard readyToSend() else { return }
 
+    orLog("Writing output mirror file") {
+      try outputMirrorFile?.write(contentsOf: dispatchData)
+    }
     sendIO.write(offset: 0, data: dispatchData, queue: sendQueue) { [weak self] done, _, errorCode in
       if errorCode != 0 {
         logger.fault("IO error sending message \(errorCode)")

Sources/SKOptions/SourceKitLSPOptions.swift

@@ -223,21 +223,29 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
     /// Useful to record and replay an entire SourceKit-LSP session.
     public var inputMirrorDirectory: String?
 
+    /// Write all data sent from SourceKit-LSP to the client to a file in this directory.
+    ///
+    /// Useful to record the raw communication between SourceKit-LSP and the client on a low level.
+    public var outputMirrorDirectory: String?
+
     public init(
       level: String? = nil,
       privacyLevel: String? = nil,
-      inputMirrorDirectory: String? = nil
+      inputMirrorDirectory: String? = nil,
+      outputMirrorDirectory: String? = nil
     ) {
       self.level = level
       self.privacyLevel = privacyLevel
       self.inputMirrorDirectory = inputMirrorDirectory
+      self.outputMirrorDirectory = outputMirrorDirectory
     }
 
     static func merging(base: LoggingOptions, override: LoggingOptions?) -> LoggingOptions {
       return LoggingOptions(
         level: override?.level ?? base.level,
         privacyLevel: override?.privacyLevel ?? base.privacyLevel,
-        inputMirrorDirectory: override?.inputMirrorDirectory ?? base.inputMirrorDirectory
+        inputMirrorDirectory: override?.inputMirrorDirectory ?? base.inputMirrorDirectory,
+        outputMirrorDirectory: override?.outputMirrorDirectory ?? base.outputMirrorDirectory
       )
     }
   }

Sources/sourcekit-lsp/SourceKitLSP.swift

@@ -226,6 +226,22 @@ struct SourceKitLSP: AsyncParsableCommand {
     return options
   }
 
+  /// Create a new file that can be used to use as an input or output mirror file and return a file handle that can be
+  /// used to write to that file.
+  private func createMirrorFile(in directory: URL) throws -> FileHandle {
+    let dateFormatter = ISO8601DateFormatter()
+    dateFormatter.timeZone = NSTimeZone.local
+    let date = dateFormatter.string(from: Date()).replacingOccurrences(of: ":", with: "-")
+
+    let inputMirrorURL = directory.appendingPathComponent("\(date).log")
+
+    logger.log("Mirroring input to \(inputMirrorURL)")
+    try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
+    FileManager.default.createFile(atPath: try inputMirrorURL.filePath, contents: nil)
+
+    return try FileHandle(forWritingTo: inputMirrorURL)
+  }
+
   func run() async throws {
     // Dup stdout and redirect the fd to stderr so that a careless print()
     // will not break our connection stream.
@@ -262,35 +278,35 @@ struct SourceKitLSP: AsyncParsableCommand {
     )
     cleanOldLogFiles(logFileDirectory: logFileDirectoryURL, maxAge: 60 * 60 /* 1h */)
 
+    let inputMirror = orLog("Setting up input mirror") {
+      if let inputMirrorDirectory = globalConfigurationOptions.loggingOrDefault.inputMirrorDirectory {
+        return try createMirrorFile(in: URL(fileURLWithPath: inputMirrorDirectory))
+      } else {
+        return nil
+      }
+    }
+
+    let outputMirror = orLog("Setting up output mirror") {
+      if let outputMirrorDirectory = globalConfigurationOptions.loggingOrDefault.outputMirrorDirectory {
+        return try createMirrorFile(in: URL(fileURLWithPath: outputMirrorDirectory))
+      } else {
+        return nil
+      }
+    }
+
     let clientConnection = JSONRPCConnection(
       name: "client",
       protocol: MessageRegistry.lspProtocol,
       inFD: FileHandle.standardInput,
-      outFD: realStdoutHandle
+      outFD: realStdoutHandle,
+      inputMirrorFile: inputMirror,
+      outputMirrorFile: outputMirror
     )
 
     // For reasons that are completely oblivious to me, `DispatchIO.write`, which is used to write LSP responses to
     // stdout fails with error code 5 on Windows unless we call `AbsolutePath(validating:)` on some URL first.
     _ = try AbsolutePath(validating: Bundle.main.bundlePath)
 
-    var inputMirror: FileHandle? = nil
-    if let inputMirrorDirectory = globalConfigurationOptions.loggingOrDefault.inputMirrorDirectory {
-      orLog("Setting up input mirror") {
-        let dateFormatter = ISO8601DateFormatter()
-        dateFormatter.timeZone = NSTimeZone.local
-        let date = dateFormatter.string(from: Date()).replacingOccurrences(of: ":", with: "-")
-
-        let inputMirrorDirectory = URL(fileURLWithPath: inputMirrorDirectory)
-        let inputMirrorURL = inputMirrorDirectory.appendingPathComponent("\(date).log")
-
-        logger.log("Mirroring input to \(inputMirrorURL)")
-        try FileManager.default.createDirectory(at: inputMirrorDirectory, withIntermediateDirectories: true)
-        FileManager.default.createFile(atPath: try inputMirrorURL.filePath, contents: nil)
-
-        inputMirror = try FileHandle(forWritingTo: inputMirrorURL)
-      }
-    }
-
     let server = SourceKitLSPServer(
       client: clientConnection,
       toolchainRegistry: ToolchainRegistry(installPath: Bundle.main.bundleURL),
@@ -302,7 +318,6 @@ struct SourceKitLSP: AsyncParsableCommand {
     )
     clientConnection.start(
       receiveHandler: server,
-      mirrorFile: inputMirror,
       closeHandler: {
         await server.prepareForExit()
         // Use _Exit to avoid running static destructors due to https://github.com/swiftlang/swift/issues/55112.

config.schema.json

@@ -173,6 +173,11 @@
           "markdownDescription" : "The level from which one onwards log messages should be written.",
           "type" : "string"
         },
+        "outputMirrorDirectory" : {
+          "description" : "Write all data sent from SourceKit-LSP to the client to a file in this directory. Useful to record the raw communication between SourceKit-LSP and the client on a low level.",
+          "markdownDescription" : "Write all data sent from SourceKit-LSP to the client to a file in this directory. Useful to record the raw communication between SourceKit-LSP and the client on a low level.",
+          "type" : "string"
+        },
         "privacyLevel" : {
           "description" : "Whether potentially sensitive information should be redacted. Default is `public`, which redacts potentially sensitive information.",
           "enum" : [