Merge pull request #2272 from divalue/syntax_tree_with_lookahead_struct

Add IncrementalParseResult

Author: ahoppen

Release Notes/511.md

@@ -10,7 +10,12 @@
   - Description: `is`, `as`, and `cast` methods for types not contained in the choice node are marked as deprecated. The deprecated methods will emit a warning, indicating that the cast will always fail.
   - Issue: https://github.com/apple/swift-syntax/issues/2092
   - Pull Request: https://github.com/apple/swift-syntax/pull/2184
-  
+
+- `IncrementalParseTransition`:
+  - Description: The initializer `IncrementalParseTransition.init(previousTree:edits:lookaheadRanges:reusedNodeCallback:)` is marked as deprecated. Use `IncrementalParseTransition.init(previousIncrementalParseResult:edits:reusedNodeCallback:)` instead.
+  - Issue: https://github.com/apple/swift-syntax/issues/2267
+  - Pull request: https://github.com/apple/swift-syntax/pull/2272
+
 ## API-Incompatible Changes
 
 - Effect specifiers:
@@ -23,6 +28,10 @@
   - Description: `IntegerLiteralExprSyntax.Radix` no longer conforms to `CaseIterable` since there is no good use case to iterate over all radix kinds. 
   - Pull request: https://github.com/apple/swift-syntax/pull/2292
 
+- `Parser.parseIncrementally(source:parseTransition:)` and `Parser.parseIncrementally(source:maximumNestingLevel:parseTransition:)`:
+  - Description: The default versions of `Parser.parseIncrementally` return a `IncrementalParseResult` instead of a tuple. Access to the struct should be compatible with the tuple in almost all cases unless the tuple is stored into a variable and then destructed or passed to a function that expects a tuple.
+  - Issue: https://github.com/apple/swift-syntax/issues/2267
+  - Pull request: https://github.com/apple/swift-syntax/pull/2272
 
 ## Template
 

Sources/SwiftParser/IncrementalParseTransition.swift

@@ -48,25 +48,39 @@ public typealias ReusedNodeCallback = (_ node: Syntax) -> ()
 /// Keeps track of a previously parsed syntax tree and the source edits that
 /// occurred since it was created.
 public final class IncrementalParseTransition {
-  fileprivate let previousTree: SourceFileSyntax
+  fileprivate let previousIncrementalParseResult: IncrementalParseResult
   fileprivate let edits: ConcurrentEdits
-  fileprivate let lookaheadRanges: LookaheadRanges
   fileprivate let reusedNodeCallback: ReusedNodeCallback?
 
   /// - Parameters:
   ///   - previousTree: The previous tree to do lookups on.
   ///   - edits: The edits that have occurred since the last parse that resulted
   ///            in the new source that is about to be parsed.
   ///   - reusedNodeCallback: Optional closure to accept information about the re-used node. For each node that gets re-used `reusedNodeCallback` is called.
+  @available(*, deprecated, message: "Use initializer taking `IncrementalParseResult` instead")
   public init(
     previousTree: SourceFileSyntax,
     edits: ConcurrentEdits,
     lookaheadRanges: LookaheadRanges,
     reusedNodeCallback: ReusedNodeCallback? = nil
   ) {
-    self.previousTree = previousTree
+    self.previousIncrementalParseResult = IncrementalParseResult(tree: previousTree, lookaheadRanges: lookaheadRanges)
+    self.edits = edits
+    self.reusedNodeCallback = reusedNodeCallback
+  }
+
+  /// - Parameters:
+  ///   - previousIncrementalParseResult: The previous incremental parse result to do lookups on.
+  ///   - edits: The edits that have occurred since the last parse that resulted
+  ///            in the new source that is about to be parsed.
+  ///   - reusedNodeCallback: Optional closure to accept information about the re-used node. For each node that gets re-used `reusedNodeCallback` is called.
+  public init(
+    previousIncrementalParseResult: IncrementalParseResult,
+    edits: ConcurrentEdits,
+    reusedNodeCallback: ReusedNodeCallback? = nil
+  ) {
+    self.previousIncrementalParseResult = previousIncrementalParseResult
     self.edits = edits
-    self.lookaheadRanges = lookaheadRanges
     self.reusedNodeCallback = reusedNodeCallback
   }
 }
@@ -81,7 +95,7 @@ struct IncrementalParseLookup {
   /// given ``IncrementalParseTransition``.
   public init(transition: IncrementalParseTransition) {
     self.transition = transition
-    self.cursor = .init(root: Syntax(transition.previousTree))
+    self.cursor = .init(root: Syntax(transition.previousIncrementalParseResult.tree))
   }
 
   fileprivate var edits: ConcurrentEdits {
@@ -148,7 +162,7 @@ struct IncrementalParseLookup {
       return true
     }
 
-    guard let nodeAffectRangeLength = transition.lookaheadRanges.lookaheadRanges[node.raw.id] else {
+    guard let nodeAffectRangeLength = transition.previousIncrementalParseResult.lookaheadRanges.lookaheadRanges[node.raw.id] else {
       return false
     }
 

Sources/SwiftParser/ParseSourceFile.swift

@@ -62,25 +62,86 @@ extension Parser {
   ///            how far the parser looked ahead while parsing a node, which is
   ///            necessary to construct an ``IncrementalParseTransition`` for a
   ///            subsequent incremental parse
+  @available(*, deprecated, message: "Use parseIncrementally with `IncrementalParseResult` return instead")
+  @_disfavoredOverload
   public static func parseIncrementally(
     source: String,
     parseTransition: IncrementalParseTransition?
   ) -> (tree: SourceFileSyntax, lookaheadRanges: LookaheadRanges) {
-    var parser = Parser(source, parseTransition: parseTransition)
-    return (SourceFileSyntax.parse(from: &parser), parser.lookaheadRanges)
+    let parseResult = parseIncrementally(source: source, parseTransition: parseTransition)
+    return (parseResult.tree, parseResult.lookaheadRanges)
   }
 
   /// Parse the source code in the given buffer as Swift source file with support
   /// for incremental parsing.
   ///
-  /// See doc comments in ``Parser/parseIncrementally(source:parseTransition:)``
+  /// See doc comments in ``Parser/parseIncrementally(source:parseTransition:)-2gtt2``
+  @available(*, deprecated, message: "Use parseIncrementally with `IncrementalParseResult` return instead")
+  @_disfavoredOverload
   public static func parseIncrementally(
     source: UnsafeBufferPointer<UInt8>,
     maximumNestingLevel: Int? = nil,
     parseTransition: IncrementalParseTransition?
   ) -> (tree: SourceFileSyntax, lookaheadRanges: LookaheadRanges) {
+    let parseResult = parseIncrementally(source: source, maximumNestingLevel: maximumNestingLevel, parseTransition: parseTransition)
+    return (parseResult.tree, parseResult.lookaheadRanges)
+  }
+
+  /// Parse the source code in the given string as Swift source file with support
+  /// for incremental parsing.
+  ///
+  /// When parsing a source file for the first time, invoke `parseIncrementally`
+  /// with `parseTransition: nil`. This returns the ``IncrementalParseResult``
+  /// If an edit is made to the source file, an ``IncrementalParseTransition``
+  /// can be constructed from  the ``IncrementalParseResult``.
+  /// When invoking `parseIncrementally` again with
+  /// the post-edit source and that parse transition, the parser will re-use
+  /// nodes that haven’t changed.
+  ///
+  /// - Parameters:
+  ///   - source: The source code to parse
+  ///   - parseTransition: If a similar source file has already been parsed, the
+  ///     ``IncrementalParseTransition`` that contains the previous tree as well
+  ///     as the edits that were performed to it.
+  /// - Returns: The ``IncrementalParseResult``, which is
+  ///            necessary to construct an ``IncrementalParseTransition`` for a
+  ///            subsequent incremental parse
+  public static func parseIncrementally(
+    source: String,
+    parseTransition: IncrementalParseTransition?
+  ) -> IncrementalParseResult {
+    var parser = Parser(source, parseTransition: parseTransition)
+    return IncrementalParseResult(tree: SourceFileSyntax.parse(from: &parser), lookaheadRanges: parser.lookaheadRanges)
+  }
+
+  /// Parse the source code in the given buffer as Swift source file with support
+  /// for incremental parsing.
+  ///
+  /// See doc comments in ``Parser/parseIncrementally(source:parseTransition:)-4kn2k``
+  public static func parseIncrementally(
+    source: UnsafeBufferPointer<UInt8>,
+    maximumNestingLevel: Int? = nil,
+    parseTransition: IncrementalParseTransition?
+  ) -> IncrementalParseResult {
     var parser = Parser(source, maximumNestingLevel: maximumNestingLevel, parseTransition: parseTransition)
-    return (SourceFileSyntax.parse(from: &parser), parser.lookaheadRanges)
+    return IncrementalParseResult(tree: SourceFileSyntax.parse(from: &parser), lookaheadRanges: parser.lookaheadRanges)
+  }
+}
+
+/// The result of incrementally parsing a file.
+///
+/// This contains the parsed syntax tree and additional information on how far the parser looked ahead to parse each node.
+/// This information is required to perform an incremental parse of the tree after applying edits to it.
+public struct IncrementalParseResult {
+  /// The syntax tree from parsing source
+  public let tree: SourceFileSyntax
+  /// The lookahead ranges for syntax nodes describe
+  /// how far the parser looked ahead while parsing a node.
+  public let lookaheadRanges: LookaheadRanges
+
+  public init(tree: SourceFileSyntax, lookaheadRanges: LookaheadRanges) {
+    self.tree = tree
+    self.lookaheadRanges = lookaheadRanges
   }
 }
 

Sources/_SwiftSyntaxTestSupport/IncrementalParseTestUtils.swift

@@ -44,18 +44,17 @@ public func assertIncrementalParse(
   let originalString = String(originalSource)
   let editedString = String(editedSource)
 
-  let (originalTree, lookaheadRanges) = Parser.parseIncrementally(source: originalString, parseTransition: nil)
+  let parseResult = Parser.parseIncrementally(source: originalString, parseTransition: nil)
 
   var reusedNodes: [Syntax] = []
   let transition = IncrementalParseTransition(
-    previousTree: originalTree,
+    previousIncrementalParseResult: parseResult,
     edits: concurrentEdits,
-    lookaheadRanges: lookaheadRanges,
     reusedNodeCallback: { reusedNodes.append($0) }
   )
 
   let newTree = Parser.parse(source: editedString)
-  let (incrementallyParsedNewTree, _) = Parser.parseIncrementally(source: editedString, parseTransition: transition)
+  let incrementallyParsedNewTree = Parser.parseIncrementally(source: editedString, parseTransition: transition).tree
 
   // Round-trip
   assertStringsEqualWithDiff(

SwiftParserCLI/Sources/swift-parser-cli/Commands/PerformanceTest.swift

@@ -47,15 +47,14 @@ struct PerformanceTest: ParsableCommand {
       /// The initial parse for incremental parsing
       for file in files {
         file.withUnsafeBytes { buf in
-          let (tree, lookaheadRanges) = Parser.parseIncrementally(
+          let parseResult = Parser.parseIncrementally(
             source: buf.bindMemory(to: UInt8.self),
             parseTransition: nil
           )
 
           fileTransition[file] = IncrementalParseTransition(
-            previousTree: tree,
-            edits: ConcurrentEdits(fromSequential: []),
-            lookaheadRanges: lookaheadRanges
+            previousIncrementalParseResult: parseResult,
+            edits: ConcurrentEdits(fromSequential: [])
           )
         }
       }