From b79e5598446999781011a084c0aa6acfc6fc0e85 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Fri, 14 Apr 2023 13:21:03 -0700 Subject: [PATCH] Remove SwiftSyntaxParser module This removes the `SwiftSyntaxParser` module, which was just a wrapper around `SwiftParser` to ease the transition for clients that previously used `SwiftSyntaxParser`. Everybody should be using `SwiftParser` by now, so remove the deprecated module. This migrates all test cases that still provide value to other modules. --- BUILD.bazel | 8 - CONTRIBUTING.md | 8 +- Package.swift | 24 +- Sources/SwiftParser/README.md | 3 +- .../SwiftParser.docc/SwiftParser.md | 3 +- Sources/SwiftSyntaxParser/Diagnostic.swift | 310 ------------------ .../Documentation.docc/Index.md | 3 - Sources/SwiftSyntaxParser/SyntaxParser.swift | 132 -------- Sources/lit-test-helper/main.swift | 94 +----- .../ParsingPerformanceTests.swift | 14 +- .../SyntaxClassifierPerformanceTests.swift | 5 +- .../VisitorPerformanceTests.swift | 8 +- .../IncrementalParsingTests.swift | 10 +- Tests/SwiftParserTest/LexerTests.swift | 16 + .../Inputs/closure.swift | 3 - .../Inputs/diagnostics.swift | 11 - .../Inputs/near-empty.swift | 1 - .../Inputs/nested-blocks.swift | 8 - .../Inputs/visitor.swift | 7 - .../ParseFileTests.swift | 70 ---- Tests/SwiftSyntaxParserTest/SyntaxTests.swift | 186 ----------- .../SyntaxVisitorTests.swift | 138 -------- Tests/SwiftSyntaxParserTest/TokenTest.swift | 66 ---- .../AbsolutePositionTests.swift | 71 +--- Tests/SwiftSyntaxTest/SyntaxTests.swift | 77 +++++ .../SwiftSyntaxTest/SyntaxVisitorTests.swift | 240 ++++++++++++++ .../SyntaxComparisonTests.swift | 24 +- build-script.py | 1 - 28 files changed, 385 insertions(+), 1156 deletions(-) delete mode 100644 Sources/SwiftSyntaxParser/Diagnostic.swift delete mode 100644 Sources/SwiftSyntaxParser/Documentation.docc/Index.md delete mode 100644 Sources/SwiftSyntaxParser/SyntaxParser.swift rename Tests/{SwiftSyntaxParserTest => SwiftParserTest}/IncrementalParsingTests.swift (89%) delete mode 100644 Tests/SwiftSyntaxParserTest/Inputs/closure.swift delete mode 100644 Tests/SwiftSyntaxParserTest/Inputs/diagnostics.swift delete mode 100644 Tests/SwiftSyntaxParserTest/Inputs/near-empty.swift delete mode 100644 Tests/SwiftSyntaxParserTest/Inputs/nested-blocks.swift delete mode 100644 Tests/SwiftSyntaxParserTest/Inputs/visitor.swift delete mode 100644 Tests/SwiftSyntaxParserTest/ParseFileTests.swift delete mode 100644 Tests/SwiftSyntaxParserTest/SyntaxTests.swift delete mode 100644 Tests/SwiftSyntaxParserTest/SyntaxVisitorTests.swift delete mode 100644 Tests/SwiftSyntaxParserTest/TokenTest.swift rename Tests/{SwiftSyntaxParserTest => SwiftSyntaxTest}/AbsolutePositionTests.swift (77%) create mode 100644 Tests/SwiftSyntaxTest/SyntaxVisitorTests.swift rename Tests/{SwiftSyntaxParserTest => SwiftSyntaxTestSupportTest}/SyntaxComparisonTests.swift (92%) diff --git a/BUILD.bazel b/BUILD.bazel index 279de9ba822..fad2dd3cd37 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -95,14 +95,6 @@ swift_syntax_library( ], ) -swift_syntax_library( - name = "SwiftSyntaxParser", - deps = [ - ":SwiftParser", - ":SwiftSyntax", - ], -) - swift_syntax_library( name = "SwiftRefactor", deps = [ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ec61c1202f1..81cb3a5b7d3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,11 +43,7 @@ If you can’t find it in your Schemes, you need to manually add it using Produc ### XCTests -The `SwiftSyntaxParser` module (the legacy parser) of this repository depends on the C++ parser library (`_InternalSwiftSyntaxParser.dylib`) to parse source code. -The syntax node definitions of that parser library need to match those used by your SwiftSyntax checkout. -Most of the time, the parser library included in the latest Swift Development Snapshot will fulfill this requirement. - -To run the tests in Xcode, select the latest Swift Development Snapshot in Xcode -> Toolchains, select the SwiftSyntax-Package scheme and hit Product -> Test. +To run the tests in Xcode, select the SwiftSyntax-Package scheme and hit Product -> Test. You can also run the tests from the command line using ```bash @@ -68,7 +64,7 @@ Tip: Running SwiftSyntax’s self-parse tests takes the majority of testing time ### `lit`-based tests -A few tests of the `SwiftSyntaxParser` module (the legacy parser), which test the interaction between SwiftSyntax and the C++ parser library (`_InternalSwiftSyntaxParser.dylib`) are based LLVM’s `lit` and `FileCheck` tools. +A few tests are based LLVM’s `lit` and `FileCheck` tools. To run these, build `FileCheck`, e.g. by building the Swift compiler and run the tests using the following command: ```bash ./build-script.py test --toolchain /Library/Developer/Toolchains/swift-DEVELOPMENT-SNAPSHOT-.xctoolchain/usr --skip-lit-tests --filecheck-exec /path/to/build/Release+Asserts/llvm-macosx-x86_64/bin/FileCheck diff --git a/Package.swift b/Package.swift index a1570007ae7..7d8248ade85 100644 --- a/Package.swift +++ b/Package.swift @@ -49,7 +49,6 @@ let package = Package( .library(name: "SwiftSyntax", type: .static, targets: ["SwiftSyntax"]), .library(name: "SwiftSyntaxBuilder", type: .static, targets: ["SwiftSyntaxBuilder"]), .library(name: "SwiftSyntaxMacros", type: .static, targets: ["SwiftSyntaxMacros"]), - .library(name: "SwiftSyntaxParser", type: .static, targets: ["SwiftSyntaxParser"]), ], targets: [ // MARK: - Internal helper targets @@ -63,6 +62,11 @@ let package = Package( dependencies: ["SwiftBasicFormat", "SwiftSyntax", "SwiftSyntaxBuilder"] ), + .testTarget( + name: "SwiftSyntaxTestSupportTest", + dependencies: ["_SwiftSyntaxTestSupport", "SwiftParser"] + ), + // MARK: - Library targets // Formatting style: // - One section for each target and its test target @@ -236,7 +240,7 @@ let package = Package( .executableTarget( name: "lit-test-helper", - dependencies: ["IDEUtils", "SwiftSyntax", "SwiftSyntaxParser"] + dependencies: ["IDEUtils", "SwiftSyntax", "SwiftParser"] ), // MARK: PerformanceTest @@ -244,21 +248,7 @@ let package = Package( .testTarget( name: "PerformanceTest", - dependencies: ["IDEUtils", "SwiftParser", "SwiftSyntax", "SwiftSyntaxParser"], - exclude: ["Inputs"] - ), - - // MARK: SwiftSyntaxParser - // TODO: All clients should use SwiftParser instead - - .target( - name: "SwiftSyntaxParser", - dependencies: ["SwiftSyntax", "SwiftParser"] - ), - - .testTarget( - name: "SwiftSyntaxParserTest", - dependencies: ["_SwiftSyntaxTestSupport", "SwiftSyntaxParser"], + dependencies: ["IDEUtils", "SwiftParser", "SwiftSyntax"], exclude: ["Inputs"] ), ] diff --git a/Sources/SwiftParser/README.md b/Sources/SwiftParser/README.md index c9fdfbb746a..2818fc65fdf 100644 --- a/Sources/SwiftParser/README.md +++ b/Sources/SwiftParser/README.md @@ -1,7 +1,6 @@ ## Overview -The `SwiftParser` framework implements a parser that accepts Swift source text -as input and produces a SwiftSyntax syntax tree. This module is under active development and is not yet ready to completely replace `SwiftSyntaxParser`. For more information about the design of this module, please see [the module documentation](SwiftParser.docc/SwiftParser.md). +The `SwiftParser` framework implements a parser that accepts Swift source text as input and produces a SwiftSyntax syntax tree. ## Quickstart diff --git a/Sources/SwiftParser/SwiftParser.docc/SwiftParser.md b/Sources/SwiftParser/SwiftParser.docc/SwiftParser.md index 2f837fcf5fc..0fd444a234e 100644 --- a/Sources/SwiftParser/SwiftParser.docc/SwiftParser.md +++ b/Sources/SwiftParser/SwiftParser.docc/SwiftParser.md @@ -4,8 +4,7 @@ A parser for the Swift programming language. ## Overview -The `SwiftParser` framework implements a parser that accepts Swift source text -as input and produces a SwiftSyntax syntax tree. This module is under active development and is not yet ready to completely replace `SwiftSyntaxParser`. +The `SwiftParser` framework implements a parser that accepts Swift source text as input and produces a SwiftSyntax syntax tree. ## Quickstart diff --git a/Sources/SwiftSyntaxParser/Diagnostic.swift b/Sources/SwiftSyntaxParser/Diagnostic.swift deleted file mode 100644 index f8abb3d685f..00000000000 --- a/Sources/SwiftSyntaxParser/Diagnostic.swift +++ /dev/null @@ -1,310 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// -// This file provides the Diagnostic, Note, and FixIt types. -//===----------------------------------------------------------------------===// - -import SwiftSyntax - -/// A FixIt represents a change to source code in order to "correct" a -/// diagnostic. -public enum FixIt: Codable, CustomDebugStringConvertible { - - /// Remove the characters from the source file over the provided source range. - case remove(SourceRange) - - /// Insert, at the provided source location, the provided string. - case insert(SourceLocation, String) - - /// Replace the characters at the provided source range with the provided - /// string. - case replace(SourceRange, String) - - enum CodingKeys: String, CodingKey { - case type - case range - case location - case string - } - - public var debugDescription: String { - return "Fixit: \(range.debugDescription) Text: \"\(text)\"" - } - - public init(from decoder: Decoder) throws { - let container = try decoder.container(keyedBy: CodingKeys.self) - let type = try container.decode(String.self, forKey: .type) - switch type { - case "remove": - let range = try container.decode(SourceRange.self, forKey: .range) - self = .remove(range) - case "insert": - let string = try container.decode(String.self, forKey: .string) - let loc = try container.decode(SourceLocation.self, forKey: .location) - self = .insert(loc, string) - case "replace": - let string = try container.decode(String.self, forKey: .string) - let range = try container.decode(SourceRange.self, forKey: .range) - self = .replace(range, string) - default: - fatalError("unknown FixIt type \(type)") - } - } - - public func encode(to encoder: Encoder) throws { - var container = encoder.container(keyedBy: CodingKeys.self) - switch self { - case let .remove(range): - try container.encode(range, forKey: .range) - case let .insert(location, string): - try container.encode(location, forKey: .location) - try container.encode(string, forKey: .string) - case let .replace(range, string): - try container.encode(range, forKey: .range) - try container.encode(string, forKey: .string) - } - } - - /// The source range associated with a FixIt. If this is an insertion, - /// it is a range with the same start and end location. - public var range: SourceRange { - switch self { - case .remove(let range), .replace(let range, _): return range - case .insert(let loc, _): return SourceRange(start: loc, end: loc) - } - } - - /// The text associated with this FixIt. If this is a removal, the text is - /// the empty string. - public var text: String { - switch self { - case .remove(_): return "" - case .insert(_, let text), .replace(_, let text): return text - } - } -} - -/// A Note attached to a Diagnostic. This provides more context for a specific -/// error, and optionally allows for FixIts. -public struct Note: Codable { - /// The note's message. - public let message: Diagnostic.Message - - /// The source location where the note should point. - public let location: SourceLocation? - - /// An array of source ranges that should be highlighted. - public let highlights: [SourceRange] - - /// An array of FixIts that apply to this note. - public let fixIts: [FixIt] - - /// Constructs a new Note from the constituent parts. - public init( - message: Diagnostic.Message, - location: SourceLocation?, - highlights: [SourceRange], - fixIts: [FixIt] - ) { - precondition( - message.severity == .note, - "notes can only have the `note` severity" - ) - self.message = message - self.location = location - self.highlights = highlights - self.fixIts = fixIts - } - - /// Converts this Note to a Diagnostic for serialization. - public func asDiagnostic() -> Diagnostic { - return Diagnostic( - message: message, - location: location, - notes: [], - highlights: highlights, - fixIts: fixIts - ) - } -} - -/// A Diagnostic message that can be emitted regarding some piece of code. -public struct Diagnostic: Codable, CustomDebugStringConvertible { - - public struct Message: Codable, CustomDebugStringConvertible { - - public var debugDescription: String { - return "\(severity): \(text)" - } - - /// The severity of diagnostic. This can be note, error, or warning. - public let severity: Severity - - /// A string containing the contents of the diagnostic. - public let text: String - - /// Creates a diagnostic message with the provided severity and text. - public init(_ severity: Severity, _ text: String) { - self.severity = severity - self.text = text - } - } - - // These values must match clang/Frontend/SerializedDiagnostics.h - /// The severity of the diagnostic. - public enum Severity: UInt8, Codable { - case note = 1 - case warning = 2 - case error = 3 - } - - /// The diagnostic's message. - public let message: Message - - /// The location the diagnostic should point. - public let location: SourceLocation? - - /// An array of notes providing more context for this diagnostic. - public let notes: [Note] - - /// An array of source ranges to highlight. - public let highlights: [SourceRange] - - /// An array of possible FixIts to apply to this diagnostic. - public let fixIts: [FixIt] - - public var debugDescription: String { - var lines: [String] = [] - let loc = location == nil ? "" : "\(location!) " - lines.append("\(loc)\(message.debugDescription)") - fixIts.forEach { lines.append("\($0.debugDescription)") } - highlights.forEach { lines.append("Highlight: \($0.debugDescription)") } - lines.append(contentsOf: notes.map({ loc + $0.asDiagnostic().debugDescription })) - return lines.joined(separator: "\n") - } - - /// A diagnostic builder that exposes mutating operations for notes, - /// highlights, and FixIts. When a Diagnostic is created, a builder - /// will be provided in a closure where the user can conditionally - /// add notes, highlights, and FixIts, that will then be wrapped - /// into the immutable Diagnostic object. - public struct Builder { - /// An in-flight array of notes. - internal var notes = [Note]() - - /// An in-flight array of highlighted source ranges. - internal var highlights = [SourceRange]() - - /// An in-flight array of FixIts. - internal var fixIts = [FixIt]() - - internal init() {} - - /// Adds a Note to the diagnostic builder. - /// - parameters: - /// - message: The message associated with the note. This must have the - /// `.note` severity. - /// - location: The source location to which this note is attached. - /// - highlights: Any source ranges that should be highlighted by this - /// note. - /// - fixIts: Any FixIts that should be attached to this note. - public mutating func note( - _ message: Message, - location: SourceLocation? = nil, - highlights: [SourceRange] = [], - fixIts: [FixIt] = [] - ) { - self.notes.append( - Note( - message: message, - location: location, - highlights: highlights, - fixIts: fixIts - ) - ) - } - - /// Adds the provided source ranges as highlights of this diagnostic. - public mutating func highlight(_ ranges: SourceRange...) { - self.highlights += ranges - } - - /// Adds a FixIt to remove the contents of the provided SourceRange. - /// When applied, this FixIt will delete the characters corresponding to - /// this range in the original source file. - public mutating func fixItRemove(_ sourceRange: SourceRange) { - fixIts.append(.remove(sourceRange)) - } - - /// Adds a FixIt to insert the provided text at the provided SourceLocation - /// in the file where the location resides. - public mutating - func fixItInsert(_ text: String, at sourceLocation: SourceLocation) - { - fixIts.append(.insert(sourceLocation, text)) - } - - /// Adds a FixIt to replace the contents of the source file corresponding - /// to the provided SourceRange with the provided text. - public mutating - func fixItReplace(_ sourceRange: SourceRange, with text: String) - { - fixIts.append(.replace(sourceRange, text)) - } - } - - /// Creates a new Diagnostic with the provided message, pointing to the - /// provided location (if any). - /// This initializer also takes a closure that will be passed a Diagnostic - /// Builder as an inout parameter. Use this closure to add notes, highlights, - /// and FixIts to the diagnostic through the Builder's API. - /// - parameters: - /// - message: The diagnostic's message. - /// - location: The location the diagnostic is attached to. - /// - actions: A closure that's used to attach notes and highlights to - /// diagnostics. - public init( - message: Message, - location: SourceLocation?, - actions: ((inout Builder) -> Void)? - ) { - var builder = Builder() - actions?(&builder) - self.init( - message: message, - location: location, - notes: builder.notes, - highlights: builder.highlights, - fixIts: builder.fixIts - ) - } - - /// Creates a new Diagnostic with the provided message, pointing to the - /// provided location (if any). - /// - parameters: - /// - message: The diagnostic's message. - /// - location: The location the diagnostic is attached to. - /// - highlights: An array of SourceRanges which will be highlighted when - /// the diagnostic is presented. - public init( - message: Message, - location: SourceLocation?, - notes: [Note], - highlights: [SourceRange], - fixIts: [FixIt] - ) { - self.message = message - self.location = location - self.notes = notes - self.highlights = highlights - self.fixIts = fixIts - } -} diff --git a/Sources/SwiftSyntaxParser/Documentation.docc/Index.md b/Sources/SwiftSyntaxParser/Documentation.docc/Index.md deleted file mode 100644 index 20a769aee35..00000000000 --- a/Sources/SwiftSyntaxParser/Documentation.docc/Index.md +++ /dev/null @@ -1,3 +0,0 @@ -# ``SwiftSyntaxParser`` - -SwiftSyntaxParser allows parsing Swift source code into a SwiftSyntax tree. \ No newline at end of file diff --git a/Sources/SwiftSyntaxParser/SyntaxParser.swift b/Sources/SwiftSyntaxParser/SyntaxParser.swift deleted file mode 100644 index 37f9c64d0b2..00000000000 --- a/Sources/SwiftSyntaxParser/SyntaxParser.swift +++ /dev/null @@ -1,132 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// -// -// Uses the C API for syntax parsing that is part of the Swift toolchain. -// -//===----------------------------------------------------------------------===// - -import Foundation -@_spi(RawSyntax) import SwiftSyntax -@_implementationOnly import SwiftParser - -/// Namespace for functions to parse swift source and retrieve a syntax tree. -public enum SyntaxParser { - /// Parses the string into a full-fidelity Syntax tree. - /// - /// - Parameters: - /// - source: The source string to parse. - /// - parseTransition: Optional mechanism for incremental re-parsing. - /// - filenameForDiagnostics: Optional file name used for SourceLocation. - /// - languageVersion: Interpret input according to a specific Swift - /// language version number. If `nil`, this inherits the default from - /// the syntax parser library. - /// - enableBareSlashRegexLiteral: Enable or disable the use of forward - /// slash regular-expression literal syntax. If `nil`, this inherits the - /// default from the syntax parser library. - /// - diagnosticHandler: Optional callback that will be called for all - /// diagnostics the parser emits - /// - Returns: A top-level Syntax node representing the contents of the tree, - /// if the parse was successful. - /// - Throws: `ParserError` - public static func parse( - source: String, - parseTransition: IncrementalParseTransition? = nil, - filenameForDiagnostics: String = "", - languageVersion: String? = nil, - enableBareSlashRegexLiteral: Bool? = nil, - diagnosticHandler: ((Diagnostic) -> Void)? = nil - ) throws -> SourceFileSyntax { - // Get a native UTF8 string for efficient indexing with UTF8 byte offsets. - // If the string is backed by an NSString then such indexing will become - // extremely slow. - var utf8Source = source - utf8Source.makeContiguousUTF8() - - return try parseImpl( - source: utf8Source, - parseTransition: parseTransition, - filenameForDiagnostics: filenameForDiagnostics, - languageVersion: languageVersion, - enableBareSlashRegexLiteral: enableBareSlashRegexLiteral, - diagnosticHandler: diagnosticHandler - ) - } - - /// Parses the file `URL` into a full-fidelity Syntax tree. - /// - /// - Parameters: - /// - url: The file URL to parse. - /// - languageVersion: Interpret input according to a specific Swift - /// language version number. If `nil`, this inherits the default from - /// the syntax parser library. - /// - enableBareSlashRegexLiteral: Enable or disable the use of forward - /// slash regular-expression literal syntax. If `nil`, this inherits the - /// default from the syntax parser library. - /// - diagnosticHandler: Optional callback that will be called for all - /// diagnostics the parser emits - /// - Returns: A top-level Syntax node representing the contents of the tree, - /// if the parse was successful. - /// - Throws: `ParserError` - public static func parse( - _ url: URL, - languageVersion: String? = nil, - enableBareSlashRegexLiteral: Bool? = nil, - diagnosticHandler: ((Diagnostic) -> Void)? = nil - ) throws -> SourceFileSyntax { - // Avoid using `String(contentsOf:)` because it creates a wrapped NSString. - let fileData = try Data(contentsOf: url) - let source = fileData.withUnsafeBytes { buf in - return String(decoding: buf.bindMemory(to: UInt8.self), as: UTF8.self) - } - return try parse( - source: source, - filenameForDiagnostics: url.path, - languageVersion: languageVersion, - enableBareSlashRegexLiteral: enableBareSlashRegexLiteral, - diagnosticHandler: diagnosticHandler - ) - } - - private static func parseImpl( - source: String, - parseTransition: IncrementalParseTransition?, - filenameForDiagnostics: String, - languageVersion: String?, - enableBareSlashRegexLiteral: Bool?, - diagnosticHandler: ((Diagnostic) -> Void)? - ) throws -> SourceFileSyntax { - return Parser.parse(source: source, parseTransition: parseTransition) - } -} - -extension Note { - init(_ diag: Diagnostic) { - self.init( - message: diag.message, - location: diag.location, - highlights: diag.highlights, - fixIts: diag.fixIts - ) - } -} - -extension Diagnostic { - init(_ diag: Diagnostic, _ notes: [Note]) { - self.init( - message: diag.message, - location: diag.location, - notes: notes, - highlights: diag.highlights, - fixIts: diag.fixIts - ) - } -} diff --git a/Sources/lit-test-helper/main.swift b/Sources/lit-test-helper/main.swift index 58b7a057a7b..582def6f580 100644 --- a/Sources/lit-test-helper/main.swift +++ b/Sources/lit-test-helper/main.swift @@ -11,9 +11,16 @@ //===----------------------------------------------------------------------===// import SwiftSyntax -import SwiftSyntaxParser +import SwiftParser import Foundation +extension Parser { + public static func parse(_ url: URL) throws -> SourceFileSyntax { + let source = try String(contentsOf: url) + return Self.parse(source: source) + } +} + /// Print the given message to stderr func printerr(_ message: String, terminator: String = "\n") { FileHandle.standardError.write((message + terminator).data(using: .utf8)!) @@ -230,13 +237,6 @@ struct IncrementalEdit { let replacement: String } -func getSwiftLanguageVersionInfo(args: CommandLineArguments) -> (languageVersion: String?, enableBareSlashRegexLiteral: Bool?) { - return ( - args["-swift-version"], - args["-enable-bare-slash-regex"].map({ $0 == "1" }) - ) -} - /// Rewrites a parsed tree with all constructed nodes. class TreeReconstructor: SyntaxRewriter { override func visit(_ token: TokenSyntax) -> TokenSyntax { @@ -252,9 +252,8 @@ class TreeReconstructor: SyntaxRewriter { func performClassifySyntax(args: CommandLineArguments) throws { let treeURL = URL(fileURLWithPath: try args.getRequired("-source-file")) - let versionInfo = getSwiftLanguageVersionInfo(args: args) - let tree = try SyntaxParser.parse(treeURL, languageVersion: versionInfo.languageVersion, enableBareSlashRegexLiteral: versionInfo.enableBareSlashRegexLiteral) + let tree = try Parser.parse(treeURL) let result = ClassifiedSyntaxTreePrinter.print(Syntax(tree)) do { // Sanity check that we get the same result if the tree has constructed nodes. @@ -337,9 +336,8 @@ func performParseIncremental(args: CommandLineArguments) throws { URL(fileURLWithPath: try args.getRequired("-old-source-file")) let postEditURL = URL(fileURLWithPath: try args.getRequired("-source-file")) let expectedReparseRegions = try args.getReparseRegions() - let versionInfo = getSwiftLanguageVersionInfo(args: args) - let preEditTree = try SyntaxParser.parse(preEditURL, languageVersion: versionInfo.languageVersion, enableBareSlashRegexLiteral: versionInfo.enableBareSlashRegexLiteral) + let preEditTree = try Parser.parse(preEditURL) let edits = try parseIncrementalEditArguments(args: args) let regionCollector = IncrementalParseReusedNodeCollector() let editTransition = IncrementalParseTransition( @@ -349,8 +347,7 @@ func performParseIncremental(args: CommandLineArguments) throws { ) let postEditText = try String(contentsOf: postEditURL) - let postEditTree = - try SyntaxParser.parse(source: postEditText, parseTransition: editTransition) + let postEditTree = Parser.parse(source: postEditText, parseTransition: editTransition) let postTreeDump = postEditTree.description @@ -455,8 +452,7 @@ func verifyReusedRegions( func performRoundtrip(args: CommandLineArguments) throws { let sourceURL = URL(fileURLWithPath: try args.getRequired("-source-file")) - let versionInfo = getSwiftLanguageVersionInfo(args: args) - let tree = try SyntaxParser.parse(sourceURL, languageVersion: versionInfo.languageVersion, enableBareSlashRegexLiteral: versionInfo.enableBareSlashRegexLiteral) + let tree = try Parser.parse(sourceURL) let treeText = tree.description if let outURL = args["-out"].map(URL.init(fileURLWithPath:)) { @@ -471,9 +467,8 @@ func performVerifyRoundtrip(args: CommandLineArguments) throws { guard let source = try String(data: Data(contentsOf: sourceURL), encoding: .utf8) else { throw TestingError.readingSourceFileFailed(sourceURL) } - let versionInfo = getSwiftLanguageVersionInfo(args: args) - let tree = try SyntaxParser.parse(source: source, languageVersion: versionInfo.languageVersion, enableBareSlashRegexLiteral: versionInfo.enableBareSlashRegexLiteral) + let tree = Parser.parse(source: source) if tree.description != source { throw TestingError.roundTripFailed } @@ -500,73 +495,16 @@ class NodePrinter: SyntaxAnyVisitor { func printSyntaxTree(args: CommandLineArguments) throws { let treeURL = URL(fileURLWithPath: try args.getRequired("-source-file")) - let versionInfo = getSwiftLanguageVersionInfo(args: args) - let tree = try SyntaxParser.parse(treeURL, languageVersion: versionInfo.languageVersion, enableBareSlashRegexLiteral: versionInfo.enableBareSlashRegexLiteral) + let tree = try Parser.parse(treeURL) let printer = NodePrinter() printer.walk(tree) } -func printParserDiags(args: CommandLineArguments) throws { - let treeURL = URL(fileURLWithPath: try args.getRequired("-source-file")) - let versionInfo = getSwiftLanguageVersionInfo(args: args) - - var diagCounter: (error: Int, warning: Int, note: Int) = (0, 0, 0) - - func handleDiagnostic(diagnostic: Diagnostic) { - switch diagnostic.message.severity { - case .error: - diagCounter.error += 1 - diagCounter.note += diagnostic.notes.count - case .warning: - diagCounter.warning += 1 - diagCounter.note += diagnostic.notes.count - case .note: - diagCounter.note += 1 - } - print(diagnostic.debugDescription) - } - - _ = try SyntaxParser.parse(treeURL, languageVersion: versionInfo.languageVersion, enableBareSlashRegexLiteral: versionInfo.enableBareSlashRegexLiteral, diagnosticHandler: handleDiagnostic) - - print("\(diagCounter.error) error(s) \(diagCounter.warning) warnings(s) \(diagCounter.note) note(s)") -} - /// Write the given string to stderr **without** appending a newline character. func writeToStderr(_ msg: String) { FileHandle.standardError.write(msg.data(using: .utf8)!) } -func diagnose(args: CommandLineArguments) throws { - func printDiagnostic(diagnostic: Diagnostic) { - if let loc = diagnostic.location { - writeToStderr("\(loc.file!):\(loc.line!):\(loc.column!): ") - } else { - writeToStderr(":0:0: ") - } - switch diagnostic.message.severity { - case .note: writeToStderr("note: ") - case .warning: writeToStderr("warning: ") - case .error: writeToStderr("error: ") - } - writeToStderr(diagnostic.message.text) - writeToStderr("\n") - - for note in diagnostic.notes { - printDiagnostic(diagnostic: note.asDiagnostic()) - } - } - - let treeURL = URL(fileURLWithPath: try args.getRequired("-source-file")) - let versionInfo = getSwiftLanguageVersionInfo(args: args) - - _ = try SyntaxParser.parse( - treeURL, - languageVersion: versionInfo.languageVersion, - enableBareSlashRegexLiteral: versionInfo.enableBareSlashRegexLiteral, - diagnosticHandler: printDiagnostic - ) -} - do { let args = try CommandLineArguments.parse(CommandLine.arguments.dropFirst()) @@ -580,10 +518,6 @@ do { try performVerifyRoundtrip(args: args) } else if args.has("-print-tree") { try printSyntaxTree(args: args) - } else if args.has("-dump-diags") { - try printParserDiags(args: args) - } else if args.has("-diagnose") { - try diagnose(args: args) } else if args.has("-help") { printHelp() } else { diff --git a/Tests/PerformanceTest/ParsingPerformanceTests.swift b/Tests/PerformanceTest/ParsingPerformanceTests.swift index f45dd0a4208..817ce93470e 100644 --- a/Tests/PerformanceTest/ParsingPerformanceTests.swift +++ b/Tests/PerformanceTest/ParsingPerformanceTests.swift @@ -12,7 +12,6 @@ import XCTest import SwiftSyntax -import SwiftSyntaxParser import SwiftParser public class ParsingPerformanceTests: XCTestCase { @@ -24,23 +23,12 @@ public class ParsingPerformanceTests: XCTestCase { .appendingPathComponent("MinimalCollections.swift.input") } - func testParsingPerformance() throws { - try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1") - measure { - do { - _ = try SyntaxParser.parse(inputFile) - } catch { - XCTFail(error.localizedDescription) - } - } - } - func testNativeParsingPerformance() throws { try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1") measure { do { let source = try String(contentsOf: inputFile) - _ = SwiftParser.Parser.parse(source: source) + _ = Parser.parse(source: source) } catch { XCTFail(error.localizedDescription) } diff --git a/Tests/PerformanceTest/SyntaxClassifierPerformanceTests.swift b/Tests/PerformanceTest/SyntaxClassifierPerformanceTests.swift index e17d3d54e6d..3b5dbd13c5b 100644 --- a/Tests/PerformanceTest/SyntaxClassifierPerformanceTests.swift +++ b/Tests/PerformanceTest/SyntaxClassifierPerformanceTests.swift @@ -13,7 +13,7 @@ import XCTest import IDEUtils import SwiftSyntax -import SwiftSyntaxParser +import SwiftParser public class SyntaxClassifierPerformanceTests: XCTestCase { @@ -28,7 +28,8 @@ public class SyntaxClassifierPerformanceTests: XCTestCase { try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1") XCTAssertNoThrow( try { - let parsed = try SyntaxParser.parse(inputFile) + let source = try String(contentsOf: inputFile) + let parsed = Parser.parse(source: source) measure { for _ in 0..<10 { diff --git a/Tests/PerformanceTest/VisitorPerformanceTests.swift b/Tests/PerformanceTest/VisitorPerformanceTests.swift index d39c8918f2d..6ec95aadce8 100644 --- a/Tests/PerformanceTest/VisitorPerformanceTests.swift +++ b/Tests/PerformanceTest/VisitorPerformanceTests.swift @@ -12,7 +12,7 @@ import XCTest import SwiftSyntax -import SwiftSyntaxParser +import SwiftParser public class VisitorPerformanceTests: XCTestCase { @@ -29,7 +29,7 @@ public class VisitorPerformanceTests: XCTestCase { XCTAssertNoThrow( try { - let parsed = try SyntaxParser.parse(inputFile) + let parsed = Parser.parse(source: try String(contentsOf: inputFile)) let emptyVisitor = EmptyVisitor(viewMode: .sourceAccurate) @@ -46,7 +46,7 @@ public class VisitorPerformanceTests: XCTestCase { XCTAssertNoThrow( try { - let parsed = try SyntaxParser.parse(inputFile) + let parsed = Parser.parse(source: try String(contentsOf: inputFile)) let emptyRewriter = EmptyRewriter() @@ -63,7 +63,7 @@ public class VisitorPerformanceTests: XCTestCase { XCTAssertNoThrow( try { - let parsed = try SyntaxParser.parse(inputFile) + let parsed = Parser.parse(source: try String(contentsOf: inputFile)) let emptyVisitor = EmptyAnyVisitor(viewMode: .sourceAccurate) diff --git a/Tests/SwiftSyntaxParserTest/IncrementalParsingTests.swift b/Tests/SwiftParserTest/IncrementalParsingTests.swift similarity index 89% rename from Tests/SwiftSyntaxParserTest/IncrementalParsingTests.swift rename to Tests/SwiftParserTest/IncrementalParsingTests.swift index fdbbee31f3b..e4d1cc1bdaf 100644 --- a/Tests/SwiftSyntaxParserTest/IncrementalParsingTests.swift +++ b/Tests/SwiftParserTest/IncrementalParsingTests.swift @@ -12,7 +12,7 @@ import XCTest import SwiftSyntax -import SwiftSyntaxParser +import SwiftParser public class IncrementalParsingTests: XCTestCase { @@ -21,10 +21,10 @@ public class IncrementalParsingTests: XCTestCase { let step: (String, (Int, Int, String)) = ("struct AA { func f() {", (8, 0, "A")) - var tree = try! SyntaxParser.parse(source: original) + var tree = Parser.parse(source: original) let sourceEdit = SourceEdit(range: ByteSourceRange(offset: step.1.0, length: step.1.1), replacementLength: step.1.2.utf8.count) let lookup = IncrementalParseTransition(previousTree: tree, edits: ConcurrentEdits(sourceEdit)) - tree = try! SyntaxParser.parse(source: step.0, parseTransition: lookup) + tree = Parser.parse(source: step.0, parseTransition: lookup) XCTAssertEqual("\(tree)", step.0) } @@ -35,11 +35,11 @@ public class IncrementalParsingTests: XCTestCase { let step: (String, (Int, Int, String)) = ("struct AA {}\nstruct B {}\n", (8, 0, "A")) - let origTree = try! SyntaxParser.parse(source: original) + let origTree = Parser.parse(source: original) let sourceEdit = SourceEdit(range: ByteSourceRange(offset: step.1.0, length: step.1.1), replacementLength: step.1.2.utf8.count) let reusedNodeCollector = IncrementalParseReusedNodeCollector() let transition = IncrementalParseTransition(previousTree: origTree, edits: ConcurrentEdits(sourceEdit), reusedNodeDelegate: reusedNodeCollector) - let newTree = try! SyntaxParser.parse(source: step.0, parseTransition: transition) + let newTree = Parser.parse(source: step.0, parseTransition: transition) XCTAssertEqual("\(newTree)", step.0) let origStructB = origTree.statements[1] diff --git a/Tests/SwiftParserTest/LexerTests.swift b/Tests/SwiftParserTest/LexerTests.swift index 047b7f9fb9e..8ea04a80eb3 100644 --- a/Tests/SwiftParserTest/LexerTests.swift +++ b/Tests/SwiftParserTest/LexerTests.swift @@ -1240,6 +1240,22 @@ public class LexerTests: XCTestCase { ) } + func testNullCharacterInSourceFile() { + assertLexemes( + "var x = 11️⃣\0\nvar y = 2", + lexemes: [ + LexemeSpec(.keyword, text: "var", trailing: " "), + LexemeSpec(.identifier, text: "x", trailing: " "), + LexemeSpec(.equal, text: "=", trailing: " "), + LexemeSpec(.integerLiteral, text: "1", trailing: "\0", diagnostic: "nul character embedded in middle of file"), + LexemeSpec(.keyword, leading: "\n", text: "var", trailing: " ", flags: .isAtStartOfLine), + LexemeSpec(.identifier, text: "y", trailing: " "), + LexemeSpec(.equal, text: "=", trailing: " "), + LexemeSpec(.integerLiteral, text: "2", trailing: ""), + ] + ) + } + func testNullCharacterInStringLiteral() { assertLexemes( """ diff --git a/Tests/SwiftSyntaxParserTest/Inputs/closure.swift b/Tests/SwiftSyntaxParserTest/Inputs/closure.swift deleted file mode 100644 index 64b2c2aa20b..00000000000 --- a/Tests/SwiftSyntaxParserTest/Inputs/closure.swift +++ /dev/null @@ -1,3 +0,0 @@ -// A closure without a signature. The test will ensure it stays the same after -// applying a rewriting pass. -let x: () -> Void = {} \ No newline at end of file diff --git a/Tests/SwiftSyntaxParserTest/Inputs/diagnostics.swift b/Tests/SwiftSyntaxParserTest/Inputs/diagnostics.swift deleted file mode 100644 index e9a6f8ce4d9..00000000000 --- a/Tests/SwiftSyntaxParserTest/Inputs/diagnostics.swift +++ /dev/null @@ -1,11 +0,0 @@ -func foo() { - -} - -func bar() { - -} - -func baz() { - -} \ No newline at end of file diff --git a/Tests/SwiftSyntaxParserTest/Inputs/near-empty.swift b/Tests/SwiftSyntaxParserTest/Inputs/near-empty.swift deleted file mode 100644 index 8ba3a16384a..00000000000 --- a/Tests/SwiftSyntaxParserTest/Inputs/near-empty.swift +++ /dev/null @@ -1 +0,0 @@ -n diff --git a/Tests/SwiftSyntaxParserTest/Inputs/nested-blocks.swift b/Tests/SwiftSyntaxParserTest/Inputs/nested-blocks.swift deleted file mode 100644 index 9993ae44877..00000000000 --- a/Tests/SwiftSyntaxParserTest/Inputs/nested-blocks.swift +++ /dev/null @@ -1,8 +0,0 @@ -struct Foo { - func foo() { - print("hello") - func bar() { - print("goodbye") - } - } -} diff --git a/Tests/SwiftSyntaxParserTest/Inputs/visitor.swift b/Tests/SwiftSyntaxParserTest/Inputs/visitor.swift deleted file mode 100644 index b2b527f8f78..00000000000 --- a/Tests/SwiftSyntaxParserTest/Inputs/visitor.swift +++ /dev/null @@ -1,7 +0,0 @@ -func foo() { - public func foo() { - func foo() { - /*Unknown token */0xG - } - } -} diff --git a/Tests/SwiftSyntaxParserTest/ParseFileTests.swift b/Tests/SwiftSyntaxParserTest/ParseFileTests.swift deleted file mode 100644 index 5f40f4f4b3e..00000000000 --- a/Tests/SwiftSyntaxParserTest/ParseFileTests.swift +++ /dev/null @@ -1,70 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import XCTest -import SwiftSyntax -import SwiftSyntaxParser - -fileprivate struct Foo { - public let x: Int - private(set) var y: [Bool] -} - -#if os(macOS) -fileprivate class Test: NSObject { - @objc var bar: Int = 0 - func test() { - print(#selector(function)) - print(#keyPath(bar)) - } - @objc func function() { - } -} -#endif - -enum Animal { - case cat - case dog -} - -public class ParseFileTests: XCTestCase { - - public func testParseSingleFile() { - let currentFile = URL(fileURLWithPath: #file) - XCTAssertNoThrow( - try { - let fileContents = try String(contentsOf: currentFile) - let parsed = try SyntaxParser.parse(currentFile) - XCTAssertEqual("\(parsed)", fileContents) - }() - ) - } - - public func testEnumCaseStructure() { - class Visitor: SyntaxVisitor { - var cases: [EnumCaseDeclSyntax] = [] - override func visit(_ node: EnumDeclSyntax) -> SyntaxVisitorContinueKind { - cases.append( - contentsOf: node.memberBlock.members.compactMap { - $0.decl.as(EnumCaseDeclSyntax.self) - } - ) - return .skipChildren - } - } - let v = Visitor(viewMode: .fixedUp) - let currentFile = URL(fileURLWithPath: #file) - let parsed = try! SyntaxParser.parse(currentFile) - v.walk(parsed) - XCTAssertEqual(v.cases.count, 2) - } -} diff --git a/Tests/SwiftSyntaxParserTest/SyntaxTests.swift b/Tests/SwiftSyntaxParserTest/SyntaxTests.swift deleted file mode 100644 index 97ed0b6abaf..00000000000 --- a/Tests/SwiftSyntaxParserTest/SyntaxTests.swift +++ /dev/null @@ -1,186 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import XCTest -import SwiftSyntax -import SwiftSyntaxParser - -fileprivate extension SyntaxProtocol { - func token(at index: Int, viewMode: SyntaxTreeViewMode) -> TokenSyntax? { - var token = self.firstToken(viewMode: viewMode) - for _ in 0.. SyntaxVisitorContinueKind { - funcCount += 1 - return .visitChildren - } - } - XCTAssertNoThrow( - try { - let parsed = try SyntaxParser.parse(getTestInput("visitor.swift")) - let counter = FuncCounter(viewMode: .fixedUp) - let hashBefore = parsed.hashValue - counter.walk(parsed) - XCTAssertEqual(counter.funcCount, 3) - XCTAssertEqual(hashBefore, parsed.hashValue) - }() - ) - } - - public func testRewritingNodeWithEmptyChild() { - class ClosureRewriter: SyntaxRewriter { - override func visit(_ node: ClosureExprSyntax) -> ExprSyntax { - // Perform a no-op transform that requires rebuilding the node. - return ExprSyntax(node.with(\.signature, node.signature)) - } - } - XCTAssertNoThrow( - try { - let parsed = try SyntaxParser.parse(getTestInput("closure.swift")) - let rewriter = ClosureRewriter() - let rewritten = rewriter.visit(parsed) - XCTAssertEqual(parsed.description, rewritten.description) - }() - ) - } - - public func testSyntaxRewriterVisitAny() { - class VisitAnyRewriter: SyntaxRewriter { - let transform: (TokenSyntax) -> TokenSyntax - init(transform: @escaping (TokenSyntax) -> TokenSyntax) { - self.transform = transform - } - override func visitAny(_ node: Syntax) -> Syntax? { - if let tok = node.as(TokenSyntax.self) { - return Syntax(transform(tok)) - } - return nil - } - } - XCTAssertNoThrow( - try { - let parsed = try SyntaxParser.parse(getTestInput("near-empty.swift")) - let rewriter = VisitAnyRewriter(transform: { _ in - return TokenSyntax.identifier("") - }) - let rewritten = rewriter.visit(parsed) - XCTAssertEqual(rewritten.description, "") - }() - ) - } - - public func testSyntaxRewriterVisitCollection() { - class VisitCollections: SyntaxVisitor { - var numberOfCodeBlockItems = 0 - - override func visit(_ items: CodeBlockItemListSyntax) -> SyntaxVisitorContinueKind { - numberOfCodeBlockItems += items.count - return .visitChildren - } - } - - XCTAssertNoThrow( - try { - let parsed = try SyntaxParser.parse(getTestInput("nested-blocks.swift")) - let visitor = VisitCollections(viewMode: .fixedUp) - visitor.walk(parsed) - XCTAssertEqual(4, visitor.numberOfCodeBlockItems) - }() - ) - } - - public func testVisitorClass() { - class FuncCounter: SyntaxVisitor { - var funcCount = 0 - override func visit(_ node: FunctionDeclSyntax) -> SyntaxVisitorContinueKind { - funcCount += 1 - return super.visit(node) - } - } - XCTAssertNoThrow( - try { - let parsed = try SyntaxParser.parse(getTestInput("visitor.swift")) - let counter = FuncCounter(viewMode: .fixedUp) - let hashBefore = parsed.hashValue - counter.walk(parsed) - XCTAssertEqual(counter.funcCount, 3) - XCTAssertEqual(hashBefore, parsed.hashValue) - }() - ) - } - - public func testRewriteTrivia() { - class TriviaRemover: SyntaxRewriter { - override func visit(_ token: TokenSyntax) -> TokenSyntax { - return token.with(\.trailingTrivia, []) - } - } - - XCTAssertNoThrow( - try { - let parsed = try SyntaxParser.parse(source: "let a = 5") - let visitor = TriviaRemover() - let rewritten = visitor.visit(parsed) - XCTAssertEqual(rewritten.description, "leta=5") - }() - ) - } -} diff --git a/Tests/SwiftSyntaxParserTest/TokenTest.swift b/Tests/SwiftSyntaxParserTest/TokenTest.swift deleted file mode 100644 index 5cd71f57f57..00000000000 --- a/Tests/SwiftSyntaxParserTest/TokenTest.swift +++ /dev/null @@ -1,66 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import XCTest -@_spi(Testing) import SwiftSyntax -import SwiftSyntaxParser - -public class TokenTests: XCTestCase { - - public func testKeywordKinds() { - XCTAssertTrue(TokenKind.keyword(.operator).isLexerClassifiedKeyword) - XCTAssertTrue(TokenKind.keyword(.func).isLexerClassifiedKeyword) - XCTAssertFalse(TokenKind.leftAngle.isLexerClassifiedKeyword) - XCTAssertFalse(TokenKind.rightAngle.isLexerClassifiedKeyword) - } - - public func testPunctuators() { - XCTAssertTrue(TokenKind.leftParen.isPunctuation) - XCTAssertTrue(TokenKind.rightParen.isPunctuation) - XCTAssertTrue(TokenKind.leftBrace.isPunctuation) - XCTAssertTrue(TokenKind.rightBrace.isPunctuation) - XCTAssertTrue(TokenKind.leftSquareBracket.isPunctuation) - XCTAssertTrue(TokenKind.rightSquareBracket.isPunctuation) - XCTAssertTrue(TokenKind.leftAngle.isPunctuation) - XCTAssertTrue(TokenKind.rightAngle.isPunctuation) - XCTAssertTrue(TokenKind.period.isPunctuation) - XCTAssertTrue(TokenKind.comma.isPunctuation) - XCTAssertTrue(TokenKind.ellipsis.isPunctuation) - XCTAssertTrue(TokenKind.colon.isPunctuation) - XCTAssertTrue(TokenKind.semicolon.isPunctuation) - XCTAssertTrue(TokenKind.equal.isPunctuation) - XCTAssertTrue(TokenKind.atSign.isPunctuation) - XCTAssertTrue(TokenKind.pound.isPunctuation) - XCTAssertTrue(TokenKind.prefixAmpersand.isPunctuation) - XCTAssertTrue(TokenKind.arrow.isPunctuation) - XCTAssertTrue(TokenKind.backtick.isPunctuation) - XCTAssertTrue(TokenKind.backslash.isPunctuation) - XCTAssertTrue(TokenKind.exclamationMark.isPunctuation) - XCTAssertTrue(TokenKind.postfixQuestionMark.isPunctuation) - XCTAssertTrue(TokenKind.infixQuestionMark.isPunctuation) - XCTAssertTrue(TokenKind.stringQuote.isPunctuation) - XCTAssertTrue(TokenKind.singleQuote.isPunctuation) - XCTAssertTrue(TokenKind.multilineStringQuote.isPunctuation) - } - - public func testTokenLgnth() { - let source = "\"\"\"\n\\(a)\n\"\"\"" - let tree = try! SyntaxParser.parse(source: source) - let tok = tree.firstToken(viewMode: .sourceAccurate)! - XCTAssertTrue(tok.tokenKind == .multilineStringQuote) - XCTAssertEqual(tok.contentLength.utf8Length, 3) - - let tok2 = TokenSyntax.multilineStringQuoteToken() - XCTAssertTrue(tok2.tokenKind == .multilineStringQuote) - XCTAssertEqual(tok2.contentLength.utf8Length, 3) - } -} diff --git a/Tests/SwiftSyntaxParserTest/AbsolutePositionTests.swift b/Tests/SwiftSyntaxTest/AbsolutePositionTests.swift similarity index 77% rename from Tests/SwiftSyntaxParserTest/AbsolutePositionTests.swift rename to Tests/SwiftSyntaxTest/AbsolutePositionTests.swift index f6b8ab181ef..518012d11b2 100644 --- a/Tests/SwiftSyntaxParserTest/AbsolutePositionTests.swift +++ b/Tests/SwiftSyntaxTest/AbsolutePositionTests.swift @@ -10,10 +10,9 @@ // //===----------------------------------------------------------------------===// -import XCTest -import SwiftSyntax -import SwiftSyntaxParser import _SwiftSyntaxTestSupport +import SwiftSyntax +import XCTest fileprivate class FuncRenamer: SyntaxRewriter { override func visit(_ node: FunctionDeclSyntax) -> DeclSyntax { @@ -24,72 +23,6 @@ fileprivate class FuncRenamer: SyntaxRewriter { } public class AbsolutePositionTests: XCTestCase { - - public func testVisitor() { - XCTAssertNoThrow( - try { - let source = try String(contentsOf: getTestInput("visitor.swift")) - let parsed = try SyntaxParser.parse(getTestInput("visitor.swift")) - XCTAssertEqual(0, parsed.position.utf8Offset) - XCTAssertEqual( - source.count, - parsed.eofToken.positionAfterSkippingLeadingTrivia.utf8Offset - ) - XCTAssertEqual(0, parsed.position.utf8Offset) - XCTAssertEqual(source.count, parsed.byteSize) - }() - ) - } - - public func testClosure() { - XCTAssertNoThrow( - try { - let source = try String(contentsOf: getTestInput("closure.swift")) - let parsed = try SyntaxParser.parse(getTestInput("closure.swift")) - XCTAssertEqual( - source.count, - parsed.eofToken.positionAfterSkippingLeadingTrivia.utf8Offset - ) - XCTAssertEqual(0, parsed.position.utf8Offset) - XCTAssertEqual(source.count, parsed.byteSize) - }() - ) - } - - public func testRename() { - XCTAssertNoThrow( - try { - let parsed = try SyntaxParser.parse(getTestInput("visitor.swift")) - let renamed = FuncRenamer().visit(parsed) - let renamedSource = renamed.description - XCTAssertEqual( - renamedSource.count, - renamed.eofToken.positionAfterSkippingLeadingTrivia.utf8Offset - ) - XCTAssertEqual(renamedSource.count, renamed.byteSize) - }() - ) - } - - public func testCurrentFile() { - XCTAssertNoThrow( - try { - let parsed = try SyntaxParser.parse(URL(fileURLWithPath: #file)) - class Visitor: SyntaxVisitor { - override func visit(_ node: TokenSyntax) -> SyntaxVisitorContinueKind { - XCTAssertEqual( - node.positionAfterSkippingLeadingTrivia.utf8Offset, - node.position.utf8Offset + node.leadingTrivia.byteSize - ) - return .skipChildren - } - } - let visitor = Visitor(viewMode: .sourceAccurate) - visitor.walk(parsed) - }() - ) - } - public func testRecursion() { var l = [CodeBlockItemSyntax]() let idx = 2000 diff --git a/Tests/SwiftSyntaxTest/SyntaxTests.swift b/Tests/SwiftSyntaxTest/SyntaxTests.swift index e37902dade1..f1ab22e4b02 100644 --- a/Tests/SwiftSyntaxTest/SyntaxTests.swift +++ b/Tests/SwiftSyntaxTest/SyntaxTests.swift @@ -48,4 +48,81 @@ public class SyntaxTests: XCTestCase { XCTAssertEqual(Syntax(s), s.memberBlock.parent) XCTAssertNil(s.memberBlock.detach().parent) } + + public func testCasting() { + let integerExpr = IntegerLiteralExprSyntax( + digits: .integerLiteral("1", trailingTrivia: .space) + ) + + let expr = ExprSyntax(integerExpr) + let node = Syntax(expr) + XCTAssertTrue(expr.is(IntegerLiteralExprSyntax.self)) + XCTAssertTrue(node.is(IntegerLiteralExprSyntax.self)) + XCTAssertTrue(node.as(ExprSyntax.self)!.is(IntegerLiteralExprSyntax.self)) + + XCTAssertTrue(node.isProtocol(ExprSyntaxProtocol.self)) + XCTAssertTrue(node.asProtocol(ExprSyntaxProtocol.self) is IntegerLiteralExprSyntax) + XCTAssertTrue(expr.asProtocol(ExprSyntaxProtocol.self) is IntegerLiteralExprSyntax) + XCTAssertTrue(expr.asProtocol(ExprSyntaxProtocol.self) as? IntegerLiteralExprSyntax == integerExpr) + + XCTAssertFalse(node.isProtocol(BracedSyntax.self)) + XCTAssertNil(node.asProtocol(BracedSyntax.self)) + XCTAssertFalse(expr.isProtocol(BracedSyntax.self)) + XCTAssertNil(expr.asProtocol(BracedSyntax.self)) + + let classDecl = CodeBlockSyntax( + leftBrace: TokenSyntax.leftBraceToken(), + statements: CodeBlockItemListSyntax([]), + rightBrace: TokenSyntax.rightBraceToken() + ) + + XCTAssertTrue(classDecl.isProtocol(BracedSyntax.self)) + XCTAssertNotNil(classDecl.asProtocol(BracedSyntax.self)) + + let optNode: Syntax? = node + switch optNode?.as(SyntaxEnum.self) { + case .integerLiteralExpr: break + default: XCTFail("failed to convert to SyntaxEnum") + } + + XCTAssertNil(ExprSyntax(nil as IntegerLiteralExprSyntax?)) + XCTAssertEqual(ExprSyntax(integerExpr).as(IntegerLiteralExprSyntax.self)!, integerExpr) + } + + public func testNodeType() { + let integerExpr = IntegerLiteralExprSyntax( + digits: TokenSyntax.integerLiteral("1", trailingTrivia: .space) + ) + let expr = ExprSyntax(integerExpr) + let node = Syntax(expr) + + XCTAssertTrue(integerExpr.syntaxNodeType == expr.syntaxNodeType) + XCTAssertTrue(integerExpr.syntaxNodeType == node.syntaxNodeType) + XCTAssertEqual("\(integerExpr.syntaxNodeType)", "IntegerLiteralExprSyntax") + } + + public func testConstructFromSyntaxProtocol() { + let integerExpr = IntegerLiteralExprSyntax( + digits: .integerLiteral("1", trailingTrivia: .space) + ) + + XCTAssertEqual(Syntax(integerExpr), Syntax(fromProtocol: integerExpr as SyntaxProtocol)) + XCTAssertEqual(Syntax(integerExpr), Syntax(fromProtocol: integerExpr as ExprSyntaxProtocol)) + } + + public func testPositions() { + let leading = Trivia(pieces: [.spaces(2)]) + let trailing = Trivia(pieces: [.spaces(1)]) + let funcKW = TokenSyntax.keyword( + .func, + leadingTrivia: leading, + trailingTrivia: trailing + ) + XCTAssertEqual("\(funcKW)", " func ") + XCTAssertEqual(funcKW.position, AbsolutePosition(utf8Offset: 0)) + XCTAssertEqual(funcKW.positionAfterSkippingLeadingTrivia, AbsolutePosition(utf8Offset: 2)) + XCTAssertEqual(funcKW.endPositionBeforeTrailingTrivia, AbsolutePosition(utf8Offset: 6)) + XCTAssertEqual(funcKW.endPosition, AbsolutePosition(utf8Offset: 7)) + XCTAssertEqual(funcKW.contentLength, SourceLength(utf8Length: 4)) + } } diff --git a/Tests/SwiftSyntaxTest/SyntaxVisitorTests.swift b/Tests/SwiftSyntaxTest/SyntaxVisitorTests.swift new file mode 100644 index 00000000000..b408b1aa073 --- /dev/null +++ b/Tests/SwiftSyntaxTest/SyntaxVisitorTests.swift @@ -0,0 +1,240 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import XCTest +import SwiftSyntax +import SwiftSyntaxBuilder +import _SwiftSyntaxTestSupport + +public class SyntaxVisitorTests: XCTestCase { + /// The syntax tree of the following source file + /// ```swift + /// func foo() { + /// public func foo() { + /// func foo() { + /// /*Unknown token */0xG + /// } + /// } + /// } + /// ``` + /// + /// The source file is hard-coded so this test case doesn't need to depend on the parser. + static var nestedFunctionsFile: SourceFileSyntax { + let source = SourceFileSyntax( + statements: CodeBlockItemListSyntax([ + CodeBlockItemSyntax( + item: CodeBlockItemSyntax.Item( + FunctionDeclSyntax( + funcKeyword: .keyword(.func, trailingTrivia: .space), + identifier: .identifier("foo"), + signature: FunctionSignatureSyntax( + input: ParameterClauseSyntax( + leftParen: .leftParenToken(), + parameterList: FunctionParameterListSyntax([]), + rightParen: .rightParenToken(trailingTrivia: .space) + ) + ), + body: CodeBlockSyntax( + leftBrace: .leftBraceToken(), + statements: CodeBlockItemListSyntax([ + CodeBlockItemSyntax( + item: CodeBlockItemSyntax.Item( + FunctionDeclSyntax( + modifiers: ModifierListSyntax([ + DeclModifierSyntax(name: .keyword(.public, leadingTrivia: [.newlines(1), .spaces(2)], trailingTrivia: .space)) + ]), + funcKeyword: .keyword(.func, trailingTrivia: .space), + identifier: .identifier("foo"), + signature: FunctionSignatureSyntax( + input: ParameterClauseSyntax( + leftParen: .leftParenToken(), + parameterList: FunctionParameterListSyntax([]), + rightParen: .rightParenToken(trailingTrivia: .space) + ) + ), + body: CodeBlockSyntax( + leftBrace: .leftBraceToken(), + statements: CodeBlockItemListSyntax([ + CodeBlockItemSyntax( + item: CodeBlockItemSyntax.Item( + FunctionDeclSyntax( + funcKeyword: .keyword(.func, leadingTrivia: [.newlines(1), .spaces(4)], trailingTrivia: .space), + identifier: .identifier("foo"), + signature: FunctionSignatureSyntax( + input: ParameterClauseSyntax( + leftParen: .leftParenToken(), + parameterList: FunctionParameterListSyntax([]), + rightParen: .rightParenToken(trailingTrivia: .space) + ) + ), + body: CodeBlockSyntax( + leftBrace: .leftBraceToken(), + statements: CodeBlockItemListSyntax([ + CodeBlockItemSyntax( + item: CodeBlockItemSyntax.Item( + IntegerLiteralExprSyntax( + digits: .integerLiteral( + "0xG", + leadingTrivia: [ + .newlines(1), + .spaces(6), + .blockComment("/*Unknown token */"), + ] + ) + ) + ) + ) + ]), + rightBrace: .rightBraceToken(leadingTrivia: [.newlines(1), .spaces(4)]) + ) + ) + ) + ) + ]), + rightBrace: .rightBraceToken(leadingTrivia: [.newlines(1), .spaces(2)]) + ) + ) + ) + ) + ]), + rightBrace: .rightBraceToken(leadingTrivia: .newline) + ) + ) + ) + ) + ]) + ) + + // Make sure we are indeed generating a syntax tree for the expected source file. + precondition( + source.description + == """ + func foo() { + public func foo() { + func foo() { + /*Unknown token */0xG + } + } + } + """ + ) + return source + } + + public func testBasic() { + class FuncCounter: SyntaxVisitor { + var funcCount = 0 + override func visit(_ node: FunctionDeclSyntax) -> SyntaxVisitorContinueKind { + funcCount += 1 + return .visitChildren + } + } + let parsed = Self.nestedFunctionsFile + let counter = FuncCounter(viewMode: .fixedUp) + let hashBefore = parsed.hashValue + counter.walk(parsed) + XCTAssertEqual(counter.funcCount, 3) + XCTAssertEqual(hashBefore, parsed.hashValue) + } + + public func testRewritingNodeWithEmptyChild() { + class ClosureRewriter: SyntaxRewriter { + override func visit(_ node: ClosureExprSyntax) -> ExprSyntax { + // Perform a no-op transform that requires rebuilding the node. + return ExprSyntax(node.with(\.statements, node.statements)) + } + } + let closure = ClosureExprSyntax( + statements: CodeBlockItemListSyntax([]) + ) + let rewriter = ClosureRewriter() + let rewritten = rewriter.visit(closure) + XCTAssertEqual(closure.description, rewritten.description) + } + + public func testSyntaxRewriterVisitAny() { + class VisitAnyRewriter: SyntaxRewriter { + let transform: (TokenSyntax) -> TokenSyntax + init(transform: @escaping (TokenSyntax) -> TokenSyntax) { + self.transform = transform + } + override func visitAny(_ node: Syntax) -> Syntax? { + if let tok = node.as(TokenSyntax.self) { + return Syntax(transform(tok)) + } + return nil + } + } + let parsed = IdentifierExprSyntax(identifier: .identifier("n")) + let rewriter = VisitAnyRewriter(transform: { _ in + return TokenSyntax.identifier("") + }) + let rewritten = rewriter.visit(parsed) + XCTAssertEqual(rewritten.description, "") + } + + public func testSyntaxRewriterVisitCollection() { + class VisitCollections: SyntaxVisitor { + var numberOfCodeBlockItems = 0 + + override func visit(_ items: CodeBlockItemListSyntax) -> SyntaxVisitorContinueKind { + numberOfCodeBlockItems += items.count + return .visitChildren + } + } + + let visitor = VisitCollections(viewMode: .fixedUp) + visitor.walk(Self.nestedFunctionsFile) + XCTAssertEqual(4, visitor.numberOfCodeBlockItems) + } + + public func testVisitorClass() { + class FuncCounter: SyntaxVisitor { + var funcCount = 0 + override func visit(_ node: FunctionDeclSyntax) -> SyntaxVisitorContinueKind { + funcCount += 1 + return super.visit(node) + } + } + let sourceFile = Self.nestedFunctionsFile + let counter = FuncCounter(viewMode: .fixedUp) + let hashBefore = sourceFile.hashValue + counter.walk(sourceFile) + XCTAssertEqual(counter.funcCount, 3) + XCTAssertEqual(hashBefore, sourceFile.hashValue) + } + + public func testRewriteTrivia() { + class TriviaRemover: SyntaxRewriter { + override func visit(_ token: TokenSyntax) -> TokenSyntax { + return token.with(\.trailingTrivia, []) + } + } + + let source = VariableDeclSyntax( + bindingKeyword: .keyword(.let, trailingTrivia: .space), + bindings: PatternBindingListSyntax([ + PatternBindingSyntax( + pattern: PatternSyntax(IdentifierPatternSyntax(identifier: .identifier("a", trailingTrivia: .space))), + initializer: InitializerClauseSyntax( + equal: .equalToken(trailingTrivia: .space), + value: ExprSyntax(IntegerLiteralExprSyntax(digits: .integerLiteral("5"))) + ) + ) + ]) + ) + XCTAssertEqual(source.description, "let a = 5") + let visitor = TriviaRemover() + let rewritten = visitor.visit(source) + XCTAssertEqual(rewritten.description, "leta=5") + } +} diff --git a/Tests/SwiftSyntaxParserTest/SyntaxComparisonTests.swift b/Tests/SwiftSyntaxTestSupportTest/SyntaxComparisonTests.swift similarity index 92% rename from Tests/SwiftSyntaxParserTest/SyntaxComparisonTests.swift rename to Tests/SwiftSyntaxTestSupportTest/SyntaxComparisonTests.swift index d6d55a0ca6c..89f34517bd2 100644 --- a/Tests/SwiftSyntaxParserTest/SyntaxComparisonTests.swift +++ b/Tests/SwiftSyntaxTestSupportTest/SyntaxComparisonTests.swift @@ -10,13 +10,13 @@ // //===----------------------------------------------------------------------===// -import SwiftSyntax import _SwiftSyntaxTestSupport -import SwiftSyntaxParser +import SwiftSyntax +import SwiftParser import XCTest -private func parse(source: String) throws -> Syntax { - return try Syntax(SyntaxParser.parse(source: source)) +private func parse(source: String) -> Syntax { + return Syntax(Parser.parse(source: source)) } public class SyntaxComparisonTests: XCTestCase { @@ -26,7 +26,7 @@ public class SyntaxComparisonTests: XCTestCase { let actual = Syntax(makeFunc(identifier: .identifier("f"))) XCTAssertNil(actual.findFirstDifference(baseline: expected)) - let matcher = try SubtreeMatcher("struct A { func f() { } }", parse: parse) + let matcher = SubtreeMatcher("struct A { func f() { } }", parse: parse) try XCTAssertNil(matcher.findFirstDifference(baseline: expected)) } @@ -53,7 +53,7 @@ public class SyntaxComparisonTests: XCTestCase { let actual = Syntax(makeFunc(identifier: .identifier("f"))) try expectations(actual.findFirstDifference(baseline: expected)) - let matcher = try SubtreeMatcher("struct A { 1️⃣func f() { } }", parse: parse) + let matcher = SubtreeMatcher("struct A { 1️⃣func f() { } }", parse: parse) try expectations(matcher.findFirstDifference(baseline: expected)) } @@ -69,7 +69,7 @@ public class SyntaxComparisonTests: XCTestCase { let actual = Syntax(makeFunc(identifier: .identifier("g"))) try expectations(actual.findFirstDifference(baseline: expected)) - let matcher = try SubtreeMatcher("struct A { 1️⃣func g() { } }", parse: parse) + let matcher = SubtreeMatcher("struct A { 1️⃣func g() { } }", parse: parse) try expectations(matcher.findFirstDifference(afterMarker: "1️⃣", baseline: expected)) } @@ -86,7 +86,7 @@ public class SyntaxComparisonTests: XCTestCase { XCTAssertNil(actual.findFirstDifference(baseline: expected)) try expectations(actual.findFirstDifference(baseline: expected, includeTrivia: true)) - let matcher = try SubtreeMatcher("struct A {func f() { }}", parse: parse) + let matcher = SubtreeMatcher("struct A {func f() { }}", parse: parse) try XCTAssertNil(matcher.findFirstDifference(baseline: expected)) try expectations(matcher.findFirstDifference(baseline: expected, includeTrivia: true)) } @@ -112,7 +112,7 @@ public class SyntaxComparisonTests: XCTestCase { let actual = Syntax(makeFunc(identifier: .identifier("f"))) try expectations(actual.findFirstDifference(baseline: expected)) - let matcher = try SubtreeMatcher("struct A { func f() { } }", parse: parse) + let matcher = SubtreeMatcher("struct A { func f() { } }", parse: parse) try expectations(matcher.findFirstDifference(baseline: expected)) } @@ -126,7 +126,7 @@ public class SyntaxComparisonTests: XCTestCase { let actual = Syntax(makeFunc(identifier: .identifier("f"))) try expectations(actual.findFirstDifference(baseline: expected)) - let matcher = try SubtreeMatcher("struct A { func f() { } }", parse: parse) + let matcher = SubtreeMatcher("struct A { func f() { } }", parse: parse) try expectations(matcher.findFirstDifference(baseline: expected)) } @@ -140,7 +140,7 @@ public class SyntaxComparisonTests: XCTestCase { let actual = Syntax(makeFunc(identifier: .identifier("f"), body: makeBody(statementCount: 1))) try expectations(actual.findFirstDifference(baseline: expected)) - let matcher = try SubtreeMatcher( + let matcher = SubtreeMatcher( """ struct A { func f() { @@ -157,7 +157,7 @@ public class SyntaxComparisonTests: XCTestCase { let expectedFunc = Syntax(makeFunc(identifier: .identifier("f"))) let expectedBody = Syntax(makeBody()) - let matcher = try SubtreeMatcher( + let matcher = SubtreeMatcher( """ struct A { 1️⃣ diff --git a/build-script.py b/build-script.py index 7e8daa5549d..0a96b7e66d1 100755 --- a/build-script.py +++ b/build-script.py @@ -482,7 +482,6 @@ def build_command(args: argparse.Namespace) -> None: # Until rdar://53881101 is implemented, we cannot request a build of multiple # targets simultaneously. For now, just build one product after the other. builder.buildProduct("SwiftSyntax") - builder.buildProduct("SwiftSyntaxParser") builder.buildProduct("SwiftSyntaxBuilder") # Build examples