Skip to content

Fix multiple cases where the parser produced token kinds that didn’t match the token kinds in the syntax node definitions #1436

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 29, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions CodeGeneration/Sources/SyntaxSupport/AttributeNodes.swift
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,7 @@ public let ATTRIBUTE_NODES: [Node] = [
children: [
Child(
name: "DeclBaseName",
kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "PrefixOperatorToken"), .keyword(text: "init")]),
kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "BinaryOperatorToken"), .keyword(text: "init"), .keyword(text: "self"), .keyword(text: "Self")]),
nameForDiagnostics: "base name",
description: "The base name of the protocol's requirement."
),
Expand Down Expand Up @@ -468,7 +468,7 @@ public let ATTRIBUTE_NODES: [Node] = [
children: [
Child(
name: "DiffKind",
kind: .token(choices: [.keyword(text: "forward"), .keyword(text: "reverse"), .keyword(text: "linear")]),
kind: .token(choices: [.keyword(text: "_forward"), .keyword(text: "reverse"), .keyword(text: "_linear")]),
isOptional: true
),
Child(
Expand Down Expand Up @@ -644,7 +644,7 @@ public let ATTRIBUTE_NODES: [Node] = [
children: [
Child(
name: "Label",
kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .keyword(text: "available"), .keyword(text: "exported"), .keyword(text: "kind"), .keyword(text: "spi"), .keyword(text: "spiModule")]),
kind: .node(kind: "Token"),
nameForDiagnostics: "label",
description: "The label of the argument"
),
Expand Down Expand Up @@ -677,7 +677,7 @@ public let ATTRIBUTE_NODES: [Node] = [
children: [
Child(
name: "Name",
kind: .token(choices: [.token(tokenKind: "IdentifierToken")]),
kind: .node(kind: "Token"),
nameForDiagnostics: "name",
isOptional: true
),
Expand Down Expand Up @@ -779,7 +779,7 @@ public let ATTRIBUTE_NODES: [Node] = [
),
Child(
name: "Name",
kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "BinaryOperatorToken"), .token(tokenKind: "PrefixOperatorToken"), .token(tokenKind: "PostfixOperatorToken")]),
kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .keyword(text: "self"), .keyword(text: "Self"), .keyword(text: "init"), .token(tokenKind: "BinaryOperatorToken")]),
nameForDiagnostics: "base name",
description: "The base name of the referenced function."
),
Expand Down
2 changes: 1 addition & 1 deletion CodeGeneration/Sources/SyntaxSupport/DeclNodes.swift
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ public let DECL_NODES: [Node] = [
children: [
Child(
name: "Name",
kind: .token(choices: [.token(tokenKind: "IdentifierToken")]),
kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "BinaryOperatorToken"), .token(tokenKind: "PrefixOperatorToken"), .token(tokenKind: "PostfixOperatorToken")]),
nameForDiagnostics: "name"
),
Child(
Expand Down
6 changes: 4 additions & 2 deletions Sources/SwiftParser/Attributes.swift
Original file line number Diff line number Diff line change
Expand Up @@ -575,11 +575,12 @@ extension Parser {
let (unexpectedBetweenOfLabelAndColon, colon) = self.expect(.colon)
let originalDeclName = self.parseQualifiedDeclarationName()
let period = self.consume(if: .period)
let unexpectedBeforeAccessor: RawUnexpectedNodesSyntax?
let accessor: RawTokenSyntax?
if period != nil {
accessor = self.parseAnyIdentifier()
(unexpectedBeforeAccessor, accessor) = self.expect(.keyword(.get), .keyword(.set), default: .keyword(.get))
} else {
accessor = nil
(unexpectedBeforeAccessor, accessor) = (nil, nil)
}
let comma = self.consume(if: .comma)
let diffParams: RawDifferentiabilityParamsClauseSyntax?
Expand All @@ -595,6 +596,7 @@ extension Parser {
colon: colon,
originalDeclName: originalDeclName,
period: period,
unexpectedBeforeAccessor,
accessorKind: accessor,
comma: comma,
diffParams: diffParams,
Expand Down
10 changes: 5 additions & 5 deletions Sources/SwiftParser/Declarations.swift
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,7 @@ extension Parser {
// Parse the 'each' keyword for a type parameter pack 'each T'.
var each = self.consume(if: .keyword(.each))

let (unexpectedBetweenEachAndName, name) = self.expectIdentifier()
let (unexpectedBetweenEachAndName, name) = self.expectIdentifier(allowSelfOrCapitalSelfAsIdentifier: true)
if attributes == nil && each == nil && unexpectedBetweenEachAndName == nil && name.isMissing && elements.isEmpty {
break
}
Expand Down Expand Up @@ -631,7 +631,7 @@ extension Parser {
body: .sameTypeRequirement(
RawSameTypeRequirementSyntax(
leftTypeIdentifier: RawTypeSyntax(RawMissingTypeSyntax(arena: self.arena)),
equalityToken: missingToken(.equal),
equalityToken: missingToken(.binaryOperator, text: "=="),
rightTypeIdentifier: RawTypeSyntax(RawMissingTypeSyntax(arena: self.arena)),
arena: self.arena
)
Expand Down Expand Up @@ -890,7 +890,7 @@ extension Parser {
var loopProgress = LoopProgressCondition()
repeat {
let unexpectedPeriod = self.consume(if: .period)
let (unexpectedBeforeName, name) = self.expectIdentifier(allowIdentifierLikeKeywords: false, keywordRecovery: true)
let (unexpectedBeforeName, name) = self.expectIdentifier(keywordRecovery: true)

let associatedValue: RawParameterClauseSyntax?
if self.at(TokenSpec(.leftParen, allowAtStartOfLine: false)) {
Expand Down Expand Up @@ -1916,7 +1916,7 @@ extension Parser {
// checking.
let precedenceAndTypes: RawOperatorPrecedenceAndTypesSyntax?
if let colon = self.consume(if: .colon) {
let (unexpectedBeforeIdentifier, identifier) = self.expectIdentifier(keywordRecovery: true)
let (unexpectedBeforeIdentifier, identifier) = self.expectIdentifier(allowSelfOrCapitalSelfAsIdentifier: true)
var types = [RawDesignatedTypeElementSyntax]()
while let comma = self.consume(if: .comma) {
// FIXME: The compiler accepts... anything here. This is a bug.
Expand Down Expand Up @@ -2001,7 +2001,7 @@ extension Parser {
_ handle: RecoveryConsumptionHandle
) -> RawPrecedenceGroupDeclSyntax {
let (unexpectedBeforeGroup, group) = self.eat(handle)
let (unexpectedBeforeIdentifier, identifier) = self.expectIdentifier(keywordRecovery: true)
let (unexpectedBeforeIdentifier, identifier) = self.expectIdentifier(allowSelfOrCapitalSelfAsIdentifier: true)
let (unexpectedBeforeLBrace, lbrace) = self.expect(.leftBrace)

let groupAttributes = self.parsePrecedenceGroupAttributeListSyntax()
Expand Down
2 changes: 1 addition & 1 deletion Sources/SwiftParser/Expressions.swift
Original file line number Diff line number Diff line change
Expand Up @@ -1796,7 +1796,7 @@ extension Parser {
let expression: RawExprSyntax
if self.peek().rawTokenKind == .equal {
// The name is a new declaration.
(unexpectedBeforeName, name) = self.expectIdentifier()
(unexpectedBeforeName, name) = self.expect(.identifier, TokenSpec(.self, remapping: .identifier), default: .identifier)
(unexpectedBeforeAssignToken, assignToken) = self.expect(.equal)
expression = self.parseExpression()
} else {
Expand Down
2 changes: 1 addition & 1 deletion Sources/SwiftParser/Lookahead.swift
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ extension Parser.Lookahead {
while let _ = self.consume(if: .atSign) {
// Consume qualified names that may or may not involve generic arguments.
repeat {
self.expectIdentifierOrRethrowsWithoutRecovery()
self.consume(if: .identifier, .keyword(.rethrows))
// We don't care whether this succeeds or fails to eat generic
// parameters.
_ = self.consumeGenericArguments()
Expand Down
18 changes: 10 additions & 8 deletions Sources/SwiftParser/Names.swift
Original file line number Diff line number Diff line change
Expand Up @@ -63,20 +63,20 @@ extension Parser {

mutating func parseDeclNameRef(_ flags: DeclNameOptions = []) -> (RawTokenSyntax, RawDeclNameArgumentsSyntax?) {
// Consume the base name.
let ident: RawTokenSyntax
if self.at(.identifier) || self.at(.keyword(.self), .keyword(.Self), .keyword(.`init`)) {
ident = self.expectIdentifierWithoutRecovery()
let base: RawTokenSyntax
if let identOrSelf = self.consume(if: .identifier, .keyword(.self), .keyword(.Self)) ?? self.consume(if: .keyword(.`init`)) {
base = identOrSelf
} else if flags.contains(.operators), let (_, _) = self.at(anyIn: Operator.self) {
ident = self.consumeAnyToken(remapping: .binaryOperator)
base = self.consumeAnyToken(remapping: .binaryOperator)
} else if flags.contains(.keywords) && self.currentToken.isLexerClassifiedKeyword {
ident = self.consumeAnyToken(remapping: .identifier)
base = self.consumeAnyToken(remapping: .identifier)
} else {
ident = self.expectIdentifierWithoutRecovery()
base = missingToken(.identifier)
}

// Parse an argument list, if the flags allow it and it's present.
let args = self.parseArgLabelList(flags)
return (ident, args)
return (base, args)
}

mutating func parseArgLabelList(_ flags: DeclNameOptions) -> RawDeclNameArgumentsSyntax? {
Expand Down Expand Up @@ -176,7 +176,7 @@ extension Parser {
var keepGoing: RawTokenSyntax? = nil
var loopProgress = LoopProgressCondition()
repeat {
let (name, _) = self.parseDeclNameRef()
let (unexpectedBeforeName, name) = self.expect(.identifier, .keyword(.self), .keyword(.Self), default: .identifier)
let generics: RawGenericArgumentClauseSyntax?
if self.atContextualPunctuator("<") {
generics = self.parseGenericArguments()
Expand All @@ -188,6 +188,7 @@ extension Parser {
RawMemberTypeIdentifierSyntax(
baseType: result!,
period: keepGoing,
unexpectedBeforeName,
name: name,
genericArgumentClause: generics,
arena: self.arena
Expand All @@ -196,6 +197,7 @@ extension Parser {
} else {
result = RawTypeSyntax(
RawSimpleTypeIdentifierSyntax(
unexpectedBeforeName,
name: name,
genericArgumentClause: generics,
arena: self.arena
Expand Down
4 changes: 2 additions & 2 deletions Sources/SwiftParser/Nominals.swift
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ extension Parser {
introucerHandle: RecoveryConsumptionHandle
) -> T where T: NominalTypeDeclarationTrait {
let (unexpectedBeforeIntroducerKeyword, introducerKeyword) = self.eat(introucerHandle)
let (unexpectedBeforeName, name) = self.expectIdentifier(allowIdentifierLikeKeywords: false, keywordRecovery: true)
let (unexpectedBeforeName, name) = self.expectIdentifier(keywordRecovery: true)
if unexpectedBeforeName == nil && name.isMissing && self.currentToken.isAtStartOfLine {
return T.init(
attributes: attrs.attributes,
Expand Down Expand Up @@ -258,7 +258,7 @@ extension Parser {
var loopProgress = LoopProgressCondition()
repeat {
// Parse the name of the parameter.
let (unexpectedBeforeName, name) = self.expectIdentifier()
let (unexpectedBeforeName, name) = self.expectIdentifier(allowSelfOrCapitalSelfAsIdentifier: true)
keepGoing = self.consume(if: .comma)
associatedTypes.append(
RawPrimaryAssociatedTypeSyntax(
Expand Down
38 changes: 12 additions & 26 deletions Sources/SwiftParser/Parser.swift
Original file line number Diff line number Diff line change
Expand Up @@ -417,18 +417,20 @@ extension Parser {
/// incorrectly used a keyword as an identifier.
/// This should be set if keywords aren't strong recovery marker at this
/// position, e.g. because the parser expects a punctuator next.
///
/// If `allowSelfOrCapitalSelfAsIdentifier` is `true`, then `self` and `Self`
/// are also accepted and remapped to identifiers. This is exclusively used
/// to maintain compatibility with the C++ parser. No new uses of this should
/// be introduced.
mutating func expectIdentifier(
allowIdentifierLikeKeywords: Bool = true,
keywordRecovery: Bool = false
keywordRecovery: Bool = false,
allowSelfOrCapitalSelfAsIdentifier: Bool = false
) -> (RawUnexpectedNodesSyntax?, RawTokenSyntax) {
if allowIdentifierLikeKeywords {
if let (_, handle) = self.canRecoverTo(anyIn: IdentifierTokens.self) {
return self.eat(handle)
}
} else {
if let identifier = self.consume(if: .identifier) {
return (nil, identifier)
}
if let identifier = self.consume(if: .identifier) {
return (nil, identifier)
}
if allowSelfOrCapitalSelfAsIdentifier, let selfOrCapitalSelf = self.consume(if: TokenSpec(.self, remapping: .identifier), TokenSpec(.Self, remapping: .identifier)) {
return (nil, selfOrCapitalSelf)
}
if let unknown = self.consume(if: .unknown) {
return (
Expand Down Expand Up @@ -457,22 +459,6 @@ extension Parser {
)
}

mutating func expectIdentifierOrRethrows() -> (RawUnexpectedNodesSyntax?, RawTokenSyntax) {
if let (_, handle) = self.canRecoverTo(anyIn: IdentifierOrRethrowsTokens.self) {
return self.eat(handle)
}
if let unknown = self.consume(if: .unknown) {
return (
RawUnexpectedNodesSyntax(elements: [RawSyntax(unknown)], arena: self.arena),
self.missingToken(.identifier)
)
}
return (
nil,
self.missingToken(.identifier)
)
}

/// Expect a right brace but with a little smart recovery logic:
/// If `leftBrace` is missing, we only recover to a `}` whose indentation is greater or equal to that of `introducer`.
/// That way, if the developer forgot to to type `{`, we won't eat `}` that were most likely intended to close an outer scope.
Expand Down
2 changes: 1 addition & 1 deletion Sources/SwiftParser/Statements.swift
Original file line number Diff line number Diff line change
Expand Up @@ -932,7 +932,7 @@ extension Parser {
return nil
}

return self.expectIdentifierWithoutRecovery()
return self.expectWithoutRecovery(.identifier)
}
}

Expand Down
16 changes: 0 additions & 16 deletions Sources/SwiftParser/TokenConsumer.swift
Original file line number Diff line number Diff line change
Expand Up @@ -219,22 +219,6 @@ extension TokenConsumer {
// MARK: Convenience functions

extension TokenConsumer {
@inline(__always)
mutating func expectIdentifierWithoutRecovery() -> Token {
if let (_, handle) = self.at(anyIn: IdentifierTokens.self) {
return self.eat(handle)
}
return missingToken(.identifier)
}

@inline(__always)
mutating func expectIdentifierOrRethrowsWithoutRecovery() -> Token {
if let (_, handle) = self.at(anyIn: IdentifierOrRethrowsTokens.self) {
return self.eat(handle)
}
return missingToken(.identifier)
}

var canHaveParameterSpecifier: Bool {
// The parameter specifiers like `isolated`, `consuming`, `borrowing` are
// also valid identifiers and could be the name of a type. Check whether
Expand Down
58 changes: 0 additions & 58 deletions Sources/SwiftParser/TokenSpecSet.swift
Original file line number Diff line number Diff line change
Expand Up @@ -298,64 +298,6 @@ enum DeclarationStart: TokenSpecSet {
}
}

enum IdentifierTokens: TokenSpecSet {
case anyKeyword
case capitalSelfKeyword
case identifier
case initKeyword
case selfKeyword

init?(lexeme: Lexer.Lexeme) {
switch PrepareForKeywordMatch(lexeme) {
case TokenSpec(.Any): self = .anyKeyword
case TokenSpec(.Self): self = .capitalSelfKeyword
case TokenSpec(.identifier): self = .identifier
case TokenSpec(.`init`): self = .initKeyword
case TokenSpec(.self): self = .selfKeyword
default: return nil
}
}

var spec: TokenSpec {
switch self {
case .anyKeyword: return .keyword(.Any)
case .capitalSelfKeyword: return .keyword(.Self)
case .identifier: return .identifier
case .initKeyword: return .keyword(.`init`)
case .selfKeyword: return .keyword(.self)
}
}
}

enum IdentifierOrRethrowsTokens: TokenSpecSet {
case anyKeyword
case capitalSelfKeyword
case identifier
case selfKeyword
case rethrowsKeyword

init?(lexeme: Lexer.Lexeme) {
switch PrepareForKeywordMatch(lexeme) {
case TokenSpec(.Any): self = .anyKeyword
case TokenSpec(.Self): self = .capitalSelfKeyword
case TokenSpec(.identifier): self = .identifier
case TokenSpec(.self): self = .selfKeyword
case TokenSpec(.rethrows): self = .rethrowsKeyword
default: return nil
}
}

var spec: TokenSpec {
switch self {
case .anyKeyword: return .keyword(.Any)
case .capitalSelfKeyword: return .keyword(.Self)
case .identifier: return .identifier
case .selfKeyword: return .keyword(.self)
case .rethrowsKeyword: return TokenSpec(.rethrows, remapping: .identifier)
}
}
}

enum Operator: TokenSpecSet {
case binaryOperator
case postfixOperator
Expand Down
15 changes: 11 additions & 4 deletions Sources/SwiftParser/Types.swift
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ extension Parser {
)
)
} else {
let (name, generics) = self.parseTypeNameWithGenerics(include: .keywords)
let (name, generics) = self.parseTypeNameWithGenerics(allowKeywordAsName: true)
base = RawTypeSyntax(
RawMemberTypeIdentifierSyntax(
baseType: base,
Expand Down Expand Up @@ -337,8 +337,15 @@ extension Parser {
)
}

mutating func parseTypeNameWithGenerics(include flags: DeclNameOptions = []) -> (RawTokenSyntax, RawGenericArgumentClauseSyntax?) {
let (name, _) = self.parseDeclNameRef(flags)
mutating func parseTypeNameWithGenerics(allowKeywordAsName: Bool) -> (RawTokenSyntax, RawGenericArgumentClauseSyntax?) {
let name: RawTokenSyntax
if let identOrSelf = self.consume(if: .identifier, .keyword(.self), .keyword(.Self)) {
name = identOrSelf
} else if allowKeywordAsName && self.currentToken.isLexerClassifiedKeyword {
name = self.consumeAnyToken(remapping: .identifier)
} else {
name = missingToken(.identifier)
}
if self.atContextualPunctuator("<") {
return (name, self.parseGenericArguments())
}
Expand All @@ -356,7 +363,7 @@ extension Parser {
return RawTypeSyntax(self.parseAnyType())
}

let (name, generics) = parseTypeNameWithGenerics()
let (name, generics) = parseTypeNameWithGenerics(allowKeywordAsName: false)
return RawTypeSyntax(
RawSimpleTypeIdentifierSyntax(
name: name,
Expand Down
Loading