diff --git a/Examples/Package.swift b/Examples/Package.swift index 4941330a261..86efecc5940 100644 --- a/Examples/Package.swift +++ b/Examples/Package.swift @@ -10,6 +10,7 @@ let package = Package( products: [ .executable(name: "AddOneToIntegerLiterals", targets: ["AddOneToIntegerLiterals"]), .executable(name: "CodeGenerationUsingSwiftSyntaxBuilder", targets: ["CodeGenerationUsingSwiftSyntaxBuilder"]), + .executable(name: "ExamplePlugin", targets: ["ExamplePlugin"]), ], dependencies: [ .package(path: "../") @@ -20,17 +21,22 @@ let package = Package( dependencies: [ .product(name: "SwiftParser", package: "swift-syntax"), .product(name: "SwiftSyntax", package: "swift-syntax"), - ], - path: ".", - exclude: ["README.md", "CodeGenerationUsingSwiftSyntaxBuilder.swift"] + ] ), .executableTarget( name: "CodeGenerationUsingSwiftSyntaxBuilder", dependencies: [ .product(name: "SwiftSyntaxBuilder", package: "swift-syntax") - ], - path: ".", - exclude: ["README.md", "AddOneToIntegerLiterals.swift"] + ] + ), + .executableTarget( + name: "ExamplePlugin", + dependencies: [ + .product(name: "SwiftCompilerPlugin", package: "swift-syntax"), + .product(name: "SwiftSyntax", package: "swift-syntax"), + .product(name: "SwiftSyntaxMacros", package: "swift-syntax"), + .product(name: "SwiftDiagnostics", package: "swift-syntax"), + ] ), ] ) diff --git a/Examples/README.md b/Examples/README.md index 916e123239d..8c048d89f48 100644 --- a/Examples/README.md +++ b/Examples/README.md @@ -2,8 +2,9 @@ Each example can be executed by navigating into this folder and running `swift run `. There is the following set of examples available: -- [AddOneToIntegerLiterals](AddOneToIntegerLiterals.swift): Command line tool to add 1 to every integer literal in a source file -- [CodeGenerationUsingSwiftSyntaxBuilder](CodeGenerationUsingSwiftSyntaxBuilder.swift): Code-generate a simple source file using SwiftSyntaxBuilder +- [AddOneToIntegerLiterals](Sources/AddOneToIntegerLiterals/AddOneToIntegerLiterals.swift): Command line tool to add 1 to every integer literal in a source file +- [CodeGenerationUsingSwiftSyntaxBuilder](Sources/CodeGenerationUsingSwiftSyntaxBuilder/CodeGenerationUsingSwiftSyntaxBuilder.swift): Code-generate a simple source file using SwiftSyntaxBuilder +- [ExamplePlugin](Sources/ExamplePlugn): Compiler plugin executable using [`SwiftCompilerPlugin`](../Sources/SwiftCompilerPlugin) ## Some Example Usages diff --git a/Examples/AddOneToIntegerLiterals.swift b/Examples/Sources/AddOneToIntegerLiterals/AddOneToIntegerLiterals.swift similarity index 100% rename from Examples/AddOneToIntegerLiterals.swift rename to Examples/Sources/AddOneToIntegerLiterals/AddOneToIntegerLiterals.swift diff --git a/Examples/CodeGenerationUsingSwiftSyntaxBuilder.swift b/Examples/Sources/CodeGenerationUsingSwiftSyntaxBuilder/CodeGenerationUsingSwiftSyntaxBuilder.swift similarity index 100% rename from Examples/CodeGenerationUsingSwiftSyntaxBuilder.swift rename to Examples/Sources/CodeGenerationUsingSwiftSyntaxBuilder/CodeGenerationUsingSwiftSyntaxBuilder.swift diff --git a/Examples/Sources/ExamplePlugin/ExamplePlugin.swift b/Examples/Sources/ExamplePlugin/ExamplePlugin.swift new file mode 100644 index 00000000000..f0b765147e3 --- /dev/null +++ b/Examples/Sources/ExamplePlugin/ExamplePlugin.swift @@ -0,0 +1,10 @@ +import SwiftCompilerPlugin +import SwiftSyntaxMacros + +@main +struct ThePlugin: CompilerPlugin { + var providingMacros: [Macro.Type] = [ + EchoExpressionMacro.self, + MetadataMacro.self, + ] +} diff --git a/Examples/Sources/ExamplePlugin/Macros.swift b/Examples/Sources/ExamplePlugin/Macros.swift new file mode 100644 index 00000000000..0509304c75e --- /dev/null +++ b/Examples/Sources/ExamplePlugin/Macros.swift @@ -0,0 +1,39 @@ +import SwiftSyntax +import SwiftSyntaxBuilder +import SwiftSyntaxMacros + +/// Returns the first argument prepending a comment '/* echo */'. +struct EchoExpressionMacro: ExpressionMacro { + static func expansion< + Node: FreestandingMacroExpansionSyntax, + Context: MacroExpansionContext + >( + of node: Node, + in context: Context + ) throws -> ExprSyntax { + let expr: ExprSyntax = node.argumentList.first!.expression + return expr.with(\.leadingTrivia, [.blockComment("/* echo */")]) + } +} + +/// Add a static property `__metadata__`. +struct MetadataMacro: MemberMacro { + static func expansion< + Declaration: DeclGroupSyntax, + Context: MacroExpansionContext + >( + of node: SwiftSyntax.AttributeSyntax, + providingMembersOf declaration: Declaration, + in context: Context + ) throws -> [DeclSyntax] { + guard let cls = declaration.as(ClassDeclSyntax.self) else { + return [] + } + let className = cls.identifier.trimmedDescription + return [ + """ + static var __metadata__: [String: String] { ["name": "\(raw: className)"] } + """ + ] + } +} diff --git a/Package.swift b/Package.swift index edc2353b0dc..715c0f66bfc 100644 --- a/Package.swift +++ b/Package.swift @@ -43,6 +43,7 @@ let package = Package( .library(name: "SwiftSyntaxParser", type: .static, targets: ["SwiftSyntaxParser"]), .library(name: "SwiftSyntaxBuilder", type: .static, targets: ["SwiftSyntaxBuilder"]), .library(name: "SwiftSyntaxMacros", type: .static, targets: ["SwiftSyntaxMacros"]), + .library(name: "SwiftCompilerPlugin", type: .static, targets: ["SwiftCompilerPlugin"]), .library(name: "SwiftRefactor", type: .static, targets: ["SwiftRefactor"]), ], targets: [ @@ -121,6 +122,12 @@ let package = Package( "CMakeLists.txt" ] ), + .target( + name: "SwiftCompilerPlugin", + dependencies: [ + "SwiftSyntax", "SwiftParser", "SwiftDiagnostics", "SwiftSyntaxMacros", "SwiftOperators", + ] + ), .target( name: "SwiftRefactor", dependencies: [ @@ -193,6 +200,12 @@ let package = Package( "SwiftRefactor", "SwiftSyntaxBuilder", "_SwiftSyntaxTestSupport", ] ), + .testTarget( + name: "SwiftCompilerPluginTest", + dependencies: [ + "SwiftCompilerPlugin" + ] + ), ] ) diff --git a/Sources/SwiftCompilerPlugin/CompilerPlugin.swift b/Sources/SwiftCompilerPlugin/CompilerPlugin.swift new file mode 100644 index 00000000000..2d15046b943 --- /dev/null +++ b/Sources/SwiftCompilerPlugin/CompilerPlugin.swift @@ -0,0 +1,246 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// NOTE: This basic plugin mechanism is mostly copied from +// https://github.com/apple/swift-package-manager/blob/main/Sources/PackagePlugin/Plugin.swift + +import SwiftSyntaxMacros + +@_implementationOnly import Foundation +#if os(Windows) +@_implementationOnly import ucrt +#endif + +// +// This source file contains the main entry point for compiler plugins. +// A plugin receives messages from the "plugin host" (typically +// 'swift-frontend'), and sends back messages in return based on its actions. +// +// Depending on the platform, plugins are invoked in a sanbox that blocks +// network access and prevents any file system changes. +// +// The host process and the plugin communicate using messages in the form of +// length-prefixed JSON-encoded Swift enums. The host sends messages to the +// plugin through its standard-input pipe, and receives messages through the +// plugin's standard-output pipe. The plugin's standard-error is considered +// to be free-form textual console output. +// +// Within the plugin process, `stdout` is redirected to `stderr` so that print +// statements from the plugin are treated as plain-text output, and `stdin` is +// closed so that any attemps by the plugin logic to read from console result +// in errors instead of blocking the process. The original `stdin` and `stdout` +// are duplicated for use as messaging pipes, and are not directly used by the +// plugin logic. +// +// The exit code of the plugin process indicates whether the plugin invocation +// is considered successful. A failure result should also be accompanied by an +// emitted error diagnostic, so that errors are understandable by the user. +// +// Using standard input and output streams for messaging avoids having to make +// allowances in the sandbox for other channels of communication, and seems a +// more portable approach than many of the alternatives. This is all somewhat +// temporary in any case — in the long term, something like distributed actors +// or something similar can hopefully replace the custom messaging. +// +// Usage: +// struct MyPlugin: CompilerPlugin { +// var providingMacros: [Macros.Type] = [ +// StringifyMacro.self +// ] +public protocol CompilerPlugin { + init() + + var providingMacros: [Macro.Type] { get } +} + +extension CompilerPlugin { + + /// Main entry point of the plugin — sets up a communication channel with + /// the plugin host and runs the main message loop. + public static func main() throws { + // Duplicate the `stdin` file descriptor, which we will then use for + // receiving messages from the plugin host. + let inputFD = dup(fileno(stdin)) + guard inputFD >= 0 else { + internalError("Could not duplicate `stdin`: \(describe(errno: errno)).") + } + + // Having duplicated the original standard-input descriptor, we close + // `stdin` so that attempts by the plugin to read console input (which + // are usually a mistake) return errors instead of blocking. + guard close(fileno(stdin)) >= 0 else { + internalError("Could not close `stdin`: \(describe(errno: errno)).") + } + + // Duplicate the `stdout` file descriptor, which we will then use for + // sending messages to the plugin host. + let outputFD = dup(fileno(stdout)) + guard outputFD >= 0 else { + internalError("Could not dup `stdout`: \(describe(errno: errno)).") + } + + // Having duplicated the original standard-output descriptor, redirect + // `stdout` to `stderr` so that all free-form text output goes there. + guard dup2(fileno(stderr), fileno(stdout)) >= 0 else { + internalError("Could not dup2 `stdout` to `stderr`: \(describe(errno: errno)).") + } + + // Turn off full buffering so printed text appears as soon as possible. + // Windows is much less forgiving than other platforms. If line + // buffering is enabled, we must provide a buffer and the size of the + // buffer. As a result, on Windows, we completely disable all + // buffering, which means that partial writes are possible. + #if os(Windows) + setvbuf(stdout, nil, _IONBF, 0) + #else + setvbuf(stdout, nil, _IOLBF, 0) + #endif + + // Open a message channel for communicating with the plugin host. + pluginHostConnection = PluginHostConnection( + inputStream: FileHandle(fileDescriptor: inputFD), + outputStream: FileHandle(fileDescriptor: outputFD) + ) + + // Handle messages from the host until the input stream is closed, + // indicating that we're done. + let instance = Self() + do { + while let message = try pluginHostConnection.waitForNextMessage() { + try instance.handleMessage(message) + } + } catch { + // Emit a diagnostic and indicate failure to the plugin host, + // and exit with an error code. + internalError(String(describing: error)) + } + } + + // Private function to report internal errors and then exit. + fileprivate static func internalError(_ message: String) -> Never { + fputs("Internal Error: \(message)\n", stderr) + exit(1) + } + + // Private function to construct an error message from an `errno` code. + fileprivate static func describe(errno: Int32) -> String { + if let cStr = strerror(errno) { return String(cString: cStr) } + return String(describing: errno) + } + + /// Handles a single message received from the plugin host. + fileprivate func handleMessage(_ message: HostToPluginMessage) throws { + switch message { + case .getCapability: + try pluginHostConnection.sendMessage( + .getCapabilityResult(capability: PluginMessage.capability) + ) + break + + case .expandFreestandingMacro(let macro, let discriminator, let expandingSyntax): + try expandFreestandingMacro( + macro: macro, + discriminator: discriminator, + expandingSyntax: expandingSyntax + ) + + case .expandAttachedMacro(let macro, let macroRole, let discriminator, let attributeSyntax, let declSyntax, let parentDeclSyntax): + try expandAttachedMacro( + macro: macro, + macroRole: macroRole, + discriminator: discriminator, + attributeSyntax: attributeSyntax, + declSyntax: declSyntax, + parentDeclSyntax: parentDeclSyntax + ) + } + } +} + +/// Message channel for bidirectional communication with the plugin host. +internal fileprivate(set) var pluginHostConnection: PluginHostConnection! + +typealias PluginHostConnection = MessageConnection + +internal struct MessageConnection where TX: Encodable, RX: Decodable { + let inputStream: FileHandle + let outputStream: FileHandle + + func sendMessage(_ message: TX) throws { + // Encode the message as JSON. + let payload = try JSONEncoder().encode(message) + + // Write the header (a 64-bit length field in little endian byte order). + var count = UInt64(payload.count).littleEndian + let header = Swift.withUnsafeBytes(of: &count) { Data($0) } + assert(header.count == 8) + + // Write the header and payload. + try outputStream._write(contentsOf: header) + try outputStream._write(contentsOf: payload) + } + + func waitForNextMessage() throws -> RX? { + // Read the header (a 64-bit length field in little endian byte order). + guard + let header = try inputStream._read(upToCount: 8), + header.count != 0 + else { + return nil + } + guard header.count == 8 else { + throw PluginMessageError.truncatedHeader + } + + // Decode the count. + let count = header.withUnsafeBytes { + UInt64(littleEndian: $0.load(as: UInt64.self)) + } + guard count >= 2 else { + throw PluginMessageError.invalidPayloadSize + } + + // Read the JSON payload. + guard + let payload = try inputStream._read(upToCount: Int(count)), + payload.count == count + else { + throw PluginMessageError.truncatedPayload + } + + // Decode and return the message. + return try JSONDecoder().decode(RX.self, from: payload) + } + + enum PluginMessageError: Swift.Error { + case truncatedHeader + case invalidPayloadSize + case truncatedPayload + } +} + +private extension FileHandle { + func _write(contentsOf data: Data) throws { + if #available(macOS 10.15.4, iOS 13.4, watchOS 6.2, tvOS 13.4, *) { + return try self.write(contentsOf: data) + } else { + return self.write(data) + } + } + + func _read(upToCount count: Int) throws -> Data? { + if #available(macOS 10.15.4, iOS 13.4, watchOS 6.2, tvOS 13.4, *) { + return try self.read(upToCount: count) + } else { + return self.readData(ofLength: 8) + } + } +} diff --git a/Sources/SwiftCompilerPlugin/Diagnostics.swift b/Sources/SwiftCompilerPlugin/Diagnostics.swift new file mode 100644 index 00000000000..f3c458a112f --- /dev/null +++ b/Sources/SwiftCompilerPlugin/Diagnostics.swift @@ -0,0 +1,128 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SwiftDiagnostics +import SwiftSyntax + +/// Errors in macro handing. +enum MacroExpansionError: String { + case macroTypeNotFound = "macro expanding type not found" + case unmathedMacroRole = "macro doesn't conform to required macro role" + case freestandingMacroSyntaxIsNotMacro = "macro syntax couldn't be parsed" + case invalidExpansionMessage = "internal message error; please file a bug report" +} + +extension MacroExpansionError: DiagnosticMessage { + var message: String { + self.rawValue + } + var diagnosticID: SwiftDiagnostics.MessageID { + .init(domain: "SwiftCompilerPlugin", id: "\(type(of: self)).\(self)") + } + var severity: SwiftDiagnostics.DiagnosticSeverity { + .error + } +} + +extension MacroExpansionError: Error {} + +extension PluginMessage.Diagnostic.Severity { + init(from syntaxDiagSeverity: SwiftDiagnostics.DiagnosticSeverity) { + switch syntaxDiagSeverity { + case .error: self = .error + case .warning: self = .warning + case .note: self = .note + } + } +} + +extension PluginMessage.Diagnostic { + init(from syntaxDiag: SwiftDiagnostics.Diagnostic, in sourceManager: SourceManager) { + if let position = sourceManager.position( + of: syntaxDiag.node, + at: .afterLeadingTrivia + ) { + self.position = .init(fileName: position.fileName, offset: position.utf8Offset) + } else { + self.position = .invalid + } + + self.severity = .init(from: syntaxDiag.diagMessage.severity) + self.message = syntaxDiag.message + + self.highlights = syntaxDiag.highlights.compactMap { + guard let range = sourceManager.range(of: $0) else { + return nil + } + return .init( + fileName: range.fileName, + startOffset: range.startUTF8Offset, + endOffset: range.endUTF8Offset + ) + } + + self.notes = syntaxDiag.notes.compactMap { + guard let pos = sourceManager.position(of: $0.node, at: .afterLeadingTrivia) else { + return nil + } + let position = PluginMessage.Diagnostic.Position( + fileName: pos.fileName, + offset: pos.utf8Offset + ) + return .init(position: position, message: $0.message) + } + + self.fixIts = syntaxDiag.fixIts.compactMap { + PluginMessage.Diagnostic.FixIt( + message: $0.message.message, + changes: $0.changes.changes.compactMap { + let range: SourceManager.SourceRange? + let text: String + switch $0 { + case .replace(let oldNode, let newNode): + range = sourceManager.range( + of: oldNode, + from: .afterLeadingTrivia, + to: .beforeTrailingTrivia + ) + text = newNode.trimmedDescription + case .replaceLeadingTrivia(let token, let newTrivia): + range = sourceManager.range( + of: Syntax(token), + from: .beforeLeadingTrivia, + to: .afterLeadingTrivia + ) + text = newTrivia.description + case .replaceTrailingTrivia(let token, let newTrivia): + range = sourceManager.range( + of: Syntax(token), + from: .beforeTrailingTrivia, + to: .afterTrailingTrivia + ) + text = newTrivia.description + } + guard let range = range else { + return nil + } + return .init( + range: PositionRange( + fileName: range.fileName, + startOffset: range.startUTF8Offset, + endOffset: range.endUTF8Offset + ), + newText: text + ) + } + ) + } + } +} diff --git a/Sources/SwiftCompilerPlugin/Macros.swift b/Sources/SwiftCompilerPlugin/Macros.swift new file mode 100644 index 00000000000..c45dc950793 --- /dev/null +++ b/Sources/SwiftCompilerPlugin/Macros.swift @@ -0,0 +1,268 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SwiftDiagnostics +import SwiftSyntax +import SwiftSyntaxMacros + +/// Implementation for `CompilerPlugin` macro related request processing. +extension CompilerPlugin { + private func resolveMacro(moduleName: String, typeName: String) -> Macro.Type? { + let qualifedName = "\(moduleName).\(typeName)" + + for type in self.providingMacros { + // FIXME: Is `String(reflecting:)` stable? + // Getting the module name and type name should be more robust. + let name = String(reflecting: type) + if name == qualifedName { + return type + } + } + return nil + } + + /// Get concrete macro type from a pair of module name and type name. + private func resolveMacro(_ ref: PluginMessage.MacroReference) -> Macro.Type? { + resolveMacro(moduleName: ref.moduleName, typeName: ref.typeName) + } + + /// Expand `@freestainding(XXX)` macros. + func expandFreestandingMacro( + macro: PluginMessage.MacroReference, + discriminator: String, + expandingSyntax: PluginMessage.Syntax + ) throws { + let sourceManager = SourceManager() + let syntax = sourceManager.add(expandingSyntax) + + let context = PluginMacroExpansionContext( + sourceManager: sourceManager, + expansionDiscriminator: discriminator + ) + + let expandedSource: String + do { + guard let macroSyntax = syntax.asProtocol(FreestandingMacroExpansionSyntax.self) else { + throw MacroExpansionError.freestandingMacroSyntaxIsNotMacro + } + guard let macroDefinition = resolveMacro(macro) else { + throw MacroExpansionError.macroTypeNotFound + } + + switch macroDefinition { + case let exprMacroDef as ExpressionMacro.Type: + let rewritten = try exprMacroDef.expansion(of: macroSyntax, in: context) + expandedSource = rewritten.description + + case let declMacroDef as DeclarationMacro.Type: + let rewritten = try declMacroDef.expansion(of: macroSyntax, in: context) + expandedSource = CodeBlockItemListSyntax(rewritten.map { CodeBlockItemSyntax(item: .decl($0)) }).description + + default: + throw MacroExpansionError.unmathedMacroRole + } + } catch { + let diagMessage: DiagnosticMessage + if let message = error as? DiagnosticMessage, message.severity == .error { + diagMessage = message + } else { + diagMessage = ThrownErrorDiagnostic(message: String(describing: error)) + } + expandedSource = "" + context.diagnose(Diagnostic(node: syntax, message: diagMessage)) + } + + let diagnostics = context.diagnostics.map { + PluginMessage.Diagnostic(from: $0, in: sourceManager) + } + try pluginHostConnection.sendMessage( + .expandFreestandingMacroResult(expandedSource: expandedSource, diagnostics: diagnostics) + ) + } + + /// Expand `@attached(XXX)` macros. + func expandAttachedMacro( + macro: PluginMessage.MacroReference, + macroRole: PluginMessage.MacroRole, + discriminator: String, + attributeSyntax: PluginMessage.Syntax, + declSyntax: PluginMessage.Syntax, + parentDeclSyntax: PluginMessage.Syntax? + ) throws { + let sourceManager = SourceManager() + let context = PluginMacroExpansionContext( + sourceManager: sourceManager, + expansionDiscriminator: discriminator + ) + + let attributeNode = sourceManager.add(attributeSyntax).cast(AttributeSyntax.self) + let declarationNode = sourceManager.add(declSyntax).cast(DeclSyntax.self) + + let expandedSources: [String] + do { + guard let macroDefinition = resolveMacro(macro) else { + throw MacroExpansionError.macroTypeNotFound + } + + switch (macroDefinition, macroRole) { + case (let attachedMacro as AccessorMacro.Type, .accessor): + let accessors = try attachedMacro.expansion( + of: attributeNode, + providingAccessorsOf: declarationNode, + in: context + ) + expandedSources = accessors.map { + $0.trimmedDescription + } + + case (let attachedMacro as MemberAttributeMacro.Type, .memberAttribute): + guard + let parentDeclSyntax = parentDeclSyntax, + let parentDeclGroup = sourceManager.add(parentDeclSyntax).asProtocol(DeclGroupSyntax.self) + else { + // Compiler error: 'parentDecl' is mandatory for MemberAttributeMacro. + throw MacroExpansionError.invalidExpansionMessage + } + + // Local function to expand a member atribute macro once we've opened up + // the existential. + func expandMemberAttributeMacro( + _ node: Node + ) throws -> [AttributeSyntax] { + return try attachedMacro.expansion( + of: attributeNode, + attachedTo: node, + providingAttributesFor: declarationNode, + in: context + ) + } + + let attributes = try _openExistential( + parentDeclGroup, + do: expandMemberAttributeMacro + ) + + // Form a buffer containing an attribute list to return to the caller. + expandedSources = attributes.map { + $0.trimmedDescription + } + + case (let attachedMacro as MemberMacro.Type, .member): + guard let declGroup = declarationNode.asProtocol(DeclGroupSyntax.self) + else { + // Compiler error: declNode for member macro must be DeclGroupSyntax. + throw MacroExpansionError.invalidExpansionMessage + } + + // Local function to expand a member macro once we've opened up + // the existential. + func expandMemberMacro( + _ node: Node + ) throws -> [DeclSyntax] { + return try attachedMacro.expansion( + of: attributeNode, + providingMembersOf: node, + in: context + ) + } + + let members = try _openExistential(declGroup, do: expandMemberMacro) + + // Form a buffer of member declarations to return to the caller. + expandedSources = members.map { $0.trimmedDescription } + + case (let attachedMacro as PeerMacro.Type, .peer): + let peers = try attachedMacro.expansion( + of: attributeNode, + providingPeersOf: declarationNode, + in: context + ) + + // Form a buffer of peer declarations to return to the caller. + expandedSources = peers.map { + $0.trimmedDescription + } + + case (let attachedMacro as ConformanceMacro.Type, .conformance): + guard + let declGroup = declarationNode.asProtocol(DeclGroupSyntax.self), + let identified = declarationNode.asProtocol(IdentifiedDeclSyntax.self) + else { + // Compiler error: type mismatch. + throw MacroExpansionError.invalidExpansionMessage + } + + // Local function to expand a conformance macro once we've opened up + // the existential. + func expandConformanceMacro( + _ node: Node + ) throws -> [(TypeSyntax, GenericWhereClauseSyntax?)] { + return try attachedMacro.expansion( + of: attributeNode, + providingConformancesOf: node, + in: context + ) + } + + let conformances = try _openExistential( + declGroup, + do: expandConformanceMacro + ) + + // Form a buffer of extension declarations to return to the caller. + expandedSources = conformances.map { typeSyntax, whereClause in + let typeName = identified.identifier.trimmedDescription + let protocolName = typeSyntax.trimmedDescription + let whereClause = whereClause?.trimmedDescription ?? "" + return "extension \(typeName) : \(protocolName) \(whereClause) {}" + } + + default: + throw MacroExpansionError.unmathedMacroRole + } + } catch { + let diagMessage: DiagnosticMessage + if let message = error as? DiagnosticMessage, message.severity == .error { + diagMessage = message + } else { + diagMessage = ThrownErrorDiagnostic(message: String(describing: error)) + } + expandedSources = [] + context.diagnose(Diagnostic(node: Syntax(attributeNode), message: diagMessage)) + } + + let diagnostics = context.diagnostics.map { + PluginMessage.Diagnostic(from: $0, in: sourceManager) + } + try pluginHostConnection.sendMessage( + .expandAttachedMacroResult(expandedSources: expandedSources, diagnostics: diagnostics) + ) + } +} + +extension CompilerPlugin { + // @testable + public func _resolveMacro(moduleName: String, typeName: String) -> Macro.Type? { + resolveMacro(moduleName: moduleName, typeName: typeName) + } +} + +/// Diagnostic message used for thrown errors. +fileprivate struct ThrownErrorDiagnostic: DiagnosticMessage { + let message: String + + var severity: DiagnosticSeverity { .error } + + var diagnosticID: MessageID { + .init(domain: "SwiftSyntaxMacros", id: "ThrownErrorDiagnostic") + } +} diff --git a/Sources/SwiftCompilerPlugin/PluginMacroExpansionContext.swift b/Sources/SwiftCompilerPlugin/PluginMacroExpansionContext.swift new file mode 100644 index 00000000000..32906f748ff --- /dev/null +++ b/Sources/SwiftCompilerPlugin/PluginMacroExpansionContext.swift @@ -0,0 +1,256 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SwiftDiagnostics +import SwiftParser +import SwiftOperators +import SwiftSyntax +import SwiftSyntaxMacros + +/// Manages known source code combined with their filename/fileID. This can be +/// used to get line/column from a syntax node in the managed source code. +class SourceManager { + class KnownSourceSyntax { + struct Location { + /// UTF-8 offset of the location in the file. + var offset: Int + var line: Int + var column: Int + /// A file ID consisting of the module name and file name (without full path), + /// as would be generated by the macro expansion `#fileID`. + var fileID: String + /// A full path name as would be generated by the macro expansion `#filePath`, + /// e.g., `/home/taylor/alison.swift`. + var fileName: String + } + + let node: Syntax + let location: Location + + init(node: Syntax, location: Location) { + self.node = node + self.location = location + } + + /// Location converter to get line/column in the node. + lazy var locationConverter: SourceLocationConverter = .init( + file: self.location.fileName, + tree: self.node + ) + } + + struct SourcePosition { + var fileName: String + var utf8Offset: Int + } + + struct SourceRange { + var fileName: String + var startUTF8Offset: Int + var endUTF8Offset: Int + } + + /// Syntax added by `add(_:)` method. Keyed by the `id` of the node. + private var knownSourceSyntax: [Syntax.ID: KnownSourceSyntax] = [:] + + /// Convert syntax information to a `Syntax` node. The location informations + /// are cached in the source manager to provide `location(of:)` et al. + func add(_ syntaxInfo: PluginMessage.Syntax) -> Syntax { + + var node: Syntax + var parser = Parser(syntaxInfo.source) + switch syntaxInfo.kind { + case .declaration: + node = Syntax(DeclSyntax.parse(from: &parser)) + case .statement: + node = Syntax(StmtSyntax.parse(from: &parser)) + case .expression: + node = Syntax(ExprSyntax.parse(from: &parser)) + case .type: + node = Syntax(TypeSyntax.parse(from: &parser)) + case .pattern: + node = Syntax(PatternSyntax.parse(from: &parser)) + case .attribute: + node = Syntax(AttributeSyntax.parse(from: &parser)) + } + node = OperatorTable.standardOperators.foldAll(node, errorHandler: { _ in /*ignore*/ }) + + // Copy the location info from the plugin message. + let location = KnownSourceSyntax.Location( + offset: syntaxInfo.location.offset, + line: syntaxInfo.location.line, + column: syntaxInfo.location.column, + fileID: syntaxInfo.location.fileID, + fileName: syntaxInfo.location.fileName + ) + + knownSourceSyntax[node.id] = KnownSourceSyntax(node: node, location: location) + + return node + } + + /// Get position (file name + UTF-8 offset) of `node` in the known root nodes. + /// The root node of `node` must be one of the retured value from `add(_:)`. + func position( + of node: Syntax, + at kind: PositionInSyntaxNode + ) -> SourcePosition? { + guard let base = self.knownSourceSyntax[node.root.id] else { + return nil + } + let localPosition = node.position(at: kind) + let positionOffset = base.location.offset + return SourcePosition( + fileName: base.location.fileName, + utf8Offset: localPosition.advanced(by: positionOffset).utf8Offset + ) + } + + /// Get `SourceRange` (file name + UTF-8 offset range) of `node` in the known root nodes. + /// The root node of `node` must be one of the retured value from `add(_:)`. + func range( + of node: Syntax, + from startKind: PositionInSyntaxNode = .afterLeadingTrivia, + to endKind: PositionInSyntaxNode = .beforeTrailingTrivia + ) -> SourceRange? { + guard let base = self.knownSourceSyntax[node.root.id] else { + return nil + } + let localStartPosition = node.position(at: startKind) + let localEndPosition = node.position(at: endKind) + assert(localStartPosition <= localEndPosition) + + let positionOffset = base.location.offset + + return SourceRange( + fileName: base.location.fileName, + startUTF8Offset: localStartPosition.advanced(by: positionOffset).utf8Offset, + endUTF8Offset: localEndPosition.advanced(by: positionOffset).utf8Offset + ) + } + + /// Get location of `node` in the known root nodes. + /// The root node of `node` must be one of the retured value from `add(_:)`. + func location(of node: Syntax, at kind: PositionInSyntaxNode, filePathMode: SourceLocationFilePathMode) -> SourceLocation? { + guard let base = self.knownSourceSyntax[node.root.id] else { + return nil + } + let file: String + switch filePathMode { + case .fileID: file = base.location.fileID + case .filePath: file = base.location.fileName + } + + let localPosition = node.position(at: kind) + let localLocation = base.locationConverter.location(for: localPosition) + + let positionOffset = base.location.offset + let lineOffset = base.location.line - 1 + let columnOffset = localLocation.line == 1 ? base.location.column : 0 + + return SourceLocation( + // NOTE: IUO because 'localLocation' is created by a location converter + // which guarantees non-nil line/column. + line: localLocation.line! + lineOffset, + column: localLocation.column! + columnOffset, + offset: localLocation.offset + positionOffset, + file: file + ) + } +} + +fileprivate extension Syntax { + /// Get a position in the node by `PositionInSyntaxNode`. + func position(at pos: PositionInSyntaxNode) -> AbsolutePosition { + switch pos { + case .beforeLeadingTrivia: + return self.position + case .afterLeadingTrivia: + return self.positionAfterSkippingLeadingTrivia + case .beforeTrailingTrivia: + return self.endPositionBeforeTrailingTrivia + case .afterTrailingTrivia: + return self.endPosition + } + } +} + +class PluginMacroExpansionContext { + private var sourceManger: SourceManager + + /// The macro expansion discriminator, which is used to form unique names + /// when requested. + /// + /// The expansion discriminator is combined with the `uniqueNames` counters + /// to produce unique names. + private var expansionDiscriminator: String + + /// Counter for each of the uniqued names. + /// + /// Used in conjunction with `expansionDiscriminator`. + private var uniqueNames: [String: Int] = [:] + + /// The set of diagnostics that were emitted as part of expanding the + /// macro. + internal private(set) var diagnostics: [Diagnostic] = [] + + init(sourceManager: SourceManager, expansionDiscriminator: String = "") { + self.sourceManger = sourceManager + self.expansionDiscriminator = expansionDiscriminator + } +} + +extension PluginMacroExpansionContext: MacroExpansionContext { + /// Generate a unique name for use in the macro. + public func createUniqueName(_ providedName: String) -> TokenSyntax { + // If provided with an empty name, substitute in something. + let name = providedName.isEmpty ? "__local" : providedName + + // Grab a unique index value for this name. + let uniqueIndex = uniqueNames[name, default: 0] + uniqueNames[name] = uniqueIndex + 1 + + // Start with the discriminator. + var resultString = expansionDiscriminator + + // Mangle the name + resultString += "\(name.count)\(name)" + + // Mangle the operator for unique macro names. + resultString += "fMu" + + // Mangle the index. + if uniqueIndex > 0 { + resultString += "\(uniqueIndex - 1)" + } + resultString += "_" + + return TokenSyntax(.identifier(resultString), presence: .present) + } + + /// Produce a diagnostic while expanding the macro. + public func diagnose(_ diagnostic: Diagnostic) { + diagnostics.append(diagnostic) + } + + public func location( + of node: Node, + at positionMode: PositionInSyntaxNode, + filePathMode: SourceLocationFilePathMode + ) -> SourceLocation? { + return sourceManger.location( + of: Syntax(node), + at: positionMode, + filePathMode: filePathMode + ) + } +} diff --git a/Sources/SwiftCompilerPlugin/PluginMessages.swift b/Sources/SwiftCompilerPlugin/PluginMessages.swift new file mode 100644 index 00000000000..de5e44f0cfa --- /dev/null +++ b/Sources/SwiftCompilerPlugin/PluginMessages.swift @@ -0,0 +1,153 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +// NOTE: This file should be synced between swift and swift-syntax repository. +// NOTE: Types in this file should be self-contained and should not depend on any non-stdlib types. + +internal enum HostToPluginMessage: Codable { + case getCapability + + case expandFreestandingMacro( + macro: PluginMessage.MacroReference, + discriminator: String, + syntax: PluginMessage.Syntax + ) + + case expandAttachedMacro( + macro: PluginMessage.MacroReference, + macroRole: PluginMessage.MacroRole, + discriminator: String, + attributeSyntax: PluginMessage.Syntax, + declSyntax: PluginMessage.Syntax, + parentDeclSyntax: PluginMessage.Syntax? + ) +} + +internal enum PluginToHostMessage: Codable { + case expandFreestandingMacroResult( + expandedSource: String?, + diagnostics: [PluginMessage.Diagnostic] + ) + + case expandAttachedMacroResult( + expandedSources: [String]?, + diagnostics: [PluginMessage.Diagnostic] + ) + + case getCapabilityResult(capability: PluginMessage.PluginCapability) +} + +/*namespace*/ internal enum PluginMessage { + static var PROTOCOL_VERSION_NUMBER: Int { 3 } // Renamed 'customAttributeSyntax' to 'attributeSyntax'. + + struct PluginCapability: Codable { + var protocolVersion: Int + } + + static var capability: PluginCapability { + PluginCapability(protocolVersion: PluginMessage.PROTOCOL_VERSION_NUMBER) + } + + struct MacroReference: Codable { + var moduleName: String + var typeName: String + + // The name of 'macro' declaration the client is using. + var name: String + } + + enum MacroRole: String, Codable { + case expression + case freeStandingDeclaration + case accessor + case memberAttribute + case member + case peer + case conformance + } + + struct SourceLocation: Codable { + /// A file ID consisting of the module name and file name (without full path), + /// as would be generated by the macro expansion `#fileID`. + var fileID: String + + /// A full path name as would be generated by the macro expansion `#filePath`, + /// e.g., `/home/taylor/alison.swift`. + var fileName: String + + /// UTF-8 offset of the location in the file. + var offset: Int + + var line: Int + var column: Int + } + + struct Diagnostic: Codable { + enum Severity: String, Codable { + case error + case warning + case note + } + struct Position: Codable { + var fileName: String + /// UTF-8 offset in the file. + var offset: Int + + static var invalid: Self { + .init(fileName: "", offset: 0) + } + } + struct PositionRange: Codable { + var fileName: String + /// UTF-8 offset of the start of the range in the file. + var startOffset: Int + /// UTF-8 offset of the end of the range in the file. + var endOffset: Int + + static var invalid: Self { + .init(fileName: "", startOffset: 0, endOffset: 0) + } + } + struct Note: Codable { + var position: Position + var message: String + } + struct FixIt: Codable { + struct Change: Codable { + var range: PositionRange + var newText: String + } + var message: String + var changes: [Change] + } + var message: String + var severity: Severity + var position: Position + var highlights: [PositionRange] + var notes: [Note] + var fixIts: [FixIt] + } + + struct Syntax: Codable { + enum Kind: String, Codable { + case declaration + case statement + case expression + case type + case pattern + case attribute + } + var kind: Kind + var source: String + var location: SourceLocation + } +} diff --git a/Tests/SwiftCompilerPluginTest/CompilerPluginTests.swift b/Tests/SwiftCompilerPluginTest/CompilerPluginTests.swift new file mode 100644 index 00000000000..2b201966ccb --- /dev/null +++ b/Tests/SwiftCompilerPluginTest/CompilerPluginTests.swift @@ -0,0 +1,70 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import XCTest +import SwiftCompilerPlugin +import SwiftSyntax +import SwiftSyntaxMacros + +/// Dummy macro +struct DummyMacro: ExpressionMacro { + static func expansion< + Node: FreestandingMacroExpansionSyntax, + Context: MacroExpansionContext + >( + of node: Node, + in context: Context + ) throws -> ExprSyntax { + fatalError() + } +} + +struct RegisteredMacro: ExpressionMacro { + static func expansion< + Node: FreestandingMacroExpansionSyntax, + Context: MacroExpansionContext + >( + of node: Node, + in context: Context + ) throws -> ExprSyntax { + fatalError() + } +} + +struct MyPlugin: CompilerPlugin { + let providingMacros: [Macro.Type] = [ + RegisteredMacro.self + ] +} + +public class CompilerPluginTests: XCTestCase { + + func testResolveMacro() { + let plugin = MyPlugin() + + let registeredMacro = plugin._resolveMacro( + moduleName: "SwiftCompilerPluginTest", + typeName: "RegisteredMacro" + ) + XCTAssertNotNil(registeredMacro) + XCTAssertTrue(registeredMacro == RegisteredMacro.self) + + /// Test the plugin doesn't provide unregistered macros. + let dummyMacro = plugin._resolveMacro( + moduleName: "SwiftCompilerPluginTest", + typeName: "DummyMacro" + ) + XCTAssertNil(dummyMacro) + XCTAssertFalse(dummyMacro == DummyMacro.self) + + } +} diff --git a/build-script.py b/build-script.py index b7a547773df..bfed21efec7 100755 --- a/build-script.py +++ b/build-script.py @@ -523,22 +523,33 @@ def check_incr_transfer_roundtrip_exec() -> None: ) -def find_lit_test_helper_exec( - toolchain: str, build_dir: Optional[str], release: bool +def find_swiftpm_bin_path( + package_dir: str, toolchain: str, build_dir: Optional[str], release: bool ) -> str: swiftpm_call = get_swiftpm_invocation( toolchain=toolchain, action="build", - package_dir=PACKAGE_DIR, + package_dir=package_dir, build_dir=build_dir, multiroot_data_file=None, release=release, ) - swiftpm_call.extend(["--product", "lit-test-helper"]) swiftpm_call.extend(["--show-bin-path"]) bin_dir = subprocess.check_output(swiftpm_call) - return os.path.join(bin_dir.strip().decode('utf-8'), "lit-test-helper") + return bin_dir.strip().decode('utf-8') + + +def find_product_bin_path( + toolchain: str, build_dir: Optional[str], release: bool +) -> str: + return find_swiftpm_bin_path(PACKAGE_DIR, toolchain, build_dir, release) + + +def find_examples_bin_path( + toolchain: str, build_dir: Optional[str], release: bool +) -> str: + return find_swiftpm_bin_path(EXAMPLES_DIR, toolchain, build_dir, release) def run_lit_tests(toolchain: str, build_dir: Optional[str], release: bool, @@ -548,9 +559,12 @@ def run_lit_tests(toolchain: str, build_dir: Optional[str], release: bool, check_lit_exec() check_incr_transfer_roundtrip_exec() - lit_test_helper_exec = find_lit_test_helper_exec( - toolchain=toolchain, build_dir=build_dir, release=release - ) + product_bin_path = find_product_bin_path( + toolchain=toolchain, build_dir=build_dir, release=release) + examples_bin_path = find_examples_bin_path( + toolchain=toolchain, build_dir=build_dir, release=release) + + lit_test_helper_exec = os.path.join(product_bin_path, "lit-test-helper") lit_call = ["python3", LIT_EXEC] lit_call.append(os.path.join(PACKAGE_DIR, "lit_tests")) @@ -562,6 +576,8 @@ def run_lit_tests(toolchain: str, build_dir: Optional[str], release: bool, lit_call.extend( ["--param", "INCR_TRANSFER_ROUND_TRIP.PY=" + INCR_TRANSFER_ROUNDTRIP_EXEC] ) + lit_call.extend(["--param", "EXAMPLES_BIN_PATH=" + examples_bin_path]) + lit_call.extend(["--param", "TOOLCHAIN=" + toolchain]) # Print all failures lit_call.extend(["--verbose"]) @@ -656,7 +672,7 @@ def verify_source_code_command(args: argparse.Namespace) -> None: def build_command(args: argparse.Namespace) -> None: try: builder = Builder( - toolchain=args.toolchain, + toolchain=realpath(args.toolchain), build_dir=realpath(args.build_dir), multiroot_data_file=args.multiroot_data_file, release=args.release, @@ -679,7 +695,7 @@ def build_command(args: argparse.Namespace) -> None: def test_command(args: argparse.Namespace) -> None: try: builder = Builder( - toolchain=args.toolchain, + toolchain=realpath(args.toolchain), build_dir=realpath(args.build_dir), multiroot_data_file=args.multiroot_data_file, release=args.release, @@ -688,9 +704,10 @@ def test_command(args: argparse.Namespace) -> None: ) builder.buildProduct("lit-test-helper") + builder.buildExample("ExamplePlugin") run_tests( - toolchain=args.toolchain, + toolchain=realpath(args.toolchain), build_dir=realpath(args.build_dir), multiroot_data_file=args.multiroot_data_file, release=args.release, diff --git a/lit_tests/compiler_plugin_basic.swift b/lit_tests/compiler_plugin_basic.swift new file mode 100644 index 00000000000..4fc3256de7e --- /dev/null +++ b/lit_tests/compiler_plugin_basic.swift @@ -0,0 +1,29 @@ +// REQUIRES: platform=Darwin +// +// RUN: %empty-directory(%t) +// +// RUN: %swift-frontend -typecheck -swift-version 5 \ +// RUN: -enable-experimental-feature Macros \ +// RUN: -dump-macro-expansions \ +// RUN: -load-plugin-executable %examples_bin_path/ExamplePlugin#ExamplePlugin \ +// RUN -module-name MyApp \ +// RUN: %s 2>&1 | tee %t/expansions-dump.txt +// +// RUN: %FileCheck %s < %t/expansions-dump.txt + +@freestanding(expression) +macro echo(_: T) -> T = #externalMacro(module: "ExamplePlugin", type: "EchoExpressionMacro") + +@attached(member) +macro Metadata() = #externalMacro(module: "ExamplePlugin", type: "MetadataMacro") + +@Metadata +class MyClass { + var value: Int = #echo(12) +} + +// For '@Metadata' +// CHECK: static var __metadata__: [String: String] { ["name": "MyClass"] } + +// For '#echo(12)' +// CHECK: /* echo */12 diff --git a/lit_tests/lit.cfg b/lit_tests/lit.cfg index f256cee13d2..7b8d6cb2bfb 100644 --- a/lit_tests/lit.cfg +++ b/lit_tests/lit.cfg @@ -11,6 +11,7 @@ # ----------------------------------------------------------------------------- import lit +import platform import tempfile @@ -25,6 +26,11 @@ def inferSwiftBinaryImpl(binaryName, envVarName): if execPath: return execPath + # Find in the toolchain. + execPath = lit_config.params.get('TOOLCHAIN') + '/bin/' + binaryName + if os.path.exists(execPath): + return execPath + # Lastly, look in the path. return lit.util.which(binaryName, config.environment["PATH"]) @@ -53,10 +59,21 @@ config.suffixes = [".swift"] config.test_format = lit.formats.ShTest(execute_external=True) config.test_exec_root = tempfile.gettempdir() +config.examples_bin_path = lit_config.params.get("EXAMPLES_BIN_PATH") config.filecheck = inferSwiftBinary("FileCheck") config.incr_transfer_round_trip = inferSwiftBinary("incr_transfer_round_trip.py") config.lit_test_helper = inferSwiftBinary("lit-test-helper") +config.swift = inferSwiftBinary("swift") +config.swiftc = inferSwiftBinary("swiftc") +config.swift_frontend = inferSwiftBinary("swift-frontend") + +# Use features like this in lit: +# // REQUIRES: platform= +# where is Linux or Darwin +# Add a platform feature. +config.available_features.add("platform=" + platform.system()) +config.substitutions.append(("%examples_bin_path", config.examples_bin_path)) config.substitutions.append( ("%empty-directory\(([^)]+)\)", 'rm -rf "\\1" && mkdir -p "\\1"') ) @@ -69,3 +86,6 @@ config.substitutions.append( ) ) config.substitutions.append(("%lit-test-helper", config.lit_test_helper)) +config.substitutions.append(("%swiftc", config.swiftc)) +config.substitutions.append(("%swift", config.swift)) +config.substitutions.append(("%swift-frontend", config.swift_frontend))