Skip to content

Commit dc85f43

Browse files
kyleconroykyleconroy
authored
feat: Codegen plugins, powered by WASM (#1684)
Generate code via WASM-based plugins
1 parent 9cc7a6d commit dc85f43

File tree

18 files changed

+401
-74
lines changed

18 files changed

+401
-74
lines changed

‎cmd/sqlc-gen-json/main.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package main
22

33
import (
44
"bufio"
5+
"context"
56
"fmt"
67
"io"
78
"os"
@@ -26,7 +27,7 @@ func run() error {
2627
if err := req.UnmarshalVT(reqBlob); err != nil {
2728
return err
2829
}
29-
resp, err := json.Generate(&req)
30+
resp, err := json.Generate(context.Background(), &req)
3031
if err != nil {
3132
return err
3233
}

‎go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ require (
2121
)
2222

2323
require (
24+
github.com/bytecodealliance/wasmtime-go v0.37.0 // indirect
2425
github.com/golang/protobuf v1.5.2 // indirect
2526
github.com/inconshreveable/mousetrap v1.0.0 // indirect
2627
github.com/jackc/chunkreader/v2 v2.0.1 // indirect

‎go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220209173558-ad29539cd2e9 h1:z
55
github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220209173558-ad29539cd2e9/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY=
66
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
77
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
8+
github.com/bytecodealliance/wasmtime-go v0.37.0 h1:eNP2Snp5UFMuGuunRPxwVETJ/WpC8LhWonZAklXJfjk=
9+
github.com/bytecodealliance/wasmtime-go v0.37.0/go.mod h1:q320gUxqyI8yB+ZqRuaJOEnGkAnHh6WtJjMaT2CW4wI=
810
github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
911
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
1012
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=

‎internal/cmd/generate.go

Lines changed: 68 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,10 @@ import (
2020
"github.com/kyleconroy/sqlc/internal/debug"
2121
"github.com/kyleconroy/sqlc/internal/ext"
2222
"github.com/kyleconroy/sqlc/internal/ext/process"
23+
"github.com/kyleconroy/sqlc/internal/ext/wasm"
2324
"github.com/kyleconroy/sqlc/internal/multierr"
2425
"github.com/kyleconroy/sqlc/internal/opts"
26+
"github.com/kyleconroy/sqlc/internal/plugin"
2527
)
2628

2729
const errMessageNoVersion = `The configuration file must have a version number.
@@ -51,6 +53,15 @@ type outPair struct {
5153
config.SQL
5254
}
5355

56+
func findPlugin(conf config.Config, name string) (*config.Plugin, error) {
57+
for _, plug := range conf.Plugins {
58+
if plug.Name == name {
59+
return &plug, nil
60+
}
61+
}
62+
return nil, fmt.Errorf("plugin not found")
63+
}
64+
5465
func readConfig(stderr io.Writer, dir, filename string) (string, *config.Config, error) {
5566
configPath := ""
5667
if filename != "" {
@@ -216,43 +227,7 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer
216227
break
217228
}
218229

219-
var region *trace.Region
220-
if debug.Traced {
221-
region = trace.StartRegion(ctx, "codegen")
222-
}
223-
var handler ext.Handler
224-
var out string
225-
switch {
226-
case sql.Gen.Go != nil:
227-
out = combo.Go.Out
228-
handler = ext.HandleFunc(golang.Generate)
229-
230-
case sql.Gen.Kotlin != nil:
231-
out = combo.Kotlin.Out
232-
handler = ext.HandleFunc(kotlin.Generate)
233-
234-
case sql.Gen.Python != nil:
235-
out = combo.Python.Out
236-
handler = ext.HandleFunc(python.Generate)
237-
238-
case sql.Gen.JSON != nil:
239-
out = combo.JSON.Out
240-
handler = ext.HandleFunc(json.Generate)
241-
242-
case sql.Plugin != nil:
243-
out = sql.Plugin.Out
244-
handler = &process.Runner{
245-
Config: combo.Global,
246-
Plugin: sql.Plugin.Plugin,
247-
}
248-
249-
default:
250-
panic("missing language backend")
251-
}
252-
resp, err := handler.Generate(codeGenRequest(result, combo))
253-
if region != nil {
254-
region.End()
255-
}
230+
out, resp, err := codegen(ctx, combo, sql, result)
256231
if err != nil {
257232
fmt.Fprintf(stderr, "# package %s\n", name)
258233
fmt.Fprintf(stderr, "error generating code: %s\n", err)
@@ -262,6 +237,7 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer
262237
}
263238
continue
264239
}
240+
265241
files := map[string]string{}
266242
for _, file := range resp.Files {
267243
files[file.Name] = string(file.Contents)
@@ -313,3 +289,58 @@ func parse(ctx context.Context, e Env, name, dir string, sql config.SQL, combo c
313289
}
314290
return c.Result(), false
315291
}
292+
293+
func codegen(ctx context.Context, combo config.CombinedSettings, sql outPair, result *compiler.Result) (string, *plugin.CodeGenResponse, error) {
294+
var region *trace.Region
295+
if debug.Traced {
296+
region = trace.StartRegion(ctx, "codegen")
297+
}
298+
var handler ext.Handler
299+
var out string
300+
switch {
301+
case sql.Gen.Go != nil:
302+
out = combo.Go.Out
303+
handler = ext.HandleFunc(golang.Generate)
304+
305+
case sql.Gen.Kotlin != nil:
306+
out = combo.Kotlin.Out
307+
handler = ext.HandleFunc(kotlin.Generate)
308+
309+
case sql.Gen.Python != nil:
310+
out = combo.Python.Out
311+
handler = ext.HandleFunc(python.Generate)
312+
313+
case sql.Gen.JSON != nil:
314+
out = combo.JSON.Out
315+
handler = ext.HandleFunc(json.Generate)
316+
317+
case sql.Plugin != nil:
318+
out = sql.Plugin.Out
319+
plug, err := findPlugin(combo.Global, sql.Plugin.Plugin)
320+
if err != nil {
321+
return "", nil, fmt.Errorf("plugin not found: %s", err)
322+
}
323+
324+
switch {
325+
case plug.Process != nil:
326+
handler = &process.Runner{
327+
Cmd: plug.Process.Cmd,
328+
}
329+
case plug.WASM != nil:
330+
handler = &wasm.Runner{
331+
URL: plug.WASM.URL,
332+
Checksum: plug.WASM.Checksum,
333+
}
334+
default:
335+
return "", nil, fmt.Errorf("unsupported plugin type")
336+
}
337+
338+
default:
339+
return "", nil, fmt.Errorf("missing language backend")
340+
}
341+
resp, err := handler.Generate(ctx, codeGenRequest(result, combo))
342+
if region != nil {
343+
region.End()
344+
}
345+
return out, resp, err
346+
}

‎internal/codegen/golang/gen.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package golang
33
import (
44
"bufio"
55
"bytes"
6+
"context"
67
"errors"
78
"fmt"
89
"go/format"
@@ -42,7 +43,7 @@ func (t *tmplCtx) OutputQuery(sourceName string) bool {
4243
return t.SourceName == sourceName
4344
}
4445

45-
func Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) {
46+
func Generate(ctx context.Context, req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) {
4647
enums := buildEnums(req)
4748
structs := buildStructs(req)
4849
queries, err := buildQueries(req, structs)

‎internal/codegen/json/gen.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package json
22

33
import (
44
"bytes"
5+
"context"
56
ejson "encoding/json"
67
"fmt"
78

@@ -31,7 +32,7 @@ func parseOptions(req *plugin.CodeGenRequest) (plugin.JSONCode, error) {
3132
return plugin.JSONCode{}, nil
3233
}
3334

34-
func Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) {
35+
func Generate(ctx context.Context, req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) {
3536
options, err := parseOptions(req)
3637
if err != nil {
3738
return nil, err

‎internal/codegen/kotlin/gen.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package kotlin
33
import (
44
"bufio"
55
"bytes"
6+
"context"
67
"errors"
78
"fmt"
89
"regexp"
@@ -759,7 +760,7 @@ func ktFormat(s string) string {
759760
return o
760761
}
761762

762-
func Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) {
763+
func Generate(ctx context.Context, req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) {
763764
enums := buildEnums(req)
764765
structs := buildDataClasses(req)
765766
queries, err := buildQueries(req, structs)

‎internal/codegen/python/gen.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package python
22

33
import (
4+
"context"
45
"errors"
56
"fmt"
67
"log"
@@ -1080,7 +1081,7 @@ func HashComment(s string) string {
10801081
return "# " + strings.ReplaceAll(s, "\n", "\n# ")
10811082
}
10821083

1083-
func Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) {
1084+
func Generate(_ context.Context, req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) {
10841085
enums := buildEnums(req)
10851086
models := buildModels(req)
10861087
queries, err := buildQueries(req, models)

‎internal/config/config.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,8 @@ type Plugin struct {
8888
Cmd string `json:"cmd" yaml:"cmd"`
8989
} `json:"process" yaml:"process"`
9090
WASM *struct {
91-
URL string `json:"url" yaml:"url"`
91+
URL string `json:"url" yaml:"url"`
92+
Checksum string `json:"checksum" yaml:"checksum"`
9293
} `json:"wasm" yaml:"wasm"`
9394
}
9495

‎internal/endtoend/endtoend_test.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,11 @@ func cmpDirectory(t *testing.T, dir string, actual map[string]string) {
116116
if file.IsDir() {
117117
return nil
118118
}
119-
if !strings.HasSuffix(path, ".go") && !strings.HasSuffix(path, ".kt") && !strings.HasSuffix(path, ".py") && !strings.HasSuffix(path, ".json") {
119+
if !strings.HasSuffix(path, ".go") && !strings.HasSuffix(path, ".kt") && !strings.HasSuffix(path, ".py") && !strings.HasSuffix(path, ".json") && !strings.HasSuffix(path, ".txt") {
120+
return nil
121+
}
122+
// TODO: Figure out a better way to ignore certain files
123+
if strings.HasSuffix(path, ".txt") && filepath.Base(path) != "hello.txt" {
120124
return nil
121125
}
122126
if filepath.Base(path) == "sqlc.json" {

‎internal/endtoend/testdata/wasm_plugin_sqlc_gen_greeter/gen/hello.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Hello World

‎internal/endtoend/testdata/wasm_plugin_sqlc_gen_greeter/query.sql

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
-- name: GetAuthor :one
2+
SELECT * FROM authors
3+
WHERE id = $1 LIMIT 1;
4+
5+
-- name: ListAuthors :many
6+
SELECT * FROM authors
7+
ORDER BY name;
8+
9+
-- name: CreateAuthor :one
10+
INSERT INTO authors (
11+
name, bio
12+
) VALUES (
13+
$1, $2
14+
)
15+
RETURNING *;
16+
17+
-- name: DeleteAuthor :exec
18+
DELETE FROM authors
19+
WHERE id = $1;

‎internal/endtoend/testdata/wasm_plugin_sqlc_gen_greeter/schema.sql

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
CREATE TABLE authors (
2+
id BIGSERIAL PRIMARY KEY,
3+
name text NOT NULL,
4+
bio text
5+
);

‎internal/endtoend/testdata/wasm_plugin_sqlc_gen_greeter/sqlc.json

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"version": "2",
3+
"sql": [
4+
{
5+
"schema": "schema.sql",
6+
"queries": "query.sql",
7+
"engine": "postgresql",
8+
"codegen": [
9+
{
10+
"out": "gen",
11+
"plugin": "greeter"
12+
}
13+
]
14+
}
15+
],
16+
"plugins": [
17+
{
18+
"name": "greeter",
19+
"wasm": {
20+
"url": "https://github.com/kyleconroy/sqlc-gen-greeter/releases/download/v0.1.0/sqlc-gen-greeter.wasm",
21+
"checksum": "sha256/afc486dac2068d741d7a4110146559d12a013fd0286f42a2fc7dcd802424ad07"
22+
}
23+
}
24+
]
25+
}

‎internal/ext/handler.go

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,23 @@
11
package ext
22

33
import (
4+
"context"
5+
46
"github.com/kyleconroy/sqlc/internal/plugin"
57
)
68

79
type Handler interface {
8-
Generate(*plugin.CodeGenRequest) (*plugin.CodeGenResponse, error)
10+
Generate(context.Context, *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error)
911
}
1012

1113
type wrapper struct {
12-
fn func(*plugin.CodeGenRequest) (*plugin.CodeGenResponse, error)
14+
fn func(context.Context, *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error)
1315
}
1416

15-
func (w *wrapper) Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) {
16-
return w.fn(req)
17+
func (w *wrapper) Generate(ctx context.Context, req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) {
18+
return w.fn(ctx, req)
1719
}
1820

19-
func HandleFunc(fn func(*plugin.CodeGenRequest) (*plugin.CodeGenResponse, error)) Handler {
21+
func HandleFunc(fn func(context.Context, *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error)) Handler {
2022
return &wrapper{fn}
2123
}

‎internal/ext/process/gen.go

Lines changed: 4 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -9,47 +9,26 @@ import (
99

1010
"google.golang.org/protobuf/proto"
1111

12-
"github.com/kyleconroy/sqlc/internal/config"
1312
"github.com/kyleconroy/sqlc/internal/plugin"
1413
)
1514

1615
type Runner struct {
17-
Config config.Config
18-
Plugin string
19-
}
20-
21-
func (r Runner) pluginCmd() (string, error) {
22-
for _, plug := range r.Config.Plugins {
23-
if plug.Name != r.Plugin {
24-
continue
25-
}
26-
if plug.Process == nil {
27-
continue
28-
}
29-
return plug.Process.Cmd, nil
30-
}
31-
return "", fmt.Errorf("plugin not found")
16+
Cmd string
3217
}
3318

3419
// TODO: Update the gen func signature to take a ctx
35-
func (r Runner) Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) {
20+
func (r Runner) Generate(ctx context.Context, req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) {
3621
stdin, err := proto.Marshal(req)
3722
if err != nil {
3823
return nil, fmt.Errorf("failed to encode codegen request: %s", err)
3924
}
4025

41-
name, err := r.pluginCmd()
42-
if err != nil {
43-
return nil, fmt.Errorf("process: unknown plugin %s", r.Plugin)
44-
}
45-
4626
// Check if the output plugin exists
47-
path, err := exec.LookPath(name)
27+
path, err := exec.LookPath(r.Cmd)
4828
if err != nil {
49-
return nil, fmt.Errorf("process: %s not found", name)
29+
return nil, fmt.Errorf("process: %s not found", r.Cmd)
5030
}
5131

52-
ctx := context.Background()
5332
cmd := exec.CommandContext(ctx, path)
5433
cmd.Stdin = bytes.NewReader(stdin)
5534
cmd.Env = []string{

0 commit comments

Comments
 (0)