Skip to content

Commit 49e08c2

Browse files
toothrottoothrot
committed
cmd/relui: add datastore support
This change replaces the file storage layer with Google Cloud Datastore. It adds a fake implementation of the datastore client, supporting only the features we use so far. Slightly simplifies Workflow configuration. Updates golang/go#40279 Change-Id: I55228f6540fbcdf5f803203ff7309232cebf6a20 Reviewed-on: https://go-review.googlesource.com/c/build/+/275237 Run-TryBot: Alexander Rakoczy <[email protected]> TryBot-Result: Go Bot <[email protected]> Trust: Alexander Rakoczy <[email protected]> Reviewed-by: Dmitri Shuralyov <[email protected]> Reviewed-by: Carlos Amedee <[email protected]>
1 parent a3221dd commit 49e08c2

File tree

13 files changed

+414
-324
lines changed

13 files changed

+414
-324
lines changed

‎cmd/relui/internal/datastore/fake/client.go

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
// Copyright 2020 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
// fake provides a fake implementation of a Datastore client to use in testing.
6+
package fake
7+
8+
import (
9+
"context"
10+
"log"
11+
"reflect"
12+
13+
"cloud.google.com/go/datastore"
14+
"github.com/googleapis/google-cloud-go-testing/datastore/dsiface"
15+
)
16+
17+
// Client is a fake implementation of dsiface.Client to use in testing.
18+
type Client struct {
19+
dsiface.Client
20+
21+
db map[string]map[string]interface{}
22+
}
23+
24+
var _ dsiface.Client = &Client{}
25+
26+
// Close is unimplemented and panics.
27+
func (f *Client) Close() error {
28+
panic("unimplemented")
29+
}
30+
31+
// AllocateIDs is unimplemented and panics.
32+
func (f *Client) AllocateIDs(context.Context, []*datastore.Key) ([]*datastore.Key, error) {
33+
panic("unimplemented")
34+
}
35+
36+
// Count is unimplemented and panics.
37+
func (f *Client) Count(context.Context, *datastore.Query) (n int, err error) {
38+
panic("unimplemented")
39+
}
40+
41+
// Delete is unimplemented and panics.
42+
func (f *Client) Delete(context.Context, *datastore.Key) error {
43+
panic("unimplemented")
44+
}
45+
46+
// DeleteMulti is unimplemented and panics.
47+
func (f *Client) DeleteMulti(context.Context, []*datastore.Key) (err error) {
48+
panic("unimplemented")
49+
}
50+
51+
// Get loads the entity stored for key into dst, which must be a struct pointer.
52+
func (f *Client) Get(_ context.Context, key *datastore.Key, dst interface{}) (err error) {
53+
if f == nil {
54+
return datastore.ErrNoSuchEntity
55+
}
56+
if dst == nil { // get catches nil interfaces; we need to catch nil ptr here
57+
return datastore.ErrInvalidEntityType
58+
}
59+
kdb := f.db[key.Kind]
60+
if kdb == nil {
61+
return datastore.ErrNoSuchEntity
62+
}
63+
rv := reflect.ValueOf(dst)
64+
if rv.Kind() != reflect.Ptr {
65+
return datastore.ErrInvalidEntityType
66+
}
67+
rd := rv.Elem()
68+
v := kdb[key.Encode()]
69+
if v == nil {
70+
return datastore.ErrNoSuchEntity
71+
}
72+
rd.Set(reflect.ValueOf(v).Elem())
73+
return nil
74+
}
75+
76+
// GetAll runs the provided query in the given context and returns all keys that match that query,
77+
// as well as appending the values to dst.
78+
//
79+
// GetAll currently only supports a query of all entities of a given Kind, and a dst of a slice of pointers to structs.
80+
func (f *Client) GetAll(_ context.Context, q *datastore.Query, dst interface{}) (keys []*datastore.Key, err error) {
81+
fv := reflect.ValueOf(q).Elem().FieldByName("kind")
82+
kdb := f.db[fv.String()]
83+
if kdb == nil {
84+
return
85+
}
86+
s := reflect.ValueOf(dst).Elem()
87+
for k, v := range kdb {
88+
dk, err := datastore.DecodeKey(k)
89+
if err != nil {
90+
log.Printf("f.GetAll() failed to decode key %q: %v", k, err)
91+
continue
92+
}
93+
keys = append(keys, dk)
94+
// This value is expected to represent a slice of pointers to structs.
95+
// ev := reflect.New(s.Type().Elem().Elem())
96+
// json.Unmarshal(v, ev.Interface())
97+
s.Set(reflect.Append(s, reflect.ValueOf(v)))
98+
}
99+
return
100+
}
101+
102+
// GetMulti is unimplemented and panics.
103+
func (f *Client) GetMulti(context.Context, []*datastore.Key, interface{}) (err error) {
104+
panic("unimplemented")
105+
}
106+
107+
// Mutate is unimplemented and panics.
108+
func (f *Client) Mutate(context.Context, ...*datastore.Mutation) (ret []*datastore.Key, err error) {
109+
panic("unimplemented")
110+
}
111+
112+
// NewTransaction is unimplemented and panics.
113+
func (f *Client) NewTransaction(context.Context, ...datastore.TransactionOption) (t dsiface.Transaction, err error) {
114+
panic("unimplemented")
115+
}
116+
117+
// Put saves the entity src into the datastore with the given key. src must be a struct pointer.
118+
func (f *Client) Put(_ context.Context, key *datastore.Key, src interface{}) (*datastore.Key, error) {
119+
if f.db == nil {
120+
f.db = make(map[string]map[string]interface{})
121+
}
122+
kdb := f.db[key.Kind]
123+
if kdb == nil {
124+
f.db[key.Kind] = make(map[string]interface{})
125+
kdb = f.db[key.Kind]
126+
}
127+
kdb[key.Encode()] = src
128+
return key, nil
129+
}
130+
131+
// PutMulti is unimplemented and panics.
132+
func (f *Client) PutMulti(context.Context, []*datastore.Key, interface{}) (ret []*datastore.Key, err error) {
133+
panic("unimplemented")
134+
}
135+
136+
// Run is unimplemented and panics.
137+
func (f *Client) Run(context.Context, *datastore.Query) dsiface.Iterator {
138+
panic("unimplemented")
139+
}
140+
141+
// RunInTransaction is unimplemented and panics.
142+
func (f *Client) RunInTransaction(context.Context, func(tx dsiface.Transaction) error, ...datastore.TransactionOption) (cmt dsiface.Commit, err error) {
143+
panic("unimplemented")
144+
}

‎cmd/relui/internal/datastore/fake/client_test.go

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
// Copyright 2020 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
package fake
6+
7+
import (
8+
"context"
9+
"testing"
10+
"time"
11+
12+
"cloud.google.com/go/datastore"
13+
"github.com/google/go-cmp/cmp"
14+
)
15+
16+
type author struct {
17+
Name string
18+
}
19+
20+
func TestClientGet(t *testing.T) {
21+
cases := []struct {
22+
desc string
23+
db map[string]map[string]interface{}
24+
key *datastore.Key
25+
dst interface{}
26+
want *author
27+
wantErr bool
28+
}{
29+
{
30+
desc: "correct key",
31+
db: map[string]map[string]interface{}{
32+
"Author": {datastore.NameKey("Author", "The Trial", nil).Encode(): &author{Name: "Kafka"}},
33+
},
34+
key: datastore.NameKey("Author", "The Trial", nil),
35+
dst: new(author),
36+
want: &author{Name: "Kafka"},
37+
},
38+
{
39+
desc: "incorrect key errors",
40+
db: map[string]map[string]interface{}{
41+
"Author": {datastore.NameKey("Author", "The Trial", nil).Encode(): &author{Name: "Kafka"}},
42+
},
43+
key: datastore.NameKey("Author", "The Go Programming Language", nil),
44+
dst: new(author),
45+
wantErr: true,
46+
},
47+
{
48+
desc: "nil dst errors",
49+
db: map[string]map[string]interface{}{
50+
"Author": {datastore.NameKey("Author", "The Trial", nil).Encode(): &author{Name: "Kafka"}},
51+
},
52+
key: datastore.NameKey("Author", "The Go Programming Language", nil),
53+
wantErr: true,
54+
},
55+
{
56+
desc: "incorrect dst type errors",
57+
db: map[string]map[string]interface{}{
58+
"Author": {datastore.NameKey("Author", "The Trial", nil).Encode(): &author{Name: "Kafka"}},
59+
},
60+
key: datastore.NameKey("Author", "The Go Programming Language", nil),
61+
dst: &time.Time{},
62+
wantErr: true,
63+
},
64+
{
65+
desc: "non-pointer dst errors",
66+
db: map[string]map[string]interface{}{
67+
"Author": {datastore.NameKey("Author", "The Trial", nil).Encode(): &author{Name: "Kafka"}},
68+
},
69+
key: datastore.NameKey("Author", "The Go Programming Language", nil),
70+
dst: author{},
71+
wantErr: true,
72+
},
73+
{
74+
desc: "nil dst errors",
75+
db: map[string]map[string]interface{}{
76+
"Author": {datastore.NameKey("Author", "The Trial", nil).Encode(): &author{Name: "Kafka"}},
77+
},
78+
key: datastore.NameKey("Author", "The Go Programming Language", nil),
79+
dst: nil,
80+
wantErr: true,
81+
},
82+
{
83+
desc: "empty db errors",
84+
key: datastore.NameKey("Author", "The Go Programming Language", nil),
85+
dst: nil,
86+
wantErr: true,
87+
},
88+
}
89+
for _, c := range cases {
90+
t.Run(c.desc, func(t *testing.T) {
91+
cl := &Client{db: c.db}
92+
93+
if err := cl.Get(context.Background(), c.key, c.dst); (err != nil) != c.wantErr {
94+
t.Fatalf("cl.Get(_, %v, %v) = %q, wantErr: %v", c.key, c.dst, err, c.wantErr)
95+
}
96+
if c.wantErr {
97+
return
98+
}
99+
if diff := cmp.Diff(c.want, c.dst); diff != "" {
100+
t.Errorf("author mismatch (-want +got):\n%s", diff)
101+
}
102+
})
103+
}
104+
}
105+
106+
func TestClientGetAll(t *testing.T) {
107+
cases := []struct {
108+
desc string
109+
db map[string]map[string]interface{}
110+
query *datastore.Query
111+
want []*author
112+
wantKeys []*datastore.Key
113+
wantErr bool
114+
}{
115+
{
116+
desc: "all of a Kind",
117+
db: map[string]map[string]interface{}{
118+
"Author": {datastore.NameKey("Author", "The Trial", nil).Encode(): &author{Name: "Kafka"}},
119+
},
120+
query: datastore.NewQuery("Author"),
121+
wantKeys: []*datastore.Key{datastore.NameKey("Author", "The Trial", nil)},
122+
want: []*author{{Name: "Kafka"}},
123+
},
124+
{
125+
desc: "all of a non-existent kind",
126+
db: map[string]map[string]interface{}{
127+
"Author": {datastore.NameKey("Author", "The Trial", nil).Encode(): &author{Name: "Kafka"}},
128+
},
129+
query: datastore.NewQuery("Book"),
130+
wantErr: false,
131+
},
132+
}
133+
for _, c := range cases {
134+
t.Run(c.desc, func(t *testing.T) {
135+
cl := &Client{db: c.db}
136+
137+
var got []*author
138+
keys, err := cl.GetAll(context.Background(), c.query, &got)
139+
if (err != nil) != c.wantErr {
140+
t.Fatalf("cl.Getall(_, %v, %v) = %q, wantErr: %v", c.query, got, err, c.wantErr)
141+
}
142+
if diff := cmp.Diff(c.want, got); diff != "" {
143+
t.Errorf("authors mismatch (-want +got):\n%s", diff)
144+
}
145+
if diff := cmp.Diff(c.wantKeys, keys); diff != "" {
146+
t.Errorf("keys mismatch (-want +got):\n%s", diff)
147+
}
148+
})
149+
}
150+
}
151+
152+
func TestClientPut(t *testing.T) {
153+
cl := &Client{}
154+
src := &author{Name: "Kafka"}
155+
key := datastore.NameKey("Author", "The Trial", nil)
156+
157+
gotKey, err := cl.Put(context.Background(), key, src)
158+
if err != nil {
159+
t.Fatalf("cl.Put(_, %v, %v) = %v, %q, wanted no error", gotKey, key, src, err)
160+
}
161+
got := cl.db["Author"][key.Encode()]
162+
163+
if diff := cmp.Diff(src, got); diff != "" {
164+
t.Errorf("author mismatch (-want +got):\n%s", diff)
165+
}
166+
if diff := cmp.Diff(key, gotKey); diff != "" {
167+
t.Errorf("keys mismatch (-want +got):\n%s", diff)
168+
}
169+
}

‎cmd/relui/main.go

Lines changed: 10 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -14,29 +14,31 @@ import (
1414
"os"
1515
"path/filepath"
1616

17+
"cloud.google.com/go/datastore"
1718
"cloud.google.com/go/pubsub"
1819
"github.com/golang/protobuf/proto"
20+
"github.com/googleapis/google-cloud-go-testing/datastore/dsiface"
1921
reluipb "golang.org/x/build/cmd/relui/protos"
2022
"google.golang.org/grpc/codes"
2123
"google.golang.org/grpc/status"
2224
)
2325

2426
var (
25-
devDataDir = flag.String("dev-data-directory", defaultDevDataDir(), "Development-only directory to use for storage of application state.")
26-
projectID = flag.String("project-id", os.Getenv("PUBSUB_PROJECT_ID"), "Pubsub project ID for communicating with workers. Uses PUBSUB_PROJECT_ID if unset.")
27-
topicID = flag.String("topic-id", "relui-development", "Pubsub topic ID for communicating with workers.")
27+
projectID = flag.String("project-id", os.Getenv("PUBSUB_PROJECT_ID"), "Pubsub project ID for communicating with workers. Uses PUBSUB_PROJECT_ID if unset.")
28+
topicID = flag.String("topic-id", "relui-development", "Pubsub topic ID for communicating with workers.")
2829
)
2930

3031
func main() {
3132
flag.Parse()
32-
fs := newFileStore(*devDataDir)
33-
if err := fs.load(); err != nil {
34-
log.Fatalf("Error loading state from %q: %v", *devDataDir, err)
35-
}
3633
ctx := context.Background()
34+
dsc, err := datastore.NewClient(ctx, *projectID)
35+
if err != nil {
36+
log.Fatalf("datastore.NewClient(_, %q) = _, %v, wanted no error", *projectID, err)
37+
}
38+
d := &dsStore{client: dsiface.AdaptClient(dsc)}
3739
s := &server{
3840
configs: loadWorkflowConfig("./workflows"),
39-
store: fs,
41+
store: d,
4042
topic: getTopic(ctx),
4143
}
4244
http.Handle("/workflows/create", http.HandlerFunc(s.createWorkflowHandler))
@@ -91,12 +93,3 @@ func loadWorkflowConfig(dir string) []*reluipb.Workflow {
9193
}
9294
return ws
9395
}
94-
95-
// defaultDevDataDir returns a directory suitable for storage of data when developing relui on most platforms.
96-
func defaultDevDataDir() string {
97-
c, err := os.UserConfigDir()
98-
if err != nil {
99-
c = os.TempDir()
100-
}
101-
return filepath.Join(c, "go-build", "relui")
102-
}

0 commit comments

Comments
 (0)