feat: use sqlc to write queries

cli/serve.go

@@ -10,7 +10,7 @@ import (
 	"github.com/goto/entropy/core"
 	"github.com/goto/entropy/core/module"
 	entropyserver "github.com/goto/entropy/internal/server"
-	"github.com/goto/entropy/internal/store/postgres"
+	"github.com/goto/entropy/internal/store/pgsql"
 	"github.com/goto/entropy/modules"
 	"github.com/goto/entropy/modules/firehose"
 	"github.com/goto/entropy/modules/job"
@@ -96,8 +96,8 @@ func setupRegistry() module.Registry {
 	return registry
 }
 
-func setupStorage(pgConStr string, syncCfg syncerConf) *postgres.Store {
-	store, err := postgres.Open(pgConStr, syncCfg.RefreshInterval, syncCfg.ExtendLockBy)
+func setupStorage(pgConStr string, syncCfg syncerConf) *pgsql.Store {
+	store, err := pgsql.Open(pgConStr, syncCfg.RefreshInterval, syncCfg.ExtendLockBy)
 	if err != nil {
 		zap.L().Fatal("failed to connect to Postgres database",
 			zap.Error(err), zap.String("conn_str", pgConStr))

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/goto/salt v0.3.3
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0
+	github.com/jackc/pgx/v5 v5.4.3
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/lib/pq v1.10.9
 	github.com/mcuadros/go-defaults v1.2.0
@@ -202,6 +203,8 @@ require (
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/newrelic/csec-go-agent v0.4.0 // indirect
 	github.com/newrelic/newrelic-telemetry-sdk-go v0.8.1 // indirect

go.sum

@@ -912,6 +912,7 @@ github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfG
 github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds=
 github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
 github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78=
 github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA=
@@ -923,6 +924,8 @@ github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX
 github.com/jackc/pgproto3/v2 v2.0.7/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
 github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
 github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
 github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg=
 github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
 github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw=
@@ -937,6 +940,8 @@ github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXg
 github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o=
 github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg=
 github.com/jackc/pgx/v4 v4.10.1/go.mod h1:QlrWebbs3kqEZPHCTGyxecvzG6tvIsYu+A5b1raylkA=
+github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY=
+github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA=
 github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=

internal/store/pgsql/modules.go

@@ -0,0 +1,77 @@
+package pgsql
+
+import (
+	"context"
+
+	"github.com/goto/entropy/core/module"
+	"github.com/goto/entropy/internal/store/pgsql/queries"
+)
+
+func (st *Store) GetModule(ctx context.Context, urn string) (*module.Module, error) {
+	mod, err := st.qu.GetModuleByURN(ctx, urn)
+	if err != nil {
+		return nil, translateSQLErr(err)
+	}
+
+	return &module.Module{
+		URN:       mod.Urn,
+		Name:      mod.Name,
+		Project:   mod.Project,
+		Configs:   mod.Configs,
+		CreatedAt: mod.CreatedAt.Time,
+		UpdatedAt: mod.UpdatedAt.Time,
+	}, nil
+}
+
+func (st *Store) ListModules(ctx context.Context, project string) ([]module.Module, error) {
+	mods, err := st.qu.ListAllModulesForProject(ctx, project)
+	if err != nil {
+		return nil, translateSQLErr(err)
+	}
+
+	var modules []module.Module
+	for _, mod := range mods {
+		modules = append(modules, module.Module{
+			URN:       mod.Urn,
+			Name:      mod.Name,
+			Project:   mod.Project,
+			Configs:   mod.Configs,
+			CreatedAt: mod.CreatedAt.Time,
+			UpdatedAt: mod.UpdatedAt.Time,
+		})
+	}
+
+	return modules, nil
+}
+
+func (st *Store) CreateModule(ctx context.Context, m module.Module) error {
+	params := queries.InsertModuleParams{
+		Urn:     m.URN,
+		Project: m.Project,
+		Name:    m.Name,
+		Configs: m.Configs,
+	}
+
+	if err := st.qu.InsertModule(ctx, params); err != nil {
+		return translateSQLErr(err)
+	}
+	return nil
+}
+
+func (st *Store) UpdateModule(ctx context.Context, m module.Module) error {
+	params := queries.UpdateModuleParams{
+		Urn:     m.URN,
+		Configs: m.Configs,
+	}
+	if err := st.qu.UpdateModule(ctx, params); err != nil {
+		return translateSQLErr(err)
+	}
+	return nil
+}
+
+func (st *Store) DeleteModule(ctx context.Context, urn string) error {
+	if err := st.qu.DeleteModule(ctx, urn); err != nil {
+		return translateSQLErr(err)
+	}
+	return nil
+}

internal/store/pgsql/pgsql.go

@@ -0,0 +1,132 @@
+package pgsql
+
+import (
+	"context"
+	"database/sql"
+	_ "embed"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	_ "github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgconn"
+
+	"github.com/goto/entropy/core/resource"
+	"github.com/goto/entropy/internal/store/pgsql/queries"
+	"github.com/goto/entropy/pkg/errors"
+)
+
+//go:generate sqlc generate
+
+// schema represents the storage schema.
+// Note: Update the constants above if the table name is changed.
+//
+//go:embed schema.sql
+var schema string
+
+type Store struct {
+	qu              *queries.Queries
+	pgx             *pgx.Conn
+	extendInterval  time.Duration
+	refreshInterval time.Duration
+}
+
+func (st *Store) Migrate(ctx context.Context) error {
+	err := st.qu.Migrate(ctx, schema)
+	return err
+}
+
+func (st *Store) Close() error { return st.pgx.Close(context.Background()) }
+
+// Open returns store instance backed by PostgresQL.
+func Open(conStr string, refreshInterval, extendInterval time.Duration) (*Store, error) {
+	conn, err := pgx.Connect(context.Background(), conStr)
+	if err != nil {
+		return nil, err
+	} else if err := conn.Ping(context.Background()); err != nil {
+		_ = conn.Close(context.Background())
+		return nil, err
+	}
+
+	if refreshInterval >= extendInterval {
+		return nil, errors.New("refreshInterval must be lower than extendInterval")
+	}
+
+	return &Store{
+		qu:              queries.New(conn),
+		pgx:             conn,
+		extendInterval:  extendInterval,
+		refreshInterval: refreshInterval,
+	}, nil
+}
+
+func translateSQLErr(err error) error {
+	if errors.Is(err, sql.ErrNoRows) {
+		return errors.ErrNotFound.WithCausef(err.Error())
+	}
+
+	var pgErr *pgconn.PgError
+	if errors.As(err, &pgErr) {
+		// Refer http://www.postgresql.org/docs/9.3/static/errcodes-appendix.html
+		switch pgErr.Code {
+		case "unique_violation":
+			return errors.ErrConflict.WithCausef(err.Error())
+
+		default:
+			return errors.ErrInternal.WithCausef(err.Error())
+		}
+	}
+
+	return err
+}
+
+func tagsToLabelMap(tags []string) map[string]string {
+	const keyValueParts = 2
+
+	labels := map[string]string{}
+	for _, tag := range tags {
+		parts := strings.SplitN(tag, "=", keyValueParts)
+		key, val := parts[0], parts[1]
+		labels[key] = val
+	}
+	return labels
+}
+
+func labelToTag(k, v string) string {
+	return fmt.Sprintf("%s=%s", k, v)
+}
+
+type TxFunc func(ctx context.Context, tx pgx.Tx) error
+
+func withinTx(ctx context.Context, db *pgx.Conn, readOnly bool, fns ...TxFunc) error {
+	var opts pgx.TxOptions
+	if readOnly {
+		opts.AccessMode = pgx.ReadOnly
+	} else {
+		opts.AccessMode = pgx.ReadWrite
+	}
+
+	tx, err := db.BeginTx(ctx, opts)
+	if err != nil {
+		return err
+	}
+
+	for _, fn := range fns {
+		if err := fn(ctx, tx); err != nil {
+			_ = tx.Rollback(ctx)
+			return err
+		}
+	}
+
+	return tx.Commit(ctx)
+}
+
+func runAllHooks(ctx context.Context, hooks []resource.MutationHook) error {
+	for _, hook := range hooks {
+		if err := hook(ctx); err != nil {
+			return err
+		}
+	}
+	return nil
+}