Merge pull request #955 from ellemouton/sql8Accounts8

[sql-8]: run itests against all DB types

Author: ellemouton

.github/workflows/main.yml

@@ -275,6 +275,17 @@ jobs:
   itest:
     name: integration test
     runs-on: ubuntu-latest
+    strategy:
+      # Allow other tests in the matrix to continue if one fails.
+      fail-fast: false
+      matrix:
+        include:
+          - name: bbolt
+            args: dbbackend=bbolt
+          - name: sqlite
+            args: dbbackend=sqlite
+          - name: postgres
+            args: dbbackend=postgres
     steps:
       - name: git checkout
         uses: actions/checkout@v4
@@ -295,8 +306,8 @@ jobs:
         working-directory: ./app
         run: yarn
 
-      - name: run check
-        run: make itest
+      - name: run itest ${{ matrix.name }}
+        run: make itest ${{ matrix.args }}
 
       - name: Zip log files on failure
         if: ${{ failure() }}

.golangci.yml

@@ -11,6 +11,7 @@ run:
     - watchtowerrpc
     - neutrinorpc
     - peersrpc
+    - dev
 
 linters-settings:
   govet:

config.go

@@ -237,6 +237,13 @@ type Config struct {
 	// over an in-memory connection on startup. This is only set in
 	// integrated lnd mode.
 	lndAdminMacaroon []byte
+
+	// DevConfig is a config struct that is empty if lit is built without
+	// the `dev` flag (in other words when it is build for a production
+	// environment). This allows us to have config values that are then
+	// only available in development mode which lets us run itests against
+	// features not yet available in production.
+	*DevConfig
 }
 
 // lndConnectParams returns the connection parameters to connect to the local
@@ -337,8 +344,9 @@ func defaultConfig() *Config {
 		Autopilot: &autopilotserver.Config{
 			PingCadence: time.Hour,
 		},
-		Firewall: firewall.DefaultConfig(),
-		Accounts: &accounts.Config{},
+		Firewall:  firewall.DefaultConfig(),
+		Accounts:  &accounts.Config{},
+		DevConfig: defaultDevConfig(),
 	}
 }
 
@@ -467,6 +475,11 @@ func loadAndValidateConfig(interceptor signal.Interceptor) (*Config, error) {
 		)
 	}
 
+	err = cfg.DevConfig.Validate(litDir, cfg.Network)
+	if err != nil {
+		return nil, err
+	}
+
 	// Initiate our listeners. For now, we only support listening on one
 	// port at a time because we can only pass in one pre-configured RPC
 	// listener into lnd.

config_dev.go

@@ -0,0 +1,117 @@
+//go:build dev
+
+package terminal
+
+import (
+	"path/filepath"
+
+	"github.com/lightninglabs/lightning-terminal/accounts"
+	"github.com/lightninglabs/lightning-terminal/db"
+	"github.com/lightningnetwork/lnd/clock"
+)
+
+const (
+	// DatabaseBackendSqlite is the name of the SQLite database backend.
+	DatabaseBackendSqlite = "sqlite"
+
+	// DatabaseBackendPostgres is the name of the Postgres database backend.
+	DatabaseBackendPostgres = "postgres"
+
+	// DatabaseBackendBbolt is the name of the bbolt database backend.
+	DatabaseBackendBbolt = "bbolt"
+
+	// defaultSqliteDatabaseFileName is the default name of the SQLite
+	// database file.
+	defaultSqliteDatabaseFileName = "litd.db"
+)
+
+// defaultSqliteDatabasePath is the default path under which we store
+// the SQLite database file.
+var defaultSqliteDatabasePath = filepath.Join(
+	DefaultLitDir, DefaultNetwork, defaultSqliteDatabaseFileName,
+)
+
+// DevConfig is a struct that holds the configuration options for a development
+// environment. The purpose of this struct is to hold config options for
+// features not yet available in production. Since our itests are built with
+// the dev tag, we can test these features in our itests.
+//
+// nolint:lll
+type DevConfig struct {
+	// DatabaseBackend is the database backend we will use for storing all
+	// account related data. While this feature is still in development, we
+	// include the bbolt type here so that our itests can continue to be
+	// tested against a bbolt backend. Once the full bbolt to SQL migration
+	// is complete, however, we will remove the bbolt option.
+	DatabaseBackend string `long:"databasebackend" description:"The database backend to use for storing all account related data." choice:"bbolt" choice:"sqlite" choice:"postgres"`
+
+	// Sqlite holds the configuration options for a SQLite database
+	// backend.
+	Sqlite *db.SqliteConfig `group:"sqlite" namespace:"sqlite"`
+
+	// Postgres holds the configuration options for a Postgres database
+	Postgres *db.PostgresConfig `group:"postgres" namespace:"postgres"`
+}
+
+// Validate checks that all the values set in our DevConfig are valid and uses
+// the passed parameters to override any defaults if necessary.
+func (c *DevConfig) Validate(dbDir, network string) error {
+	// We'll update the database file location if it wasn't set.
+	if c.Sqlite.DatabaseFileName == defaultSqliteDatabasePath {
+		c.Sqlite.DatabaseFileName = filepath.Join(
+			dbDir, network, defaultSqliteDatabaseFileName,
+		)
+	}
+
+	return nil
+}
+
+// defaultDevConfig returns a new DevConfig with default values set.
+func defaultDevConfig() *DevConfig {
+	return &DevConfig{
+		Sqlite: &db.SqliteConfig{
+			DatabaseFileName: defaultSqliteDatabasePath,
+		},
+		Postgres: &db.PostgresConfig{
+			Host:               "localhost",
+			Port:               5432,
+			MaxOpenConnections: 10,
+		},
+	}
+}
+
+// NewAccountStore creates a new account store based on the chosen database
+// backend.
+func NewAccountStore(cfg *Config, clock clock.Clock) (accounts.Store, error) {
+	switch cfg.DatabaseBackend {
+	case DatabaseBackendSqlite:
+		// Before we initialize the SQLite store, we'll make sure that
+		// the directory where we will store the database file exists.
+		networkDir := filepath.Join(cfg.LitDir, cfg.Network)
+		err := makeDirectories(networkDir)
+		if err != nil {
+			return nil, err
+		}
+
+		sqlStore, err := db.NewSqliteStore(cfg.Sqlite)
+		if err != nil {
+			return nil, err
+		}
+
+		return accounts.NewSQLStore(sqlStore.BaseDB, clock), nil
+
+	case DatabaseBackendPostgres:
+		sqlStore, err := db.NewPostgresStore(cfg.Postgres)
+		if err != nil {
+			return nil, err
+		}
+
+		return accounts.NewSQLStore(sqlStore.BaseDB, clock), nil
+
+	default:
+		return accounts.NewBoltStore(
+			filepath.Dir(cfg.MacaroonPath), accounts.DBFilename,
+			clock,
+		)
+	}
+}

config_prod.go

@@ -0,0 +1,33 @@
+//go:build !dev
+
+package terminal
+
+import (
+	"path/filepath"
+
+	"github.com/lightninglabs/lightning-terminal/accounts"
+	"github.com/lightningnetwork/lnd/clock"
+)
+
+// DevConfig is an empty shell struct that allows us to build without the dev
+// tag. This struct is embedded in the main Config struct, and it adds no new
+// functionality in a production build.
+type DevConfig struct{}
+
+// defaultDevConfig returns an empty DevConfig struct.
+func defaultDevConfig() *DevConfig {
+	return &DevConfig{}
+}
+
+// Validate is a no-op function during a production build.
+func (c *DevConfig) Validate(_, _ string) error {
+	return nil
+}
+
+// NewAccountStore creates a new account store using the default Bolt backend
+// since in production, this is the only backend supported currently.
+func NewAccountStore(cfg *Config, clock clock.Clock) (accounts.Store, error) {
+	return accounts.NewBoltStore(
+		filepath.Dir(cfg.MacaroonPath), accounts.DBFilename, clock,
+	)
+}

itest/litd_node.go

@@ -8,6 +8,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"errors"
+	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -25,6 +26,7 @@ import (
 	"github.com/btcsuite/btcd/wire"
 	"github.com/lightninglabs/faraday/frdrpc"
 	terminal "github.com/lightninglabs/lightning-terminal"
+	"github.com/lightninglabs/lightning-terminal/db"
 	"github.com/lightninglabs/lightning-terminal/litrpc"
 	"github.com/lightninglabs/lightning-terminal/subservers"
 	"github.com/lightninglabs/loop/looprpc"
@@ -60,7 +62,12 @@ var (
 	numActiveNodes    = 0
 	numActiveNodesMtx sync.Mutex
 
-	defaultLndPassphrase = []byte("default-wallet-password")
+	// litDBBackend is a command line flag for specifying the database
+	// backend to use when starting a LiT daemon.
+	litDBBackend = flag.String(
+		"litdbbackend", terminal.DatabaseBackendBbolt, "Set the "+
+			"database backend to use when starting a LiT daemon.",
+	)
 )
 
 type LitNodeConfig struct {
@@ -80,6 +87,9 @@ type LitNodeConfig struct {
 	LitTLSCertPath string
 	LitMacPath     string
 
+	DBBackend      string
+	PostgresConfig *db.PostgresConfig
+
 	UIPassword string
 	LitDir     string
 	FaradayDir string
@@ -220,8 +230,20 @@ func (cfg *LitNodeConfig) defaultLitdArgs() *litArgs {
 			"restcors":               "*",
 			"lnd.debuglevel":         "trace,GRPC=error,PEER=info",
 			"lndconnectinterval":     "200ms",
+			"databasebackend":        cfg.DBBackend,
 		}
 	)
+
+	if cfg.DBBackend == terminal.DatabaseBackendPostgres {
+		args["postgres.host"] = cfg.PostgresConfig.Host
+		args["postgres.port"] = fmt.Sprintf(
+			"%d", cfg.PostgresConfig.Port,
+		)
+		args["postgres.user"] = cfg.PostgresConfig.User
+		args["postgres.password"] = cfg.PostgresConfig.Password
+		args["postgres.dbname"] = cfg.PostgresConfig.DBName
+	}
+
 	for _, arg := range cfg.LitArgs {
 		parts := strings.Split(arg, "=")
 		option := strings.TrimLeft(parts[0], "--")
@@ -417,6 +439,28 @@ func NewNode(t *testing.T, cfg *LitNodeConfig,
 	cfg.LitTLSCertPath = filepath.Join(cfg.LitDir, "tls.cert")
 	cfg.GenerateListeningPorts()
 
+	// Decide which DB backend to use.
+	switch *litDBBackend {
+	case terminal.DatabaseBackendSqlite:
+		cfg.DBBackend = terminal.DatabaseBackendSqlite
+
+	case terminal.DatabaseBackendPostgres:
+		fixture := db.NewTestPgFixture(
+			t, db.DefaultPostgresFixtureLifetime, true,
+		)
+		t.Cleanup(func() {
+			fixture.TearDown(t)
+		})
+
+		cfg.DBBackend = terminal.DatabaseBackendPostgres
+		cfg.PostgresConfig = fixture.GetConfig()
+
+	default:
+		cfg.DBBackend = terminal.DatabaseBackendBbolt
+	}
+
+	t.Logf("Using %v database backend", cfg.DBBackend)
+
 	// Generate a random UI password by reading 16 random bytes and base64
 	// encoding them.
 	var randomBytes [16]byte

make/testing_flags.mk

@@ -1,6 +1,5 @@
 include make/compile_flags.mk
 
-ITEST_FLAGS =
 TEST_FLAGS =
 DEV_TAGS = dev
 
@@ -9,6 +8,11 @@ ifneq ($(icase),)
 ITEST_FLAGS += -test.run="TestLightningTerminal/$(icase)"
 endif
 
+# Run itests with specified db backend.
+ifneq ($(dbbackend),)
+ITEST_FLAGS += -litdbbackend=$(dbbackend)
+endif
+
 # If a specific unit test case is being targeted, construct test.run filter.
 ifneq ($(case),)
 TEST_FLAGS += -test.run=$(case)

terminal.go

@@ -216,7 +216,7 @@ type LightningTerminal struct {
 	middleware        *mid.Manager
 	middlewareStarted bool
 
-	accountsStore         *accounts.BoltStore
+	accountsStore         accounts.Store
 	accountService        *accounts.InterceptorService
 	accountServiceStarted bool
 
@@ -415,10 +415,14 @@ func (g *LightningTerminal) start(ctx context.Context) error {
 		)
 	}
 
-	g.accountsStore, err = accounts.NewBoltStore(
-		filepath.Dir(g.cfg.MacaroonPath), accounts.DBFilename,
-		clock.NewDefaultClock(),
-	)
+	networkDir := filepath.Join(g.cfg.LitDir, g.cfg.Network)
+	err = makeDirectories(networkDir)
+	if err != nil {
+		return fmt.Errorf("could not create network directory: %v", err)
+	}
+
+	clock := clock.NewDefaultClock()
+	g.accountsStore, err = NewAccountStore(g.cfg, clock)
 	if err != nil {
 		return fmt.Errorf("error creating accounts store: %w", err)
 	}
@@ -445,10 +449,7 @@ func (g *LightningTerminal) start(ctx context.Context) error {
 	g.ruleMgrs = rules.NewRuleManagerSet()
 
 	// Create an instance of the local Terminal Connect session store DB.
-	networkDir := filepath.Join(g.cfg.LitDir, g.cfg.Network)
-	g.sessionDB, err = session.NewDB(
-		networkDir, session.DBFilename, clock.NewDefaultClock(),
-	)
+	g.sessionDB, err = session.NewDB(networkDir, session.DBFilename, clock)
 	if err != nil {
 		return fmt.Errorf("error creating session DB: %v", err)
 	}