From 7a091fc773c206f56a0b9af20b7b8fe424133555 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Mon, 6 Jan 2025 11:38:04 +0200 Subject: [PATCH 1/4] config: introduce the concept of DevConfig In this commit, we introduce a DevConfig struct which is embedded in the main lit Config struct. It is defined in two separate files that are made mutually exclusive using the `dev` build tag. The idea of this struct is that it allows us to expose additional config options to LiT if we build in a development environment such as our itests. In other words, we will be able to start developing features and testing them before making them available to users via the production build. As of this commit, both implementations of the struct are the same. --- .golangci.yml | 1 + config.go | 17 +++++++++++++++-- config_dev.go | 38 ++++++++++++++++++++++++++++++++++++++ config_prod.go | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 config_dev.go create mode 100644 config_prod.go diff --git a/.golangci.yml b/.golangci.yml index 25bf2ec12..62cf7fa17 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -11,6 +11,7 @@ run: - watchtowerrpc - neutrinorpc - peersrpc + - dev linters-settings: govet: diff --git a/config.go b/config.go index ec800a2e9..4e201f27f 100644 --- a/config.go +++ b/config.go @@ -237,6 +237,13 @@ type Config struct { // over an in-memory connection on startup. This is only set in // integrated lnd mode. lndAdminMacaroon []byte + + // DevConfig is a config struct that is empty if lit is built without + // the `dev` flag (in other words when it is build for a production + // environment). This allows us to have config values that are then + // only available in development mode which lets us run itests against + // features not yet available in production. + *DevConfig } // lndConnectParams returns the connection parameters to connect to the local @@ -337,8 +344,9 @@ func defaultConfig() *Config { Autopilot: &autopilotserver.Config{ PingCadence: time.Hour, }, - Firewall: firewall.DefaultConfig(), - Accounts: &accounts.Config{}, + Firewall: firewall.DefaultConfig(), + Accounts: &accounts.Config{}, + DevConfig: defaultDevConfig(), } } @@ -467,6 +475,11 @@ func loadAndValidateConfig(interceptor signal.Interceptor) (*Config, error) { ) } + err = cfg.DevConfig.Validate(litDir, cfg.Network) + if err != nil { + return nil, err + } + // Initiate our listeners. For now, we only support listening on one // port at a time because we can only pass in one pre-configured RPC // listener into lnd. diff --git a/config_dev.go b/config_dev.go new file mode 100644 index 000000000..93aca3416 --- /dev/null +++ b/config_dev.go @@ -0,0 +1,38 @@ +//go:build dev + +package terminal + +import ( + "path/filepath" + + "github.com/lightninglabs/lightning-terminal/accounts" + "github.com/lightningnetwork/lnd/clock" +) + +// DevConfig is a struct that holds the configuration options for a development +// environment. The purpose of this struct is to hold config options for +// features not yet available in production. Since our itests are built with +// the dev tag, we can test these features in our itests. +// +// nolint:lll +type DevConfig struct { +} + +// Validate checks that all the values set in our DevConfig are valid and uses +// the passed parameters to override any defaults if necessary. +func (c *DevConfig) Validate(dbDir, network string) error { + return nil +} + +// defaultDevConfig returns a new DevConfig with default values set. +func defaultDevConfig() *DevConfig { + return &DevConfig{} +} + +// NewAccountStore creates a new account store based on the chosen database +// backend. +func NewAccountStore(cfg *Config, clock clock.Clock) (accounts.Store, error) { + return accounts.NewBoltStore( + filepath.Dir(cfg.MacaroonPath), accounts.DBFilename, clock, + ) +} diff --git a/config_prod.go b/config_prod.go new file mode 100644 index 000000000..3bb64e746 --- /dev/null +++ b/config_prod.go @@ -0,0 +1,33 @@ +//go:build !dev + +package terminal + +import ( + "path/filepath" + + "github.com/lightninglabs/lightning-terminal/accounts" + "github.com/lightningnetwork/lnd/clock" +) + +// DevConfig is an empty shell struct that allows us to build without the dev +// tag. This struct is embedded in the main Config struct, and it adds no new +// functionality in a production build. +type DevConfig struct{} + +// defaultDevConfig returns an empty DevConfig struct. +func defaultDevConfig() *DevConfig { + return &DevConfig{} +} + +// Validate is a no-op function during a production build. +func (c *DevConfig) Validate(_, _ string) error { + return nil +} + +// NewAccountStore creates a new account store using the default Bolt backend +// since in production, this is the only backend supported currently. +func NewAccountStore(cfg *Config, clock clock.Clock) (accounts.Store, error) { + return accounts.NewBoltStore( + filepath.Dir(cfg.MacaroonPath), accounts.DBFilename, clock, + ) +} From 381d7d17769bd242e2ca6051788ec1cfb5deb052 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Mon, 6 Jan 2025 11:44:18 +0200 Subject: [PATCH 2/4] config: Postgres and SQLite config options for accounts to dev build In this commit, we expand the dev build's DevConfig options struct with Postgres and SQL backend options that can be used to initialise the account store. --- config_dev.go | 87 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 4 deletions(-) diff --git a/config_dev.go b/config_dev.go index 93aca3416..28357852c 100644 --- a/config_dev.go +++ b/config_dev.go @@ -6,9 +6,31 @@ import ( "path/filepath" "github.com/lightninglabs/lightning-terminal/accounts" + "github.com/lightninglabs/lightning-terminal/db" "github.com/lightningnetwork/lnd/clock" ) +const ( + // DatabaseBackendSqlite is the name of the SQLite database backend. + DatabaseBackendSqlite = "sqlite" + + // DatabaseBackendPostgres is the name of the Postgres database backend. + DatabaseBackendPostgres = "postgres" + + // DatabaseBackendBbolt is the name of the bbolt database backend. + DatabaseBackendBbolt = "bbolt" + + // defaultSqliteDatabaseFileName is the default name of the SQLite + // database file. + defaultSqliteDatabaseFileName = "litd.db" +) + +// defaultSqliteDatabasePath is the default path under which we store +// the SQLite database file. +var defaultSqliteDatabasePath = filepath.Join( + DefaultLitDir, DefaultNetwork, defaultSqliteDatabaseFileName, +) + // DevConfig is a struct that holds the configuration options for a development // environment. The purpose of this struct is to hold config options for // features not yet available in production. Since our itests are built with @@ -16,23 +38,80 @@ import ( // // nolint:lll type DevConfig struct { + // DatabaseBackend is the database backend we will use for storing all + // account related data. While this feature is still in development, we + // include the bbolt type here so that our itests can continue to be + // tested against a bbolt backend. Once the full bbolt to SQL migration + // is complete, however, we will remove the bbolt option. + DatabaseBackend string `long:"databasebackend" description:"The database backend to use for storing all account related data." choice:"bbolt" choice:"sqlite" choice:"postgres"` + + // Sqlite holds the configuration options for a SQLite database + // backend. + Sqlite *db.SqliteConfig `group:"sqlite" namespace:"sqlite"` + + // Postgres holds the configuration options for a Postgres database + Postgres *db.PostgresConfig `group:"postgres" namespace:"postgres"` } // Validate checks that all the values set in our DevConfig are valid and uses // the passed parameters to override any defaults if necessary. func (c *DevConfig) Validate(dbDir, network string) error { + // We'll update the database file location if it wasn't set. + if c.Sqlite.DatabaseFileName == defaultSqliteDatabasePath { + c.Sqlite.DatabaseFileName = filepath.Join( + dbDir, network, defaultSqliteDatabaseFileName, + ) + } + return nil } // defaultDevConfig returns a new DevConfig with default values set. func defaultDevConfig() *DevConfig { - return &DevConfig{} + return &DevConfig{ + Sqlite: &db.SqliteConfig{ + DatabaseFileName: defaultSqliteDatabasePath, + }, + Postgres: &db.PostgresConfig{ + Host: "localhost", + Port: 5432, + MaxOpenConnections: 10, + }, + } } // NewAccountStore creates a new account store based on the chosen database // backend. func NewAccountStore(cfg *Config, clock clock.Clock) (accounts.Store, error) { - return accounts.NewBoltStore( - filepath.Dir(cfg.MacaroonPath), accounts.DBFilename, clock, - ) + switch cfg.DatabaseBackend { + case DatabaseBackendSqlite: + // Before we initialize the SQLite store, we'll make sure that + // the directory where we will store the database file exists. + networkDir := filepath.Join(cfg.LitDir, cfg.Network) + err := makeDirectories(networkDir) + if err != nil { + return nil, err + } + + sqlStore, err := db.NewSqliteStore(cfg.Sqlite) + if err != nil { + return nil, err + } + + return accounts.NewSQLStore(sqlStore.BaseDB, clock), nil + + case DatabaseBackendPostgres: + sqlStore, err := db.NewPostgresStore(cfg.Postgres) + if err != nil { + return nil, err + } + + return accounts.NewSQLStore(sqlStore.BaseDB, clock), nil + + default: + return accounts.NewBoltStore( + filepath.Dir(cfg.MacaroonPath), accounts.DBFilename, + clock, + ) + } } From b6891b4b38df45f83cb368098a4a7e0fd02d8686 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Sun, 5 Jan 2025 12:31:05 +0200 Subject: [PATCH 3/4] itest: run itests against chosen backend Add the ability to specify the db backend to run use for accounts when running itests. With this commit, you can run something like: `make itest dbbackend=sqlite`. --- itest/litd_node.go | 46 ++++++++++++++++++++++++++++++++++++++++++- make/testing_flags.mk | 6 +++++- terminal.go | 19 +++++++++--------- 3 files changed, 60 insertions(+), 11 deletions(-) diff --git a/itest/litd_node.go b/itest/litd_node.go index e9be55972..48cd6353a 100644 --- a/itest/litd_node.go +++ b/itest/litd_node.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "flag" "fmt" "io" "io/ioutil" @@ -25,6 +26,7 @@ import ( "github.com/btcsuite/btcd/wire" "github.com/lightninglabs/faraday/frdrpc" terminal "github.com/lightninglabs/lightning-terminal" + "github.com/lightninglabs/lightning-terminal/db" "github.com/lightninglabs/lightning-terminal/litrpc" "github.com/lightninglabs/lightning-terminal/subservers" "github.com/lightninglabs/loop/looprpc" @@ -60,7 +62,12 @@ var ( numActiveNodes = 0 numActiveNodesMtx sync.Mutex - defaultLndPassphrase = []byte("default-wallet-password") + // litDBBackend is a command line flag for specifying the database + // backend to use when starting a LiT daemon. + litDBBackend = flag.String( + "litdbbackend", terminal.DatabaseBackendBbolt, "Set the "+ + "database backend to use when starting a LiT daemon.", + ) ) type LitNodeConfig struct { @@ -80,6 +87,9 @@ type LitNodeConfig struct { LitTLSCertPath string LitMacPath string + DBBackend string + PostgresConfig *db.PostgresConfig + UIPassword string LitDir string FaradayDir string @@ -220,8 +230,20 @@ func (cfg *LitNodeConfig) defaultLitdArgs() *litArgs { "restcors": "*", "lnd.debuglevel": "trace,GRPC=error,PEER=info", "lndconnectinterval": "200ms", + "databasebackend": cfg.DBBackend, } ) + + if cfg.DBBackend == terminal.DatabaseBackendPostgres { + args["postgres.host"] = cfg.PostgresConfig.Host + args["postgres.port"] = fmt.Sprintf( + "%d", cfg.PostgresConfig.Port, + ) + args["postgres.user"] = cfg.PostgresConfig.User + args["postgres.password"] = cfg.PostgresConfig.Password + args["postgres.dbname"] = cfg.PostgresConfig.DBName + } + for _, arg := range cfg.LitArgs { parts := strings.Split(arg, "=") option := strings.TrimLeft(parts[0], "--") @@ -417,6 +439,28 @@ func NewNode(t *testing.T, cfg *LitNodeConfig, cfg.LitTLSCertPath = filepath.Join(cfg.LitDir, "tls.cert") cfg.GenerateListeningPorts() + // Decide which DB backend to use. + switch *litDBBackend { + case terminal.DatabaseBackendSqlite: + cfg.DBBackend = terminal.DatabaseBackendSqlite + + case terminal.DatabaseBackendPostgres: + fixture := db.NewTestPgFixture( + t, db.DefaultPostgresFixtureLifetime, true, + ) + t.Cleanup(func() { + fixture.TearDown(t) + }) + + cfg.DBBackend = terminal.DatabaseBackendPostgres + cfg.PostgresConfig = fixture.GetConfig() + + default: + cfg.DBBackend = terminal.DatabaseBackendBbolt + } + + t.Logf("Using %v database backend", cfg.DBBackend) + // Generate a random UI password by reading 16 random bytes and base64 // encoding them. var randomBytes [16]byte diff --git a/make/testing_flags.mk b/make/testing_flags.mk index 0370bc29f..f3a261b6f 100644 --- a/make/testing_flags.mk +++ b/make/testing_flags.mk @@ -1,6 +1,5 @@ include make/compile_flags.mk -ITEST_FLAGS = TEST_FLAGS = DEV_TAGS = dev @@ -9,6 +8,11 @@ ifneq ($(icase),) ITEST_FLAGS += -test.run="TestLightningTerminal/$(icase)" endif +# Run itests with specified db backend. +ifneq ($(dbbackend),) +ITEST_FLAGS += -litdbbackend=$(dbbackend) +endif + # If a specific unit test case is being targeted, construct test.run filter. ifneq ($(case),) TEST_FLAGS += -test.run=$(case) diff --git a/terminal.go b/terminal.go index 590403d10..e297d0fe1 100644 --- a/terminal.go +++ b/terminal.go @@ -216,7 +216,7 @@ type LightningTerminal struct { middleware *mid.Manager middlewareStarted bool - accountsStore *accounts.BoltStore + accountsStore accounts.Store accountService *accounts.InterceptorService accountServiceStarted bool @@ -415,10 +415,14 @@ func (g *LightningTerminal) start(ctx context.Context) error { ) } - g.accountsStore, err = accounts.NewBoltStore( - filepath.Dir(g.cfg.MacaroonPath), accounts.DBFilename, - clock.NewDefaultClock(), - ) + networkDir := filepath.Join(g.cfg.LitDir, g.cfg.Network) + err = makeDirectories(networkDir) + if err != nil { + return fmt.Errorf("could not create network directory: %v", err) + } + + clock := clock.NewDefaultClock() + g.accountsStore, err = NewAccountStore(g.cfg, clock) if err != nil { return fmt.Errorf("error creating accounts store: %w", err) } @@ -445,10 +449,7 @@ func (g *LightningTerminal) start(ctx context.Context) error { g.ruleMgrs = rules.NewRuleManagerSet() // Create an instance of the local Terminal Connect session store DB. - networkDir := filepath.Join(g.cfg.LitDir, g.cfg.Network) - g.sessionDB, err = session.NewDB( - networkDir, session.DBFilename, clock.NewDefaultClock(), - ) + g.sessionDB, err = session.NewDB(networkDir, session.DBFilename, clock) if err != nil { return fmt.Errorf("error creating session DB: %v", err) } From 5eba32816114a93f93897fd5d3f559b2b8bf3384 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Sun, 5 Jan 2025 12:41:48 +0200 Subject: [PATCH 4/4] .github: run itests against all types of account stores In this commit, we ensure that our CI runs our itests against all available DB backends. Currently, this will only affect the accounts store. --- .github/workflows/main.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 62393b938..a3442c8a5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -275,6 +275,17 @@ jobs: itest: name: integration test runs-on: ubuntu-latest + strategy: + # Allow other tests in the matrix to continue if one fails. + fail-fast: false + matrix: + include: + - name: bbolt + args: dbbackend=bbolt + - name: sqlite + args: dbbackend=sqlite + - name: postgres + args: dbbackend=postgres steps: - name: git checkout uses: actions/checkout@v4 @@ -295,8 +306,8 @@ jobs: working-directory: ./app run: yarn - - name: run check - run: make itest + - name: run itest ${{ matrix.name }} + run: make itest ${{ matrix.args }} - name: Zip log files on failure if: ${{ failure() }}