From 7bc9135ac9bb7c2ed839eacd7be7960c5af92b81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= <vtigerstrom@gmail.com> Date: Wed, 30 Apr 2025 00:12:39 +0700 Subject: [PATCH 1/4] session: update NewTestDB funcs to return Store In preparation for upcoming migration tests from a kvdb to an SQL store, this commit updates the NewTestDB function to return the Store interface rather than a concrete store implementation. This change ensures that migration tests can call NewTestDB under any build tag while receiving a consistent return type. --- session/test_kvdb.go | 8 ++++---- session/test_postgres.go | 4 ++-- session/test_sql.go | 2 +- session/test_sqlite.go | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/session/test_kvdb.go b/session/test_kvdb.go index 241448410..cc939159d 100644 --- a/session/test_kvdb.go +++ b/session/test_kvdb.go @@ -11,14 +11,14 @@ import ( ) // NewTestDB is a helper function that creates an BBolt database for testing. -func NewTestDB(t *testing.T, clock clock.Clock) *BoltStore { +func NewTestDB(t *testing.T, clock clock.Clock) Store { return NewTestDBFromPath(t, t.TempDir(), clock) } // NewTestDBFromPath is a helper function that creates a new BoltStore with a // connection to an existing BBolt database for testing. func NewTestDBFromPath(t *testing.T, dbPath string, - clock clock.Clock) *BoltStore { + clock clock.Clock) Store { acctStore := accounts.NewTestDB(t, clock) @@ -28,13 +28,13 @@ func NewTestDBFromPath(t *testing.T, dbPath string, // NewTestDBWithAccounts creates a new test session Store with access to an // existing accounts DB. func NewTestDBWithAccounts(t *testing.T, clock clock.Clock, - acctStore accounts.Store) *BoltStore { + acctStore accounts.Store) Store { return newDBFromPathWithAccounts(t, clock, t.TempDir(), acctStore) } func newDBFromPathWithAccounts(t *testing.T, clock clock.Clock, dbPath string, - acctStore accounts.Store) *BoltStore { + acctStore accounts.Store) Store { store, err := NewDB(dbPath, DBFilename, clock, acctStore) require.NoError(t, err) diff --git a/session/test_postgres.go b/session/test_postgres.go index db392fe7f..decf3bcc2 100644 --- a/session/test_postgres.go +++ b/session/test_postgres.go @@ -15,14 +15,14 @@ import ( var ErrDBClosed = errors.New("database is closed") // NewTestDB is a helper function that creates an SQLStore database for testing. -func NewTestDB(t *testing.T, clock clock.Clock) *SQLStore { +func NewTestDB(t *testing.T, clock clock.Clock) Store { return NewSQLStore(db.NewTestPostgresDB(t).BaseDB, clock) } // NewTestDBFromPath is a helper function that creates a new SQLStore with a // connection to an existing postgres database for testing. func NewTestDBFromPath(t *testing.T, dbPath string, - clock clock.Clock) *SQLStore { + clock clock.Clock) Store { return NewSQLStore(db.NewTestPostgresDB(t).BaseDB, clock) } diff --git a/session/test_sql.go b/session/test_sql.go index ab4b32a6c..ceb02194c 100644 --- a/session/test_sql.go +++ b/session/test_sql.go @@ -11,7 +11,7 @@ import ( ) func NewTestDBWithAccounts(t *testing.T, clock clock.Clock, - acctStore accounts.Store) *SQLStore { + acctStore accounts.Store) Store { accounts, ok := acctStore.(*accounts.SQLStore) require.True(t, ok) diff --git a/session/test_sqlite.go b/session/test_sqlite.go index 87519f4f1..dccbefe85 100644 --- a/session/test_sqlite.go +++ b/session/test_sqlite.go @@ -15,14 +15,14 @@ import ( var ErrDBClosed = errors.New("database is closed") // NewTestDB is a helper function that creates an SQLStore database for testing. -func NewTestDB(t *testing.T, clock clock.Clock) *SQLStore { +func NewTestDB(t *testing.T, clock clock.Clock) Store { return NewSQLStore(db.NewTestSqliteDB(t).BaseDB, clock) } // NewTestDBFromPath is a helper function that creates a new SQLStore with a // connection to an existing sqlite database for testing. func NewTestDBFromPath(t *testing.T, dbPath string, - clock clock.Clock) *SQLStore { + clock clock.Clock) Store { return NewSQLStore( db.NewTestSqliteDbHandleFromPath(t, dbPath).BaseDB, clock, From db050ba7da823f005994ed87f01587bc1c63d45a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= <vtigerstrom@gmail.com> Date: Wed, 30 Apr 2025 00:18:43 +0700 Subject: [PATCH 2/4] session: fix kvdb & sql session store difference When a session has a MacaroonRecipe set, which has a nil value set for either the `Permissions` or the `Caveats` field, the kvdb session store would return that field as nil, while the sql prior to this commit would return an empty array. When the we migrate the session store from kvdb to sql, that will cause the kvdb and the migrate session store to differ when comparing them, which will cause the migration to fail. This commit fixes that by ensuring that the sql session store returns a nil value for those fields in that scenario, rather than an empty array. --- session/sql_store.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/session/sql_store.go b/session/sql_store.go index b1d366fe7..3b95c42f6 100644 --- a/session/sql_store.go +++ b/session/sql_store.go @@ -745,9 +745,12 @@ func unmarshalSession(ctx context.Context, db SQLQueries, var macRecipe *MacaroonRecipe if perms != nil || caveats != nil { - macRecipe = &MacaroonRecipe{ - Permissions: unmarshalMacPerms(perms), - Caveats: unmarshalMacCaveats(caveats), + macRecipe = &MacaroonRecipe{} + if perms != nil { + macRecipe.Permissions = unmarshalMacPerms(perms) + } + if caveats != nil { + macRecipe.Caveats = unmarshalMacCaveats(caveats) } } From cf82fa2ce8202d48119de259dd6203b11562ec75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= <vtigerstrom@gmail.com> Date: Wed, 30 Apr 2025 00:42:35 +0700 Subject: [PATCH 3/4] session: add migration code from kvdb to SQL This commit introduces the migration logic for transitioning the sessions store from kvdb to SQL. Note that as of this commit, the migration is not yet triggered by any production code, i.e. only tests execute the migration logic. --- go.mod | 4 +- session/sql_migration.go | 403 ++++++++++++++++++++++++++++++++++ session/sql_migration_test.go | 337 ++++++++++++++++++++++++++++ 3 files changed, 742 insertions(+), 2 deletions(-) create mode 100644 session/sql_migration.go create mode 100644 session/sql_migration_test.go diff --git a/go.mod b/go.mod index c647825a7..2ee741358 100644 --- a/go.mod +++ b/go.mod @@ -35,11 +35,13 @@ require ( github.com/lightningnetwork/lnd/fn v1.2.3 github.com/lightningnetwork/lnd/fn/v2 v2.0.8 github.com/lightningnetwork/lnd/kvdb v1.4.16 + github.com/lightningnetwork/lnd/sqldb v1.0.9 github.com/lightningnetwork/lnd/tlv v1.3.0 github.com/lightningnetwork/lnd/tor v1.1.6 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f github.com/mwitkow/grpc-proxy v0.0.0-20230212185441-f345521cb9c9 github.com/ory/dockertest/v3 v3.10.0 + github.com/pmezard/go-difflib v1.0.0 github.com/stretchr/testify v1.10.0 github.com/urfave/cli v1.22.14 go.etcd.io/bbolt v1.3.11 @@ -143,7 +145,6 @@ require ( github.com/lightningnetwork/lightning-onion v1.2.1-0.20240712235311-98bd56499dfb // indirect github.com/lightningnetwork/lnd/healthcheck v1.2.6 // indirect github.com/lightningnetwork/lnd/queue v1.1.1 // indirect - github.com/lightningnetwork/lnd/sqldb v1.0.9 // indirect github.com/lightningnetwork/lnd/ticker v1.1.1 // indirect github.com/ltcsuite/ltcd v0.0.0-20190101042124-f37f8bf35796 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -161,7 +162,6 @@ require ( github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/runc v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.37.0 // indirect diff --git a/session/sql_migration.go b/session/sql_migration.go new file mode 100644 index 000000000..91bbd9036 --- /dev/null +++ b/session/sql_migration.go @@ -0,0 +1,403 @@ +package session + +import ( + "context" + "database/sql" + "errors" + "fmt" + "reflect" + "time" + + "github.com/davecgh/go-spew/spew" + "github.com/lightninglabs/lightning-terminal/accounts" + "github.com/lightninglabs/lightning-terminal/db/sqlc" + "github.com/pmezard/go-difflib/difflib" + "gopkg.in/macaroon-bakery.v2/bakery" + "gopkg.in/macaroon.v2" +) + +var ( + // ErrMigrationMismatch is returned when the migrated session does not + // match the original session. + ErrMigrationMismatch = fmt.Errorf("migrated session does not match " + + "original session") +) + +// MigrateSessionStoreToSQL runs the migration of all sessions from the KV +// database to the SQL database. The migration is done in a single transaction +// to ensure that all sessions are migrated or none at all. +// +// NOTE: As sessions may contain linked accounts, the accounts sql migration +// MUST be run prior to this migration. +func MigrateSessionStoreToSQL(ctx context.Context, kvStore *BoltStore, + tx SQLQueries) error { + + log.Infof("Starting migration of the KV sessions store to SQL") + + kvSessions, err := kvStore.ListAllSessions(ctx) + if err != nil { + return err + } + + // If sessions are linked to a group, we must insert the initial session + // of each group before the other sessions in that group. This ensures + // we can retrieve the SQL group ID when inserting the remaining + // sessions. Therefore, we first insert all initial group sessions, + // allowing us to fetch the group IDs and insert the rest of the + // sessions afterward. + // We therefore filter out the initial sessions first, and then migrate + // them prior to the rest of the sessions. + var ( + initialGroupSessions []*Session + linkedSessions []*Session + ) + + for _, kvSession := range kvSessions { + if kvSession.GroupID == kvSession.ID { + initialGroupSessions = append( + initialGroupSessions, kvSession, + ) + } else { + linkedSessions = append(linkedSessions, kvSession) + } + } + + err = migrateSessionsToSQLAndValidate(ctx, tx, initialGroupSessions) + if err != nil { + return err + } + + err = migrateSessionsToSQLAndValidate(ctx, tx, linkedSessions) + if err != nil { + return err + } + + total := len(initialGroupSessions) + len(linkedSessions) + + log.Infof("All sessions migrated from KV to SQL. Total number of "+ + "sessions migrated: %d", total) + + return nil +} + +// migrateSessionsToSQLAndValidate runs the migration for the passed sessions +// from the KV database to the SQL database, and validates that the migrated +// sessions match the original sessions. +func migrateSessionsToSQLAndValidate(ctx context.Context, + tx SQLQueries, kvSessions []*Session) error { + + for i, kvSession := range kvSessions { + err := migrateSingleSessionToSQL(ctx, tx, kvSessions[i]) + if err != nil { + return fmt.Errorf("unable to migrate session(%v): %w", + kvSession.ID, err) + } + + // Validate that the session was correctly migrated and matches + // the original session in the kv store. + sqlSess, err := tx.GetSessionByAlias(ctx, kvSession.ID[:]) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + err = ErrSessionNotFound + } + return fmt.Errorf("unable to get migrated session "+ + "from sql store: %w", err) + } + + migratedSession, err := unmarshalSession(ctx, tx, sqlSess) + if err != nil { + return fmt.Errorf("unable to unmarshal migrated "+ + "session: %w", err) + } + + overrideSessionTimeZone(kvSession) + overrideSessionTimeZone(migratedSession) + + if !reflect.DeepEqual(kvSession, migratedSession) { + diff := difflib.UnifiedDiff{ + A: difflib.SplitLines( + spew.Sdump(kvSession), + ), + B: difflib.SplitLines( + spew.Sdump(migratedSession), + ), + FromFile: "Expected", + FromDate: "", + ToFile: "Actual", + ToDate: "", + Context: 3, + } + diffText, _ := difflib.GetUnifiedDiffString(diff) + + return fmt.Errorf("%w: %v.\n%v", ErrMigrationMismatch, + kvSession.ID, diffText) + } + } + + return nil +} + +// migrateSingleSessionToSQL runs the migration for a single session from the +// KV database to the SQL database. Note that if the session links to an +// account, the linked accounts store MUST have been migrated before that +// session is migrated. +func migrateSingleSessionToSQL(ctx context.Context, + tx SQLQueries, session *Session) error { + + var ( + acctID sql.NullInt64 + err error + ) + + session.AccountID.WhenSome(func(alias accounts.AccountID) { + // Check that the account exists in the SQL store, before + // linking it. + var acctAlias int64 + acctAlias, err = alias.ToInt64() + if err != nil { + return + } + + var acctDBID int64 + acctDBID, err = tx.GetAccountIDByAlias(ctx, acctAlias) + if errors.Is(err, sql.ErrNoRows) { + err = accounts.ErrAccNotFound + return + } else if err != nil { + return + } + + acctID = sql.NullInt64{ + Int64: acctDBID, + Valid: true, + } + }) + if err != nil { + return err + } + + // First lets insert the session into the sql db. + insertSessionParams, err := makeInsertSessionParams(session, acctID) + if err != nil { + return err + } + + sqlId, err := tx.InsertSession(ctx, insertSessionParams) + if err != nil { + return err + } + + // Since the InsertSession query doesn't support that we set the revoked + // field during the insert, we need to set the field after the session + // has been created. + if !session.RevokedAt.IsZero() { + err = tx.SetSessionRevokedAt( + ctx, + makeSetRevokedAtParams(sqlId, session.RevokedAt), + ) + if err != nil { + return err + } + } + + // After the session has been inserted, we need to update the session + // with the group ID if it is linked to a group. We need to do this + // after the session has been inserted, because the group ID can be the + // session itself, and therefore the SQL id for the session won't exist + // prior to inserting the session. + groupID, err := tx.GetSessionIDByAlias(ctx, session.GroupID[:]) + if errors.Is(err, sql.ErrNoRows) { + return ErrUnknownGroup + } else if err != nil { + return fmt.Errorf("unable to fetch group(%x): %w", + session.GroupID[:], err) + } + + // Now lets set the group ID for the session. + err = tx.SetSessionGroupID(ctx, sqlc.SetSessionGroupIDParams{ + ID: sqlId, + GroupID: sql.NullInt64{ + Int64: groupID, + Valid: true, + }, + }) + if err != nil { + return fmt.Errorf("unable to set group Alias: %w", err) + } + + // Once we have the sqlID for the session, we can proceed to insert rows + // into the linked child tables. + if session.MacaroonRecipe != nil { + // We start by inserting the macaroon permissions. + for _, sessionPerm := range session.MacaroonRecipe.Permissions { + permParams := makeInsertMacaroonPermissionParams( + sqlId, sessionPerm, + ) + + err = tx.InsertSessionMacaroonPermission( + ctx, permParams, + ) + if err != nil { + return err + } + } + + // Next we insert the macaroon caveats. + for _, sessCaveat := range session.MacaroonRecipe.Caveats { + caveatParams := makeInsertMacaroonCaveatParams( + sqlId, sessCaveat, + ) + + err = tx.InsertSessionMacaroonCaveat( + ctx, caveatParams, + ) + if err != nil { + return err + } + } + } + + // That's followed by the feature config. + if session.FeatureConfig != nil { + for featureName, config := range *session.FeatureConfig { + fConfParams := makeInsertFeatureConfigParams( + sqlId, featureName, config, + ) + + err = tx.InsertSessionFeatureConfig(ctx, fConfParams) + if err != nil { + return err + } + } + } + + // Finally we insert the privacy flags. + for _, privacyFlag := range session.PrivacyFlags { + privacyFlagParams := makeInsertPrivacyFlagParams( + sqlId, int32(privacyFlag), + ) + + err = tx.InsertSessionPrivacyFlag( + ctx, privacyFlagParams, + ) + if err != nil { + return err + } + } + + return nil +} + +// overrideSessionTimeZone overrides the time zone of the session to the local +// time zone and chops off the nanosecond part for comparison. This is needed +// because KV database stores times as-is which as an unwanted side effect would +// fail migration due to time comparison expecting both the original and +// migrated sessions to be in the same local time zone and in microsecond +// precision. Note that PostgresSQL stores times in microsecond precision while +// SQLite can store times in nanosecond precision if using TEXT storage class. +func overrideSessionTimeZone(session *Session) { + fixTime := func(t time.Time) time.Time { + return t.In(time.Local).Truncate(time.Microsecond) + } + + if !session.Expiry.IsZero() { + session.Expiry = fixTime(session.Expiry) + } + + if !session.CreatedAt.IsZero() { + session.CreatedAt = fixTime(session.CreatedAt) + } + + if !session.RevokedAt.IsZero() { + session.RevokedAt = fixTime(session.RevokedAt) + } +} + +func makeInsertSessionParams(session *Session, acctID sql.NullInt64) ( + sqlc.InsertSessionParams, error) { + + var remotePubKey []byte + + // The remote public key is currently only set for autopilot sessions, + // else it's an empty byte array. + if session.RemotePublicKey != nil { + remotePubKey = session.RemotePublicKey.SerializeCompressed() + } + + params := sqlc.InsertSessionParams{ + Alias: session.ID[:], + Label: session.Label, + State: int16(session.State), + Type: int16(session.Type), + Expiry: session.Expiry.UTC(), + CreatedAt: session.CreatedAt.UTC(), + ServerAddress: session.ServerAddr, + DevServer: session.DevServer, + MacaroonRootKey: int64(session.MacaroonRootKey), + PairingSecret: session.PairingSecret[:], + LocalPrivateKey: session.LocalPrivateKey.Serialize(), + LocalPublicKey: session.LocalPublicKey.SerializeCompressed(), + RemotePublicKey: remotePubKey, + Privacy: session.WithPrivacyMapper, + AccountID: acctID, + } + + return params, nil +} + +func makeSetRevokedAtParams(sqlID int64, + revokedAt time.Time) sqlc.SetSessionRevokedAtParams { + + return sqlc.SetSessionRevokedAtParams{ + ID: sqlID, + RevokedAt: sql.NullTime{ + Time: revokedAt.UTC(), + Valid: true, + }, + } +} + +func makeInsertMacaroonPermissionParams(sqlID int64, + permission bakery.Op) sqlc.InsertSessionMacaroonPermissionParams { + + return sqlc.InsertSessionMacaroonPermissionParams{ + SessionID: sqlID, + Entity: permission.Entity, + Action: permission.Action, + } +} + +func makeInsertMacaroonCaveatParams(sqlID int64, + caveat macaroon.Caveat) sqlc.InsertSessionMacaroonCaveatParams { + + location := sql.NullString{ + String: caveat.Location, + Valid: caveat.Location != "", + } + + return sqlc.InsertSessionMacaroonCaveatParams{ + SessionID: sqlID, + CaveatID: caveat.Id, + VerificationID: caveat.VerificationId, + Location: location, + } +} + +func makeInsertFeatureConfigParams(sqlID int64, name string, + config []byte) sqlc.InsertSessionFeatureConfigParams { + + return sqlc.InsertSessionFeatureConfigParams{ + SessionID: sqlID, + FeatureName: name, + Config: config, + } +} + +func makeInsertPrivacyFlagParams(sqlID int64, + privacyFlag int32) sqlc.InsertSessionPrivacyFlagParams { + + return sqlc.InsertSessionPrivacyFlagParams{ + SessionID: sqlID, + Flag: privacyFlag, + } +} diff --git a/session/sql_migration_test.go b/session/sql_migration_test.go new file mode 100644 index 000000000..fd9d78134 --- /dev/null +++ b/session/sql_migration_test.go @@ -0,0 +1,337 @@ +package session + +import ( + "context" + "database/sql" + "fmt" + "testing" + "time" + + "github.com/lightninglabs/lightning-terminal/accounts" + "github.com/lightninglabs/lightning-terminal/db" + "github.com/lightningnetwork/lnd/clock" + "github.com/lightningnetwork/lnd/macaroons" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/stretchr/testify/require" + "gopkg.in/macaroon-bakery.v2/bakery/checkers" + "gopkg.in/macaroon.v2" +) + +// TestSessionsStoreMigration tests the migration of session store from a bolt +// backed to a SQL database. Note that this test does not attempt to be a +// complete migration test. +func TestSessionsStoreMigration(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + // When using build tags that creates a kvdb store for NewTestDB, we + // skip this test as it is only applicable for postgres and sqlite tags. + store := NewTestDB(t, clock.NewTestClock(time.Now())) + if _, ok := store.(*BoltStore); ok { + t.Skipf("Skipping session store migration test for kvdb build") + } + + makeSQLDB := func(t *testing.T, acctStore accounts.Store, + clock *clock.TestClock) (*SQLStore, + *db.TransactionExecutor[SQLQueries]) { + + // Create a sql store with a linked account store. + testDBStore := NewTestDBWithAccounts(t, clock, acctStore) + store, ok := testDBStore.(*SQLStore) + require.True(t, ok) + + baseDB := store.BaseDB + + genericExecutor := db.NewTransactionExecutor( + baseDB, func(tx *sql.Tx) SQLQueries { + return baseDB.WithTx(tx) + }, + ) + + return store, genericExecutor + } + + migrationTest := func(t *testing.T, kvStore *BoltStore, + acctStore accounts.Store, clock *clock.TestClock) { + + sqlStore, txEx := makeSQLDB(t, acctStore, clock) + + var opts sqldb.MigrationTxOptions + err := txEx.ExecTx( + ctx, &opts, func(tx SQLQueries) error { + return MigrateSessionStoreToSQL( + ctx, kvStore, tx, + ) + }, + ) + require.NoError(t, err) + + // MigrateSessionStoreToSQL will check if the inserted sessions + // equals the migrated ones, but as a sanity check we'll also + // fetch the sessions from the store and compare them to the + // original. + kvSessions, err := kvStore.ListAllSessions(ctx) + require.NoError(t, err) + + for _, kvSession := range kvSessions { + // Fetch the migrated session from the sql store. + sqlSession, err := sqlStore.GetSession( + ctx, kvSession.ID, + ) + require.NoError(t, err) + + assertEqualSessions(t, kvSession, sqlSession) + } + + // Finally we ensure that the sql store doesn't contain more + // sessions than the kv store. + sqlSessions, err := sqlStore.ListAllSessions(ctx) + require.NoError(t, err) + require.Equal(t, len(kvSessions), len(sqlSessions)) + } + + tests := []struct { + name string + populateDB func( + t *testing.T, kvStore *BoltStore, + accountStore accounts.Store, + ) + }{ + { + "empty", + func(t *testing.T, store *BoltStore, _ accounts.Store) { + // Don't populate the DB. + }, + }, + { + "one session no options", + func(t *testing.T, store *BoltStore, _ accounts.Store) { + _, err := store.NewSession( + ctx, "test", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + ) + require.NoError(t, err) + }, + }, + { + "multiple sessions no options", + func(t *testing.T, store *BoltStore, _ accounts.Store) { + _, err := store.NewSession( + ctx, "session1", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + ) + require.NoError(t, err) + + _, err = store.NewSession( + ctx, "session2", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + ) + require.NoError(t, err) + + _, err = store.NewSession( + ctx, "session3", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + ) + require.NoError(t, err) + }, + }, + { + "one session with one privacy flag", + func(t *testing.T, store *BoltStore, _ accounts.Store) { + _, err := store.NewSession( + ctx, "test", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + WithPrivacy(PrivacyFlags{ClearPubkeys}), + ) + require.NoError(t, err) + }, + }, + { + "one session with multiple privacy flags", + func(t *testing.T, store *BoltStore, _ accounts.Store) { + _, err := store.NewSession( + ctx, "test", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + WithPrivacy(PrivacyFlags{ + ClearChanInitiator, ClearHTLCs, + ClearClosingTxIds, + }), + ) + require.NoError(t, err) + }, + }, + { + "one session with a feature config", + func(t *testing.T, store *BoltStore, _ accounts.Store) { + featureConfig := map[string][]byte{ + "AutoFees": {1, 2, 3, 4}, + "AutoSomething": {4, 3, 4, 5, 6, 6}, + } + + _, err := store.NewSession( + ctx, "test", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + WithFeatureConfig(featureConfig), + ) + require.NoError(t, err) + }, + }, + { + "one session with dev server", + func(t *testing.T, store *BoltStore, _ accounts.Store) { + _, err := store.NewSession( + ctx, "test", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + WithDevServer(), + ) + require.NoError(t, err) + }, + }, + { + "one session with macaroon recipe", + func(t *testing.T, store *BoltStore, _ accounts.Store) { + // this test uses caveats & perms from the + // tlv_test.go + _, err := store.NewSession( + ctx, "test", TypeMacaroonAdmin, + time.Unix(1000, 0), "foo.bar.baz:1234", + WithMacaroonRecipe(caveats, perms), + ) + require.NoError(t, err) + }, + }, + { + "one session with macaroon recipe nil caveats", + func(t *testing.T, store *BoltStore, _ accounts.Store) { + // this test uses perms from the tlv_test.go + _, err := store.NewSession( + ctx, "test", TypeMacaroonAdmin, + time.Unix(1000, 0), "foo.bar.baz:1234", + WithMacaroonRecipe(nil, perms), + ) + require.NoError(t, err) + }, + }, + { + "one session with macaroon recipe nil perms", + func(t *testing.T, store *BoltStore, _ accounts.Store) { + // this test uses caveats from the tlv_test.go + _, err := store.NewSession( + ctx, "test", TypeMacaroonAdmin, + time.Unix(1000, 0), "foo.bar.baz:1234", + WithMacaroonRecipe(caveats, nil), + ) + require.NoError(t, err) + }, + }, + { + "one session with a linked account", + func( + t *testing.T, store *BoltStore, + acctStore accounts.Store, + ) { + + // Create an account with balance + acct, err := acctStore.NewAccount( + ctx, 1234, time.Now().Add(time.Hour), + "", + ) + require.NoError(t, err) + require.False(t, acct.HasExpired()) + + // For now, we manually add the account caveat + // for bbolt compatibility. + accountCaveat := checkers.Condition( + macaroons.CondLndCustom, + fmt.Sprintf("%s %x", + accounts.CondAccount, + acct.ID[:], + ), + ) + + sessCaveats := []macaroon.Caveat{ + { + Id: []byte(accountCaveat), + }, + } + + _, err = store.NewSession( + ctx, "test", TypeMacaroonAccount, + time.Unix(1000, 0), "", + WithAccount(acct.ID), + WithMacaroonRecipe(sessCaveats, nil), + ) + require.NoError(t, err) + }, + }, + { + "linked session", + func(t *testing.T, store *BoltStore, _ accounts.Store) { + // First create the initial session for the + // group. + sess1, err := store.NewSession( + ctx, "initSession", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + ) + require.NoError(t, err) + + // As the store won't allow us to link a + // session before all sessions in the group have + // been revoked, we revoke the session before + // creating a new session that links to the + // initial session. + err = store.ShiftState( + ctx, sess1.ID, StateCreated, + ) + require.NoError(t, err) + + err = store.ShiftState( + ctx, sess1.ID, StateRevoked, + ) + require.NoError(t, err) + + _, err = store.NewSession( + ctx, "linkedSession", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + WithLinkedGroupID(&sess1.ID), + ) + require.NoError(t, err) + }, + }, + } + + for _, test := range tests { + tc := test + + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + var kvStore *BoltStore + clock := clock.NewTestClock(time.Now()) + + // First let's create an account store to link to in + // the sessions store. Note that this is will be a sql + // store due to the build tags enabled when running this + // test, which means that we won't need to migrate the + // account store in this test. + accountStore := accounts.NewTestDB(t, clock) + t.Cleanup(func() { + require.NoError(t, accountStore.Close()) + }) + + kvStore, err := NewDB( + t.TempDir(), DBFilename, clock, accountStore, + ) + require.NoError(t, err) + + t.Cleanup(func() { + require.NoError(t, kvStore.DB.Close()) + }) + + tc.populateDB(t, kvStore, accountStore) + + migrationTest(t, kvStore, accountStore, clock) + }) + } +} From a868a934293ded34766fa93f33fbbe7db5c1f94f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= <vtigerstrom@gmail.com> Date: Wed, 30 Apr 2025 00:43:26 +0700 Subject: [PATCH 4/4] session: add randomized session migration test --- session/sql_migration_test.go | 401 ++++++++++++++++++++++++++++++++++ 1 file changed, 401 insertions(+) diff --git a/session/sql_migration_test.go b/session/sql_migration_test.go index fd9d78134..a52f0af86 100644 --- a/session/sql_migration_test.go +++ b/session/sql_migration_test.go @@ -10,9 +10,13 @@ import ( "github.com/lightninglabs/lightning-terminal/accounts" "github.com/lightninglabs/lightning-terminal/db" "github.com/lightningnetwork/lnd/clock" + "github.com/lightningnetwork/lnd/lnwire" "github.com/lightningnetwork/lnd/macaroons" "github.com/lightningnetwork/lnd/sqldb" "github.com/stretchr/testify/require" + "go.etcd.io/bbolt" + "golang.org/x/exp/rand" + "gopkg.in/macaroon-bakery.v2/bakery" "gopkg.in/macaroon-bakery.v2/bakery/checkers" "gopkg.in/macaroon.v2" ) @@ -299,6 +303,10 @@ func TestSessionsStoreMigration(t *testing.T) { require.NoError(t, err) }, }, + { + "randomized sessions", + randomizedSessions, + }, } for _, test := range tests { @@ -335,3 +343,396 @@ func TestSessionsStoreMigration(t *testing.T) { }) } } + +// randomizedSessions adds 100 randomized sessions to the kvStore, where 25% of +// them will contain up to 10 linked sessions. The rest of the session will have +// the rest of the session options randomized. +func randomizedSessions(t *testing.T, kvStore *BoltStore, + accountsStore accounts.Store) { + + ctx := context.Background() + + var ( + // numberOfSessions is set to 100 to add enough sessions to get + // enough variation between number of invoices and payments, but + // kept low enough for the test not take too long to run, as the + // test time increases drastically by the number of sessions we + // migrate. + numberOfSessions = 100 + ) + + for i := 0; i < numberOfSessions; i++ { + var ( + opts []Option + serverAddr string + ) + macType := macaroonType(i) + expiry := time.Unix(rand.Int63n(10000), rand.Int63n(10000)) + label := fmt.Sprintf("session%d", i+1) + + // Half of the sessions will get a set server address. + if rand.Intn(2) == 0 { + serverAddr = "foo.bar.baz:1234" + } + + // Every 10th session will get no added options. + if i%10 != 0 { + // Add random privacy flags to 50% of the sessions. + if rand.Intn(2) == 0 { + opts = append( + opts, + WithPrivacy(randomPrivacyFlags()), + ) + } + + // Add random feature configs to 50% of the sessions. + if rand.Intn(2) == 0 { + opts = append( + opts, + WithFeatureConfig( + randomFeatureConfig(), + ), + ) + } + + // Set that the session uses a dev server for 50% of the + // sessions. + if rand.Intn(2) == 0 { + opts = append(opts, WithDevServer()) + } + + // Add a random macaroon recipe to 50% of the sessions. + if rand.Intn(2) == 0 { + // In 50% of those cases, we add a random + // macaroon recipe with caveats and perms, + // and for the other 50% we added a linked + // account with the correct macaroon recipe (to + // simulate realistic data). + if rand.Intn(2) == 0 { + opts = append( + opts, randomMacaroonRecipe(), + ) + } else { + acctOpts := randomAccountOptions( + ctx, t, accountsStore, + ) + + opts = append(opts, acctOpts...) + } + } + } + + // We insert the session with the randomized params and options. + activeSess, err := kvStore.NewSession( + ctx, label, macType, expiry, serverAddr, opts..., + ) + require.NoError(t, err) + + // For 25% of the sessions, we link a random number of sessions + // to the session. + if rand.Intn(4) == 0 { + // Link up to 10 sessions to the session, and set the + // same opts as the initial group session. + for j := 0; j < rand.Intn(10); j++ { + // We first need to revoke the previous session + // before we can create a new session that links + // to the session. + err = kvStore.ShiftState( + ctx, activeSess.ID, StateCreated, + ) + require.NoError(t, err) + + err = kvStore.ShiftState( + ctx, activeSess.ID, StateRevoked, + ) + require.NoError(t, err) + + opts = []Option{ + WithLinkedGroupID(&activeSess.GroupID), + } + + if activeSess.DevServer { + opts = append(opts, WithDevServer()) + } + + if activeSess.FeatureConfig != nil { + opts = append(opts, WithFeatureConfig( + *activeSess.FeatureConfig, + )) + } + + if activeSess.PrivacyFlags != nil { + opts = append(opts, WithPrivacy( + activeSess.PrivacyFlags, + )) + } + + if activeSess.MacaroonRecipe != nil { + macRec := activeSess.MacaroonRecipe + opts = append(opts, WithMacaroonRecipe( + macRec.Caveats, + macRec.Permissions, + )) + } + + activeSess.AccountID.WhenSome( + func(alias accounts.AccountID) { + opts = append( + opts, + WithAccount(alias), + ) + }, + ) + + label = fmt.Sprintf("linkedSession%d", j+1) + + activeSess, err = kvStore.NewSession( + ctx, label, activeSess.Type, + time.Unix(1000, 0), + activeSess.ServerAddr, opts..., + ) + require.NoError(t, err) + } + } + + // Finally, we shift the active session to a random state. + // As the state we set may be a state that's no longer set + // through the current code base, or be an illegal state + // transition, we use an alternative test state shifting method + // that doesn't check that we transition the state in the legal + // order. + err = kvStore.shiftStateUnsafe(ctx, activeSess.ID, lastState(i)) + require.NoError(t, err) + } +} + +// macaroonType returns a macaroon type based on the given index by taking the +// index modulo 6. This ensures an approximately equal distribution of macaroon +// types. +func macaroonType(i int) Type { + switch i % 6 { + case 0: + return TypeMacaroonReadonly + case 1: + return TypeMacaroonAdmin + case 2: + return TypeMacaroonCustom + case 3: + return TypeUIPassword + case 4: + return TypeAutopilot + default: + return TypeMacaroonAccount + } +} + +// lastState returns a state based on the given index by taking the index modulo +// 5. This ensures an approximately equal distribution of states. +func lastState(i int) State { + switch i % 5 { + case 0: + return StateCreated + case 1: + return StateInUse + case 2: + return StateRevoked + case 3: + return StateExpired + default: + return StateReserved + } +} + +// randomPrivacyFlags returns a random set of privacy flags. +func randomPrivacyFlags() PrivacyFlags { + allFlags := []PrivacyFlag{ + ClearPubkeys, + ClearChanIDs, + ClearTimeStamps, + ClearChanInitiator, + ClearHTLCs, + ClearClosingTxIds, + ClearNetworkAddresses, + } + + var privFlags []PrivacyFlag + for _, flag := range allFlags { + if rand.Intn(2) == 0 { + privFlags = append(privFlags, flag) + } + } + + return privFlags +} + +// randomFeatureConfig returns a random feature config with a random number of +// features. The feature names are generated as "feature0", "feature1", etc. +func randomFeatureConfig() FeaturesConfig { + featureConfig := make(FeaturesConfig) + for i := 0; i < rand.Intn(10); i++ { + featureName := fmt.Sprintf("feature%d", i) + featureValue := []byte{byte(rand.Int31())} + featureConfig[featureName] = featureValue + } + + return featureConfig +} + +// randomMacaroonRecipe returns a random macaroon recipe with a random number of +// caveats and permissions. The returned macaroon recipe may have nil set for +// either the caveats or permissions, but not both. +func randomMacaroonRecipe() Option { + var ( + macCaveats []macaroon.Caveat + macPerms []bakery.Op + ) + + loopLen := rand.Intn(10) + 1 + + if rand.Intn(2) == 0 { + for i := 0; i < loopLen; i++ { + var macCaveat macaroon.Caveat + + // We always have a caveat.Id, but the rest are + // randomized if they exist or not. + macCaveat.Id = randomBytes(rand.Intn(10) + 1) + + if rand.Intn(2) == 0 { + macCaveat.VerificationId = + randomBytes(rand.Intn(32) + 1) + } + + if rand.Intn(2) == 0 { + macCaveat.Location = + randomString(rand.Intn(10) + 1) + } + + macCaveats = append(macCaveats, macCaveat) + } + } else { + macCaveats = nil + } + + // We can't do both nil caveats and nil perms, so if we have nil + // caveats, we set perms to a value. + if rand.Intn(2) == 0 || macCaveats == nil { + for i := 0; i < loopLen; i++ { + var macPerm bakery.Op + + macPerm.Action = randomString(rand.Intn(10) + 1) + macPerm.Entity = randomString(rand.Intn(10) + 1) + + macPerms = append(macPerms, macPerm) + } + } else { + macPerms = nil + } + + return WithMacaroonRecipe(macCaveats, macPerms) +} + +// randomAccountOptions creates a random account with a random balance and +// expiry time, that's linked in the returned options. The returned options also +// returns the macaroon recipe with the account caveat. +func randomAccountOptions(ctx context.Context, t *testing.T, + acctStore accounts.Store) []Option { + + balance := lnwire.MilliSatoshi(rand.Int63()) + + // randomize expiry from 10 to 10,000 minutes + expiry := time.Now().Add( + time.Minute * time.Duration(rand.Intn(10000-10)+10), + ) + + // As the store has a unique constraint for inserting labels, we suffix + // it with a sufficiently large random number avoid collisions. + label := fmt.Sprintf("account:%d", rand.Int63()) + + // Create an account with balance + acct, err := acctStore.NewAccount( + ctx, balance, expiry, label, + ) + require.NoError(t, err) + require.False(t, acct.HasExpired()) + + // For now, we manually add the account caveat + // for bbolt compatibility. + accountCaveat := checkers.Condition( + macaroons.CondLndCustom, + fmt.Sprintf("%s %x", + accounts.CondAccount, + acct.ID[:], + ), + ) + + sessCaveats := []macaroon.Caveat{} + sessCaveats = append( + sessCaveats, + macaroon.Caveat{ + Id: []byte(accountCaveat), + }, + ) + + opts := []Option{ + WithAccount(acct.ID), + WithMacaroonRecipe(sessCaveats, nil), + } + + return opts +} + +// randomBytes generates a random byte array of the passed length n. +func randomBytes(n int) []byte { + b := make([]byte, n) + for i := range b { + b[i] = byte(rand.Intn(256)) // Random int between 0-255, then cast to byte + } + return b +} + +// randomString generates a random string of the passed length n. +func randomString(n int) string { + letterBytes := "abcdefghijklmnopqrstuvwxyz" + + b := make([]byte, n) + for i := range b { + b[i] = letterBytes[rand.Intn(len(letterBytes))] + } + return string(b) +} + +// shiftStateUnsafe updates the state of the session with the given ID to the +// "dest" state, without checking if the state transition is legal. +// +// NOTE: this function should only be used for testing purposes. +func (db *BoltStore) shiftStateUnsafe(_ context.Context, id ID, + dest State) error { + + return db.Update(func(tx *bbolt.Tx) error { + sessionBucket, err := getBucket(tx, sessionBucketKey) + if err != nil { + return err + } + + session, err := getSessionByID(sessionBucket, id) + if err != nil { + return err + } + + // If the session is already in the desired state, we return + // with no error to maintain idempotency. + if session.State == dest { + return nil + } + + session.State = dest + + // If the session is terminal, we set the revoked at time to the + // current time. + if dest.Terminal() { + session.RevokedAt = db.clock.Now().UTC() + } + + return putSession(sessionBucket, session) + }) +}