From b48e40f000918d216c1d28c9dcec241d507aea68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Wed, 30 Apr 2025 00:12:39 +0700 Subject: [PATCH 1/4] session: update NewTestDB funcs to return Store In preparation for upcoming migration tests from a kvdb to an SQL store, this commit updates the NewTestDB function to return the Store interface rather than a concrete store implementation. This change ensures that migration tests can call NewTestDB under any build tag while receiving a consistent return type. --- session/test_kvdb.go | 8 ++++---- session/test_postgres.go | 4 ++-- session/test_sql.go | 2 +- session/test_sqlite.go | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/session/test_kvdb.go b/session/test_kvdb.go index 241448410..cc939159d 100644 --- a/session/test_kvdb.go +++ b/session/test_kvdb.go @@ -11,14 +11,14 @@ import ( ) // NewTestDB is a helper function that creates an BBolt database for testing. -func NewTestDB(t *testing.T, clock clock.Clock) *BoltStore { +func NewTestDB(t *testing.T, clock clock.Clock) Store { return NewTestDBFromPath(t, t.TempDir(), clock) } // NewTestDBFromPath is a helper function that creates a new BoltStore with a // connection to an existing BBolt database for testing. func NewTestDBFromPath(t *testing.T, dbPath string, - clock clock.Clock) *BoltStore { + clock clock.Clock) Store { acctStore := accounts.NewTestDB(t, clock) @@ -28,13 +28,13 @@ func NewTestDBFromPath(t *testing.T, dbPath string, // NewTestDBWithAccounts creates a new test session Store with access to an // existing accounts DB. func NewTestDBWithAccounts(t *testing.T, clock clock.Clock, - acctStore accounts.Store) *BoltStore { + acctStore accounts.Store) Store { return newDBFromPathWithAccounts(t, clock, t.TempDir(), acctStore) } func newDBFromPathWithAccounts(t *testing.T, clock clock.Clock, dbPath string, - acctStore accounts.Store) *BoltStore { + acctStore accounts.Store) Store { store, err := NewDB(dbPath, DBFilename, clock, acctStore) require.NoError(t, err) diff --git a/session/test_postgres.go b/session/test_postgres.go index db392fe7f..decf3bcc2 100644 --- a/session/test_postgres.go +++ b/session/test_postgres.go @@ -15,14 +15,14 @@ import ( var ErrDBClosed = errors.New("database is closed") // NewTestDB is a helper function that creates an SQLStore database for testing. -func NewTestDB(t *testing.T, clock clock.Clock) *SQLStore { +func NewTestDB(t *testing.T, clock clock.Clock) Store { return NewSQLStore(db.NewTestPostgresDB(t).BaseDB, clock) } // NewTestDBFromPath is a helper function that creates a new SQLStore with a // connection to an existing postgres database for testing. func NewTestDBFromPath(t *testing.T, dbPath string, - clock clock.Clock) *SQLStore { + clock clock.Clock) Store { return NewSQLStore(db.NewTestPostgresDB(t).BaseDB, clock) } diff --git a/session/test_sql.go b/session/test_sql.go index ab4b32a6c..ceb02194c 100644 --- a/session/test_sql.go +++ b/session/test_sql.go @@ -11,7 +11,7 @@ import ( ) func NewTestDBWithAccounts(t *testing.T, clock clock.Clock, - acctStore accounts.Store) *SQLStore { + acctStore accounts.Store) Store { accounts, ok := acctStore.(*accounts.SQLStore) require.True(t, ok) diff --git a/session/test_sqlite.go b/session/test_sqlite.go index 87519f4f1..dccbefe85 100644 --- a/session/test_sqlite.go +++ b/session/test_sqlite.go @@ -15,14 +15,14 @@ import ( var ErrDBClosed = errors.New("database is closed") // NewTestDB is a helper function that creates an SQLStore database for testing. -func NewTestDB(t *testing.T, clock clock.Clock) *SQLStore { +func NewTestDB(t *testing.T, clock clock.Clock) Store { return NewSQLStore(db.NewTestSqliteDB(t).BaseDB, clock) } // NewTestDBFromPath is a helper function that creates a new SQLStore with a // connection to an existing sqlite database for testing. func NewTestDBFromPath(t *testing.T, dbPath string, - clock clock.Clock) *SQLStore { + clock clock.Clock) Store { return NewSQLStore( db.NewTestSqliteDbHandleFromPath(t, dbPath).BaseDB, clock, From 6c4c6a4319cbb854f6564feaf6487fa1d0e34ca1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Sun, 8 Jun 2025 13:31:03 +0200 Subject: [PATCH 2/4] session: ensure that test SQL store is closed We add a helper function to the functions that creates the test SQL stores, in order to ensure that the store is properly closed when the test is cleaned up. --- session/test_postgres.go | 4 ++-- session/test_sql.go | 14 +++++++++++++- session/test_sqlite.go | 6 +++--- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/session/test_postgres.go b/session/test_postgres.go index decf3bcc2..cb5aa061d 100644 --- a/session/test_postgres.go +++ b/session/test_postgres.go @@ -16,7 +16,7 @@ var ErrDBClosed = errors.New("database is closed") // NewTestDB is a helper function that creates an SQLStore database for testing. func NewTestDB(t *testing.T, clock clock.Clock) Store { - return NewSQLStore(db.NewTestPostgresDB(t).BaseDB, clock) + return createStore(t, db.NewTestPostgresDB(t).BaseDB, clock) } // NewTestDBFromPath is a helper function that creates a new SQLStore with a @@ -24,5 +24,5 @@ func NewTestDB(t *testing.T, clock clock.Clock) Store { func NewTestDBFromPath(t *testing.T, dbPath string, clock clock.Clock) Store { - return NewSQLStore(db.NewTestPostgresDB(t).BaseDB, clock) + return createStore(t, db.NewTestPostgresDB(t).BaseDB, clock) } diff --git a/session/test_sql.go b/session/test_sql.go index ceb02194c..a83186069 100644 --- a/session/test_sql.go +++ b/session/test_sql.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/lightninglabs/lightning-terminal/accounts" + "github.com/lightninglabs/lightning-terminal/db" "github.com/lightningnetwork/lnd/clock" "github.com/stretchr/testify/require" ) @@ -16,5 +17,16 @@ func NewTestDBWithAccounts(t *testing.T, clock clock.Clock, accounts, ok := acctStore.(*accounts.SQLStore) require.True(t, ok) - return NewSQLStore(accounts.BaseDB, clock) + return createStore(t, accounts.BaseDB, clock) +} + +// createStore is a helper function that creates a new SQLStore and ensure that +// it is closed when during the test cleanup. +func createStore(t *testing.T, sqlDB *db.BaseDB, clock clock.Clock) *SQLStore { + store := NewSQLStore(sqlDB, clock) + t.Cleanup(func() { + require.NoError(t, store.Close()) + }) + + return store } diff --git a/session/test_sqlite.go b/session/test_sqlite.go index dccbefe85..0ceb0e046 100644 --- a/session/test_sqlite.go +++ b/session/test_sqlite.go @@ -16,7 +16,7 @@ var ErrDBClosed = errors.New("database is closed") // NewTestDB is a helper function that creates an SQLStore database for testing. func NewTestDB(t *testing.T, clock clock.Clock) Store { - return NewSQLStore(db.NewTestSqliteDB(t).BaseDB, clock) + return createStore(t, db.NewTestSqliteDB(t).BaseDB, clock) } // NewTestDBFromPath is a helper function that creates a new SQLStore with a @@ -24,7 +24,7 @@ func NewTestDB(t *testing.T, clock clock.Clock) Store { func NewTestDBFromPath(t *testing.T, dbPath string, clock clock.Clock) Store { - return NewSQLStore( - db.NewTestSqliteDbHandleFromPath(t, dbPath).BaseDB, clock, + return createStore( + t, db.NewTestSqliteDbHandleFromPath(t, dbPath).BaseDB, clock, ) } From 8f1aa99ee56260358bc33d818bad62459e8ff52a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Wed, 30 Apr 2025 00:42:35 +0700 Subject: [PATCH 3/4] session: add migration code from kvdb to SQL This commit introduces the migration logic for transitioning the sessions store from kvdb to SQL. Note that as of this commit, the migration is not yet triggered by any production code, i.e. only tests execute the migration logic. --- session/sql_migration.go | 396 ++++++++++++++++++++++++++++++++++ session/sql_migration_test.go | 381 ++++++++++++++++++++++++++++++++ 2 files changed, 777 insertions(+) create mode 100644 session/sql_migration.go create mode 100644 session/sql_migration_test.go diff --git a/session/sql_migration.go b/session/sql_migration.go new file mode 100644 index 000000000..428cc0fce --- /dev/null +++ b/session/sql_migration.go @@ -0,0 +1,396 @@ +package session + +import ( + "bytes" + "context" + "database/sql" + "errors" + "fmt" + "reflect" + "time" + + "github.com/davecgh/go-spew/spew" + "github.com/lightninglabs/lightning-terminal/accounts" + "github.com/lightninglabs/lightning-terminal/db/sqlc" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/pmezard/go-difflib/difflib" + "go.etcd.io/bbolt" +) + +var ( + // ErrMigrationMismatch is returned when the migrated session does not + // match the original session. + ErrMigrationMismatch = fmt.Errorf("migrated session does not match " + + "original session") +) + +// MigrateSessionStoreToSQL runs the migration of all sessions from the KV +// database to the SQL database. The migration is done in a single transaction +// to ensure that all sessions are migrated or none at all. +// +// NOTE: As sessions may contain linked accounts, the accounts sql migration +// MUST be run prior to this migration. +func MigrateSessionStoreToSQL(ctx context.Context, kvStore *bbolt.DB, + tx SQLQueries) error { + + log.Infof("Starting migration of the KV sessions store to SQL") + + kvSessions, err := getBBoltSessions(kvStore) + if err != nil { + return err + } + + // If sessions are linked to a group, we must insert the initial session + // of each group before the other sessions in that group. This ensures + // we can retrieve the SQL group ID when inserting the remaining + // sessions. Therefore, we first insert all initial group sessions, + // allowing us to fetch the group IDs and insert the rest of the + // sessions afterward. + // We therefore filter out the initial sessions first, and then migrate + // them prior to the rest of the sessions. + var ( + initialGroupSessions []*Session + linkedSessions []*Session + ) + + for _, kvSession := range kvSessions { + if kvSession.GroupID == kvSession.ID { + initialGroupSessions = append( + initialGroupSessions, kvSession, + ) + } else { + linkedSessions = append(linkedSessions, kvSession) + } + } + + err = migrateSessionsToSQLAndValidate(ctx, tx, initialGroupSessions) + if err != nil { + return fmt.Errorf("migration of non-linked session failed: %w", + err) + } + + err = migrateSessionsToSQLAndValidate(ctx, tx, linkedSessions) + if err != nil { + return fmt.Errorf("migration of linked session failed: %w", err) + } + + total := len(initialGroupSessions) + len(linkedSessions) + log.Infof("All sessions migrated from KV to SQL. Total number of "+ + "sessions migrated: %d", total) + + return nil +} + +// getBBoltSessions is a helper function that fetches all sessions from the +// Bbolt store, by iterating directly over the buckets, without needing to +// use any public functions of the BoltStore struct. +func getBBoltSessions(db *bbolt.DB) ([]*Session, error) { + var sessions []*Session + + err := db.View(func(tx *bbolt.Tx) error { + sessionBucket, err := getBucket(tx, sessionBucketKey) + if err != nil { + return err + } + + return sessionBucket.ForEach(func(k, v []byte) error { + // We'll also get buckets here, skip those (identified + // by nil value). + if v == nil { + return nil + } + + session, err := DeserializeSession(bytes.NewReader(v)) + if err != nil { + return err + } + + sessions = append(sessions, session) + + return nil + }) + }) + + return sessions, err +} + +// migrateSessionsToSQLAndValidate runs the migration for the passed sessions +// from the KV database to the SQL database, and validates that the migrated +// sessions match the original sessions. +func migrateSessionsToSQLAndValidate(ctx context.Context, + tx SQLQueries, kvSessions []*Session) error { + + for _, kvSession := range kvSessions { + err := migrateSingleSessionToSQL(ctx, tx, kvSession) + if err != nil { + return fmt.Errorf("unable to migrate session(%v): %w", + kvSession.ID, err) + } + + // Validate that the session was correctly migrated and matches + // the original session in the kv store. + sqlSess, err := tx.GetSessionByAlias(ctx, kvSession.ID[:]) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + err = ErrSessionNotFound + } + return fmt.Errorf("unable to get migrated session "+ + "from sql store: %w", err) + } + + migratedSession, err := unmarshalSession(ctx, tx, sqlSess) + if err != nil { + return fmt.Errorf("unable to unmarshal migrated "+ + "session: %w", err) + } + + overrideSessionTimeZone(kvSession) + overrideSessionTimeZone(migratedSession) + overrideMacaroonRecipe(kvSession, migratedSession) + + if !reflect.DeepEqual(kvSession, migratedSession) { + diff := difflib.UnifiedDiff{ + A: difflib.SplitLines( + spew.Sdump(kvSession), + ), + B: difflib.SplitLines( + spew.Sdump(migratedSession), + ), + FromFile: "Expected", + FromDate: "", + ToFile: "Actual", + ToDate: "", + Context: 3, + } + diffText, _ := difflib.GetUnifiedDiffString(diff) + + return fmt.Errorf("%w: %v.\n%v", ErrMigrationMismatch, + kvSession.ID, diffText) + } + } + + return nil +} + +// migrateSingleSessionToSQL runs the migration for a single session from the +// KV database to the SQL database. Note that if the session links to an +// account, the linked accounts store MUST have been migrated before that +// session is migrated. +func migrateSingleSessionToSQL(ctx context.Context, + tx SQLQueries, session *Session) error { + + var ( + acctID sql.NullInt64 + err error + remotePubKey []byte + ) + + session.AccountID.WhenSome(func(alias accounts.AccountID) { + // Fetch the SQL ID for the account from the SQL store. + var acctAlias int64 + acctAlias, err = alias.ToInt64() + if err != nil { + return + } + + var acctDBID int64 + acctDBID, err = tx.GetAccountIDByAlias(ctx, acctAlias) + if errors.Is(err, sql.ErrNoRows) { + err = accounts.ErrAccNotFound + return + } else if err != nil { + return + } + + acctID = sqldb.SQLInt64(acctDBID) + }) + if err != nil { + return err + } + + if session.RemotePublicKey != nil { + remotePubKey = session.RemotePublicKey.SerializeCompressed() + } + + // Proceed to insert the session into the sql db. + sqlId, err := tx.InsertSession(ctx, sqlc.InsertSessionParams{ + Alias: session.ID[:], + Label: session.Label, + State: int16(session.State), + Type: int16(session.Type), + Expiry: session.Expiry.UTC(), + CreatedAt: session.CreatedAt.UTC(), + ServerAddress: session.ServerAddr, + DevServer: session.DevServer, + MacaroonRootKey: int64(session.MacaroonRootKey), + PairingSecret: session.PairingSecret[:], + LocalPrivateKey: session.LocalPrivateKey.Serialize(), + LocalPublicKey: session.LocalPublicKey.SerializeCompressed(), + RemotePublicKey: remotePubKey, + Privacy: session.WithPrivacyMapper, + AccountID: acctID, + }) + if err != nil { + return err + } + + // Since the InsertSession query doesn't support that we set the revoked + // field during the insert, we need to set the field after the session + // has been created. + if !session.RevokedAt.IsZero() { + err = tx.SetSessionRevokedAt( + ctx, sqlc.SetSessionRevokedAtParams{ + ID: sqlId, + RevokedAt: sqldb.SQLTime( + session.RevokedAt.UTC(), + ), + }, + ) + if err != nil { + return err + } + } + + // After the session has been inserted, we need to update the session + // with the group ID if it is linked to a group. We need to do this + // after the session has been inserted, because the group ID can be the + // session itself, and therefore the SQL id for the session won't exist + // prior to inserting the session. + groupID, err := tx.GetSessionIDByAlias(ctx, session.GroupID[:]) + if errors.Is(err, sql.ErrNoRows) { + return ErrUnknownGroup + } else if err != nil { + return fmt.Errorf("unable to fetch group(%x): %w", + session.GroupID[:], err) + } + + // Now lets set the group ID for the session. + err = tx.SetSessionGroupID(ctx, sqlc.SetSessionGroupIDParams{ + ID: sqlId, + GroupID: sqldb.SQLInt64(groupID), + }) + if err != nil { + return fmt.Errorf("unable to set group Alias: %w", err) + } + + // Once we have the sqlID for the session, we can proceed to insert rows + // into the linked child tables. + if session.MacaroonRecipe != nil { + // We start by inserting the macaroon permissions. + for _, sessionPerm := range session.MacaroonRecipe.Permissions { + err = tx.InsertSessionMacaroonPermission( + ctx, sqlc.InsertSessionMacaroonPermissionParams{ + SessionID: sqlId, + Entity: sessionPerm.Entity, + Action: sessionPerm.Action, + }, + ) + if err != nil { + return err + } + } + + // Next we insert the macaroon caveats. + for _, caveat := range session.MacaroonRecipe.Caveats { + err = tx.InsertSessionMacaroonCaveat( + ctx, sqlc.InsertSessionMacaroonCaveatParams{ + SessionID: sqlId, + CaveatID: caveat.Id, + VerificationID: caveat.VerificationId, + Location: sqldb.SQLStr( + caveat.Location, + ), + }, + ) + if err != nil { + return err + } + } + } + + // That's followed by the feature config. + if session.FeatureConfig != nil { + for featureName, config := range *session.FeatureConfig { + err = tx.InsertSessionFeatureConfig( + ctx, sqlc.InsertSessionFeatureConfigParams{ + SessionID: sqlId, + FeatureName: featureName, + Config: config, + }, + ) + if err != nil { + return err + } + } + } + + // Finally we insert the privacy flags. + for _, privacyFlag := range session.PrivacyFlags { + err = tx.InsertSessionPrivacyFlag( + ctx, sqlc.InsertSessionPrivacyFlagParams{ + SessionID: sqlId, + Flag: int32(privacyFlag), + }, + ) + if err != nil { + return err + } + } + + return nil +} + +// overrideSessionTimeZone overrides the time zone of the session to the local +// time zone and chops off the nanosecond part for comparison. This is needed +// because KV database stores times as-is which as an unwanted side effect would +// fail migration due to time comparison expecting both the original and +// migrated sessions to be in the same local time zone and in microsecond +// precision. Note that PostgresSQL stores times in microsecond precision while +// SQLite can store times in nanosecond precision if using TEXT storage class. +func overrideSessionTimeZone(session *Session) { + fixTime := func(t time.Time) time.Time { + return t.In(time.Local).Truncate(time.Microsecond) + } + + if !session.Expiry.IsZero() { + session.Expiry = fixTime(session.Expiry) + } + + if !session.CreatedAt.IsZero() { + session.CreatedAt = fixTime(session.CreatedAt) + } + + if !session.RevokedAt.IsZero() { + session.RevokedAt = fixTime(session.RevokedAt) + } +} + +// overrideMacaroonRecipe overrides the MacaroonRecipe for the SQL session in a +// certain scenario: +// In the bbolt store, a session can have a non-nil macaroon struct, despite +// both the permissions and caveats being nil. There is no way to represent this +// in the SQL store, as the macaroon permissions and caveats are separate +// tables. Therefore, in the scenario where a MacaroonRecipe exists for the +// bbolt version, but both the permissions and caveats are nil, we override the +// MacaroonRecipe for the SQL version and set it to a MacaroonRecipe with +// nil permissions and caveats. This is needed to ensure that the deep equals +// check in the migration validation does not fail in this scenario. +// Additionally, if either the permissions or caveats aren't set, for the +// MacaroonRecipe, that is represented as empty array in the SQL store, but +// as nil in the bbolt store. Therefore, we also override the permissions +// or caveats to nil for the migrated session in that scenario, so that the +// deep equals check does not fail in this scenario either. +func overrideMacaroonRecipe(kvSession *Session, migratedSession *Session) { + if kvSession.MacaroonRecipe != nil { + kvPerms := kvSession.MacaroonRecipe.Permissions + kvCaveats := kvSession.MacaroonRecipe.Caveats + + if kvPerms == nil && kvCaveats == nil { + migratedSession.MacaroonRecipe = &MacaroonRecipe{} + } else if kvPerms == nil { + migratedSession.MacaroonRecipe.Permissions = nil + } else if kvCaveats == nil { + migratedSession.MacaroonRecipe.Caveats = nil + } + } +} diff --git a/session/sql_migration_test.go b/session/sql_migration_test.go new file mode 100644 index 000000000..93097ccb9 --- /dev/null +++ b/session/sql_migration_test.go @@ -0,0 +1,381 @@ +package session + +import ( + "context" + "database/sql" + "fmt" + "testing" + "time" + + "github.com/lightninglabs/lightning-terminal/accounts" + "github.com/lightninglabs/lightning-terminal/db" + "github.com/lightningnetwork/lnd/clock" + "github.com/lightningnetwork/lnd/macaroons" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/stretchr/testify/require" + "gopkg.in/macaroon-bakery.v2/bakery/checkers" + "gopkg.in/macaroon.v2" +) + +// TestSessionsStoreMigration tests the migration of session store from a bolt +// backend to a SQL database. Note that this test does not attempt to be a +// complete migration test. +func TestSessionsStoreMigration(t *testing.T) { + t.Parallel() + + ctx := context.Background() + clock := clock.NewTestClock(time.Now()) + + // When using build tags that creates a kvdb store for NewTestDB, we + // skip this test as it is only applicable for postgres and sqlite tags. + store := NewTestDB(t, clock) + if _, ok := store.(*BoltStore); ok { + t.Skipf("Skipping session store migration test for kvdb build") + } + + makeSQLDB := func(t *testing.T, acctStore accounts.Store) (*SQLStore, + *db.TransactionExecutor[SQLQueries]) { + + // Create a sql store with a linked account store. + testDBStore := NewTestDBWithAccounts(t, clock, acctStore) + + store, ok := testDBStore.(*SQLStore) + require.True(t, ok) + + baseDB := store.BaseDB + + genericExecutor := db.NewTransactionExecutor( + baseDB, func(tx *sql.Tx) SQLQueries { + return baseDB.WithTx(tx) + }, + ) + + return store, genericExecutor + } + + // assertMigrationResults asserts that the sql store contains the + // same sessions as the passed kv store sessions. This is intended to be + // run after the migration. + assertMigrationResults := func(t *testing.T, sqlStore *SQLStore, + kvSessions []*Session) { + + for _, kvSession := range kvSessions { + // Fetch the migrated session from the sql store. + sqlSession, err := sqlStore.GetSession( + ctx, kvSession.ID, + ) + require.NoError(t, err) + + // Since the SQL store can't represent a session with + // a non-nil MacaroonRecipe, but with nil caveats and + // perms, we need to override the macaroon recipe if the + // kvSession has such a recipe stored. + overrideMacaroonRecipe(kvSession, sqlSession) + + assertEqualSessions(t, kvSession, sqlSession) + } + + // Finally we ensure that the sql store doesn't contain more + // sessions than the kv store. + sqlSessions, err := sqlStore.ListAllSessions(ctx) + require.NoError(t, err) + require.Equal(t, len(kvSessions), len(sqlSessions)) + } + + tests := []struct { + name string + populateDB func( + t *testing.T, kvStore *BoltStore, + accountStore accounts.Store, + ) + }{ + { + name: "empty", + populateDB: func(t *testing.T, store *BoltStore, + _ accounts.Store) { + + // Don't populate the DB. + }, + }, + { + name: "one session no options", + populateDB: func(t *testing.T, store *BoltStore, + _ accounts.Store) { + + _, err := store.NewSession( + ctx, "test", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + ) + require.NoError(t, err) + }, + }, + { + name: "multiple sessions no options", + populateDB: func(t *testing.T, store *BoltStore, + _ accounts.Store) { + + _, err := store.NewSession( + ctx, "session1", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + ) + require.NoError(t, err) + + _, err = store.NewSession( + ctx, "session2", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + ) + require.NoError(t, err) + + _, err = store.NewSession( + ctx, "session3", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + ) + require.NoError(t, err) + }, + }, + { + name: "one session with one privacy flag", + populateDB: func(t *testing.T, store *BoltStore, + _ accounts.Store) { + + _, err := store.NewSession( + ctx, "test", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + WithPrivacy(PrivacyFlags{ClearPubkeys}), + ) + require.NoError(t, err) + }, + }, + { + name: "one session with multiple privacy flags", + populateDB: func(t *testing.T, store *BoltStore, + _ accounts.Store) { + + _, err := store.NewSession( + ctx, "test", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + WithPrivacy(PrivacyFlags{ + ClearChanInitiator, ClearHTLCs, + ClearClosingTxIds, + }), + ) + require.NoError(t, err) + }, + }, + { + name: "one session with a feature config", + populateDB: func(t *testing.T, store *BoltStore, + _ accounts.Store) { + + featureConfig := map[string][]byte{ + "AutoFees": {1, 2, 3, 4}, + "AutoSomething": {4, 3, 4, 5, 6, 6}, + } + + _, err := store.NewSession( + ctx, "test", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + WithFeatureConfig(featureConfig), + ) + require.NoError(t, err) + }, + }, + { + name: "one session with dev server", + populateDB: func(t *testing.T, store *BoltStore, + _ accounts.Store) { + + _, err := store.NewSession( + ctx, "test", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + WithDevServer(), + ) + require.NoError(t, err) + }, + }, + { + name: "one session with macaroon recipe", + populateDB: func(t *testing.T, store *BoltStore, + _ accounts.Store) { + + // this test uses caveats & perms from the + // tlv_test.go + _, err := store.NewSession( + ctx, "test", TypeMacaroonAdmin, + time.Unix(1000, 0), "foo.bar.baz:1234", + WithMacaroonRecipe(caveats, perms), + ) + require.NoError(t, err) + }, + }, + { + name: "one session with macaroon recipe nil caveats", + populateDB: func(t *testing.T, store *BoltStore, + _ accounts.Store) { + + // this test uses perms from the tlv_test.go + _, err := store.NewSession( + ctx, "test", TypeMacaroonAdmin, + time.Unix(1000, 0), "foo.bar.baz:1234", + WithMacaroonRecipe(nil, perms), + ) + require.NoError(t, err) + }, + }, + { + name: "one session with macaroon recipe nil perms", + populateDB: func(t *testing.T, store *BoltStore, + _ accounts.Store) { + + // this test uses caveats from the tlv_test.go + _, err := store.NewSession( + ctx, "test", TypeMacaroonAdmin, + time.Unix(1000, 0), "foo.bar.baz:1234", + WithMacaroonRecipe(caveats, nil), + ) + require.NoError(t, err) + }, + }, + { + name: "macaroon recipe with nil perms and caveats", + populateDB: func(t *testing.T, store *BoltStore, + _ accounts.Store) { + + _, err := store.NewSession( + ctx, "test", TypeMacaroonAdmin, + time.Unix(1000, 0), "foo.bar.baz:1234", + WithMacaroonRecipe(nil, nil), + ) + require.NoError(t, err) + }, + }, + { + name: "one session with a linked account", + populateDB: func(t *testing.T, store *BoltStore, + acctStore accounts.Store) { + + // Create an account with balance + acct, err := acctStore.NewAccount( + ctx, 1234, time.Now().Add(time.Hour), + "", + ) + require.NoError(t, err) + require.False(t, acct.HasExpired()) + + // For now, we manually add the account caveat + // for bbolt compatibility. + accountCaveat := checkers.Condition( + macaroons.CondLndCustom, + fmt.Sprintf("%s %x", + accounts.CondAccount, + acct.ID[:], + ), + ) + + sessCaveats := []macaroon.Caveat{ + { + Id: []byte(accountCaveat), + }, + } + + _, err = store.NewSession( + ctx, "test", TypeMacaroonAccount, + time.Unix(1000, 0), "", + WithAccount(acct.ID), + WithMacaroonRecipe(sessCaveats, nil), + ) + require.NoError(t, err) + }, + }, + { + name: "linked session", + populateDB: func(t *testing.T, store *BoltStore, + _ accounts.Store) { + + // First create the initial session for the + // group. + sess1, err := store.NewSession( + ctx, "initSession", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + ) + require.NoError(t, err) + + // As the store won't allow us to link a + // session before all sessions in the group have + // been revoked, we revoke the session before + // creating a new session that links to the + // initial session. + err = store.ShiftState( + ctx, sess1.ID, StateCreated, + ) + require.NoError(t, err) + + err = store.ShiftState( + ctx, sess1.ID, StateRevoked, + ) + require.NoError(t, err) + + _, err = store.NewSession( + ctx, "linkedSession", TypeMacaroonAdmin, + time.Unix(1000, 0), "", + WithLinkedGroupID(&sess1.ID), + ) + require.NoError(t, err) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + // First let's create an account store to link to in + // the sessions store. Note that this is will be a sql + // store due to the build tags enabled when running this + // test, which means that we won't need to migrate the + // account store in this test. + accountStore := accounts.NewTestDB(t, clock) + + kvStore, err := NewDB( + t.TempDir(), DBFilename, clock, accountStore, + ) + require.NoError(t, err) + + t.Cleanup(func() { + require.NoError(t, kvStore.Close()) + }) + + // populate the kvStore with the test data, in + // preparation for the test. + test.populateDB(t, kvStore, accountStore) + + // Before we migrate the sessions, we fetch all sessions + // from the kv store, to ensure that the migration + // function doesn't mutate the bbolt store sessions. + // We can then compare them to the sql sessions after + // the migration has been executed. + kvSessions, err := kvStore.ListAllSessions(ctx) + require.NoError(t, err) + + // Proceed to create the sql store and execute the + // migration. + sqlStore, txEx := makeSQLDB(t, accountStore) + + var opts sqldb.MigrationTxOptions + err = txEx.ExecTx( + ctx, &opts, func(tx SQLQueries) error { + return MigrateSessionStoreToSQL( + ctx, kvStore.DB, tx, + ) + }, + ) + require.NoError(t, err) + + // The migration function will check if the inserted + // sessions equals the migrated ones, but as a sanity + // check we'll also fetch migrated sessions from the sql + // store and compare them to the original. + assertMigrationResults(t, sqlStore, kvSessions) + }) + } +} From 2007d538ec679634ddc57044fb7aba7d9013d51e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tigerstr=C3=B6m?= Date: Wed, 30 Apr 2025 00:43:26 +0700 Subject: [PATCH 4/4] session: add randomized session migration test --- session/sql_migration_test.go | 395 ++++++++++++++++++++++++++++++++++ 1 file changed, 395 insertions(+) diff --git a/session/sql_migration_test.go b/session/sql_migration_test.go index 93097ccb9..dfe495628 100644 --- a/session/sql_migration_test.go +++ b/session/sql_migration_test.go @@ -10,9 +10,13 @@ import ( "github.com/lightninglabs/lightning-terminal/accounts" "github.com/lightninglabs/lightning-terminal/db" "github.com/lightningnetwork/lnd/clock" + "github.com/lightningnetwork/lnd/lnwire" "github.com/lightningnetwork/lnd/macaroons" "github.com/lightningnetwork/lnd/sqldb" "github.com/stretchr/testify/require" + "go.etcd.io/bbolt" + "golang.org/x/exp/rand" + "gopkg.in/macaroon-bakery.v2/bakery" "gopkg.in/macaroon-bakery.v2/bakery/checkers" "gopkg.in/macaroon.v2" ) @@ -323,6 +327,10 @@ func TestSessionsStoreMigration(t *testing.T) { require.NoError(t, err) }, }, + { + name: "randomized sessions", + populateDB: randomizedSessions, + }, } for _, test := range tests { @@ -379,3 +387,390 @@ func TestSessionsStoreMigration(t *testing.T) { }) } } + +// randomizedSessions adds 100 randomized sessions to the kvStore, where 25% of +// them will contain up to 10 linked sessions. The rest of the session will have +// the rest of the session options randomized. +func randomizedSessions(t *testing.T, kvStore *BoltStore, + accountsStore accounts.Store) { + + ctx := context.Background() + + var ( + // numberOfSessions is set to 100 to add enough sessions to get + // enough variation between randomized sessions, but kept low + // enough for the test not take too long to run, as the test + // time increases drastically by the number of sessions we + // migrate. + numberOfSessions = 100 + ) + + for i := range numberOfSessions { + var ( + opts []Option + serverAddr string + ) + macType := macaroonType(i) + expiry := time.Unix(rand.Int63n(10000), rand.Int63n(10000)) + label := fmt.Sprintf("session%d", i+1) + + // Half of the sessions will get a set server address. + if rand.Intn(2) == 0 { + serverAddr = "foo.bar.baz:1234" + } + + // Every 10th session will get no added options. + if i%10 != 0 { + // Add random privacy flags to 50% of the sessions. + if rand.Intn(2) == 0 { + opts = append( + opts, WithPrivacy(randomPrivacyFlags()), + ) + } + + // Add random feature configs to 50% of the sessions. + if rand.Intn(2) == 0 { + opts = append( + opts, + WithFeatureConfig( + randomFeatureConfig(), + ), + ) + } + + // Set that the session uses a dev server for 50% of the + // sessions. + if rand.Intn(2) == 0 { + opts = append(opts, WithDevServer()) + } + + // Add a random macaroon recipe to 50% of the sessions. + if rand.Intn(2) == 0 { + // In 50% of those cases, we add a random + // macaroon recipe with caveats and perms, + // and for the other 50% we added a linked + // account with the correct macaroon recipe (to + // simulate realistic data). + if rand.Intn(2) == 0 { + opts = append( + opts, randomMacaroonRecipe(), + ) + } else { + acctOpts := randomAccountOptions( + ctx, t, accountsStore, + ) + + opts = append(opts, acctOpts...) + } + } + } + + // We insert the session with the randomized params and options. + activeSess, err := kvStore.NewSession( + ctx, label, macType, expiry, serverAddr, opts..., + ) + require.NoError(t, err) + + // For 25% of the sessions, we link a random number of sessions + // to the session. + if rand.Intn(4) == 0 { + // Link up to 10 sessions to the session, and set the + // same opts as the initial group session. + for j := range rand.Intn(10) { + // We first need to revoke the previous session + // before we can create a new session that links + // to the session. + err = kvStore.ShiftState( + ctx, activeSess.ID, StateCreated, + ) + require.NoError(t, err) + + err = kvStore.ShiftState( + ctx, activeSess.ID, StateRevoked, + ) + require.NoError(t, err) + + opts = []Option{ + WithLinkedGroupID(&activeSess.GroupID), + } + + if activeSess.DevServer { + opts = append(opts, WithDevServer()) + } + + if activeSess.FeatureConfig != nil { + opts = append(opts, WithFeatureConfig( + *activeSess.FeatureConfig, + )) + } + + if activeSess.PrivacyFlags != nil { + opts = append(opts, WithPrivacy( + activeSess.PrivacyFlags, + )) + } + + if activeSess.MacaroonRecipe != nil { + macRec := activeSess.MacaroonRecipe + opts = append(opts, WithMacaroonRecipe( + macRec.Caveats, + macRec.Permissions, + )) + } + + activeSess.AccountID.WhenSome( + func(alias accounts.AccountID) { + opts = append( + opts, + WithAccount(alias), + ) + }, + ) + + label = fmt.Sprintf("linkedSession%d", j+1) + + activeSess, err = kvStore.NewSession( + ctx, label, activeSess.Type, + time.Unix(1000, 0), + activeSess.ServerAddr, opts..., + ) + require.NoError(t, err) + } + } + + // Finally, we shift the active session to a random state. + // As the state we set may be a state that's no longer set + // through the current code base, or be an illegal state + // transition, we use an alternative test state shifting method + // that doesn't check that we transition the state in the legal + // order. + err = shiftStateUnsafe(kvStore, activeSess.ID, lastState(i)) + require.NoError(t, err) + } +} + +// macaroonType returns a macaroon type based on the given index by taking the +// index modulo 6. This ensures an approximately equal distribution of macaroon +// types. +func macaroonType(i int) Type { + switch i % 6 { + case 0: + return TypeMacaroonReadonly + case 1: + return TypeMacaroonAdmin + case 2: + return TypeMacaroonCustom + case 3: + return TypeUIPassword + case 4: + return TypeAutopilot + default: + return TypeMacaroonAccount + } +} + +// lastState returns a state based on the given index by taking the index modulo +// 5. This ensures an approximately equal distribution of states. +func lastState(i int) State { + switch i % 5 { + case 0: + return StateCreated + case 1: + return StateInUse + case 2: + return StateRevoked + case 3: + return StateExpired + default: + return StateReserved + } +} + +// randomPrivacyFlags returns a random set of privacy flags. +func randomPrivacyFlags() PrivacyFlags { + allFlags := []PrivacyFlag{ + ClearPubkeys, + ClearChanIDs, + ClearTimeStamps, + ClearChanInitiator, + ClearHTLCs, + ClearClosingTxIds, + ClearNetworkAddresses, + } + + var privFlags []PrivacyFlag + for _, flag := range allFlags { + if rand.Intn(2) == 0 { + privFlags = append(privFlags, flag) + } + } + + return privFlags +} + +// randomFeatureConfig returns a random feature config with a random number of +// features. The feature names are generated as "feature0", "feature1", etc. +func randomFeatureConfig() FeaturesConfig { + featureConfig := make(FeaturesConfig) + for i := range rand.Intn(10) { + featureName := fmt.Sprintf("feature%d", i) + featureValue := []byte{byte(rand.Int31())} + featureConfig[featureName] = featureValue + } + + return featureConfig +} + +// randomMacaroonRecipe returns a random macaroon recipe with a random number of +// caveats and permissions. The returned macaroon recipe may have nil set for +// either the caveats or permissions, but not both. +func randomMacaroonRecipe() Option { + var ( + macCaveats []macaroon.Caveat + macPerms []bakery.Op + ) + + loopLen := rand.Intn(10) + 1 + + if rand.Intn(2) == 0 { + for range loopLen { + var macCaveat macaroon.Caveat + + // We always have a caveat.Id, but the rest are + // randomized if they exist or not. + macCaveat.Id = randomBytes(rand.Intn(10) + 1) + + if rand.Intn(2) == 0 { + macCaveat.VerificationId = + randomBytes(rand.Intn(32) + 1) + } + + if rand.Intn(2) == 0 { + macCaveat.Location = + randomString(rand.Intn(10) + 1) + } + + macCaveats = append(macCaveats, macCaveat) + } + } else { + macCaveats = nil + } + + // We can't do both nil caveats and nil perms, so if we have nil + // caveats, we set perms to a value. + if rand.Intn(2) == 0 || macCaveats == nil { + for range loopLen { + var macPerm bakery.Op + + macPerm.Action = randomString(rand.Intn(10) + 1) + macPerm.Entity = randomString(rand.Intn(10) + 1) + + macPerms = append(macPerms, macPerm) + } + } else { + macPerms = nil + } + + return WithMacaroonRecipe(macCaveats, macPerms) +} + +// randomAccountOptions creates a random account with a random balance and +// expiry time, that's linked in the returned options. The returned options also +// returns the macaroon recipe with the account caveat. +func randomAccountOptions(ctx context.Context, t *testing.T, + acctStore accounts.Store) []Option { + + balance := lnwire.MilliSatoshi(rand.Int63()) + + // randomize expiry from 10 to 10,000 minutes + expiry := time.Now().Add( + time.Minute * time.Duration(rand.Intn(10000-10)+10), + ) + + // As the store has a unique constraint for inserting labels, we suffix + // it with a sufficiently large random number avoid collisions. + label := fmt.Sprintf("account:%d", rand.Int63()) + + // Create an account with balance + acct, err := acctStore.NewAccount(ctx, balance, expiry, label) + require.NoError(t, err) + require.False(t, acct.HasExpired()) + + // For now, we manually add the account caveat + // for bbolt compatibility. + accountCaveat := checkers.Condition( + macaroons.CondLndCustom, + fmt.Sprintf("%s %x", + accounts.CondAccount, + acct.ID[:], + ), + ) + + sessCaveats := []macaroon.Caveat{} + sessCaveats = append( + sessCaveats, + macaroon.Caveat{ + Id: []byte(accountCaveat), + }, + ) + + opts := []Option{ + WithAccount(acct.ID), WithMacaroonRecipe(sessCaveats, nil), + } + + return opts +} + +// randomBytes generates a random byte array of the passed length n. +func randomBytes(n int) []byte { + b := make([]byte, n) + for i := range b { + b[i] = byte(rand.Intn(256)) // Random int between 0-255, then cast to byte + } + return b +} + +// randomString generates a random string of the passed length n. +func randomString(n int) string { + letterBytes := "abcdefghijklmnopqrstuvwxyz" + + b := make([]byte, n) + for i := range b { + b[i] = letterBytes[rand.Intn(len(letterBytes))] + } + return string(b) +} + +// shiftStateUnsafe updates the state of the session with the given ID to the +// "dest" state, without checking if the state transition is legal. +// +// NOTE: this function should only be used for testing purposes. +func shiftStateUnsafe(db *BoltStore, id ID, dest State) error { + return db.Update(func(tx *bbolt.Tx) error { + sessionBucket, err := getBucket(tx, sessionBucketKey) + if err != nil { + return err + } + + session, err := getSessionByID(sessionBucket, id) + if err != nil { + return err + } + + // If the session is already in the desired state, we return + // with no error to maintain idempotency. + if session.State == dest { + return nil + } + + session.State = dest + + // If the session is terminal, we set the revoked at time to the + // current time. + if dest.Terminal() { + session.RevokedAt = db.clock.Now().UTC() + } + + return putSession(sessionBucket, session) + }) +}