From 660997e4cc94c6ad63f2fb34fa05bf63bf67276a Mon Sep 17 00:00:00 2001 From: andy-shi88 Date: Mon, 20 Apr 2020 14:27:24 +0800 Subject: [PATCH 01/17] add spineblock height field --- common/model/spineBlockManifest.pb.go | 64 ++++++++++++++++----------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/common/model/spineBlockManifest.pb.go b/common/model/spineBlockManifest.pb.go index 5e50f7ce4..db240d062 100644 --- a/common/model/spineBlockManifest.pb.go +++ b/common/model/spineBlockManifest.pb.go @@ -51,15 +51,18 @@ type SpineBlockManifest struct { FullFileHash []byte `protobuf:"bytes,2,opt,name=FullFileHash,proto3" json:"FullFileHash,omitempty"` // FileChunkHashes sequence of hashes (sha256 = 32 byte) of file chunks (sorted) referenced by the spineBlockManifest FileChunkHashes []byte `protobuf:"bytes,3,opt,name=FileChunkHashes,proto3" json:"FileChunkHashes,omitempty"` - // SpineBlockManifestHeight height (on the mainchain) at which the (snapshot) file started been computed + // ManifestReferenceHeight height (on the mainchain) at which the (snapshot) file started been computed // Note: this is not the last mainchain height contained in the snapshot file (that one should be = SpineBlockManifestHeight - MinRollbackBlocks) - SpineBlockManifestHeight uint32 `protobuf:"varint,4,opt,name=SpineBlockManifestHeight,proto3" json:"SpineBlockManifestHeight,omitempty"` + ManifestReferenceHeight uint32 `protobuf:"varint,4,opt,name=ManifestReferenceHeight,proto3" json:"ManifestReferenceHeight,omitempty"` + // ManifestSpineBlockHeight (on spinechain) at which the (manifest) got included in the block, this data + // is tightly coupled to the spine block. + ManifestSpineBlockHeight uint32 `protobuf:"varint,5,opt,name=ManifestSpineBlockHeight,proto3" json:"ManifestSpineBlockHeight,omitempty"` // Number indicating chaintype (at the moment it can only be mainchain, but in future could be others) - ChainType int32 `protobuf:"varint,5,opt,name=ChainType,proto3" json:"ChainType,omitempty"` + ChainType int32 `protobuf:"varint,6,opt,name=ChainType,proto3" json:"ChainType,omitempty"` // SpineBlockManifestType type of spineBlockManifest - SpineBlockManifestType SpineBlockManifestType `protobuf:"varint,6,opt,name=SpineBlockManifestType,proto3,enum=model.SpineBlockManifestType" json:"SpineBlockManifestType,omitempty"` + SpineBlockManifestType SpineBlockManifestType `protobuf:"varint,7,opt,name=SpineBlockManifestType,proto3,enum=model.SpineBlockManifestType" json:"SpineBlockManifestType,omitempty"` // ExpirationTimestamp timestamp that marks the end of spineBlockManifest processing - ExpirationTimestamp int64 `protobuf:"varint,7,opt,name=ExpirationTimestamp,proto3" json:"ExpirationTimestamp,omitempty"` + ExpirationTimestamp int64 `protobuf:"varint,8,opt,name=ExpirationTimestamp,proto3" json:"ExpirationTimestamp,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -111,9 +114,16 @@ func (m *SpineBlockManifest) GetFileChunkHashes() []byte { return nil } -func (m *SpineBlockManifest) GetSpineBlockManifestHeight() uint32 { +func (m *SpineBlockManifest) GetManifestReferenceHeight() uint32 { if m != nil { - return m.SpineBlockManifestHeight + return m.ManifestReferenceHeight + } + return 0 +} + +func (m *SpineBlockManifest) GetManifestSpineBlockHeight() uint32 { + if m != nil { + return m.ManifestSpineBlockHeight } return 0 } @@ -147,23 +157,25 @@ func init() { func init() { proto.RegisterFile("model/spineBlockManifest.proto", fileDescriptor_28f5b9e6a17937ec) } var fileDescriptor_28f5b9e6a17937ec = []byte{ - // 285 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x91, 0x3f, 0x4f, 0x03, 0x21, - 0x18, 0xc6, 0xe5, 0x6a, 0xab, 0x92, 0xfa, 0x27, 0x98, 0x18, 0x06, 0x35, 0x97, 0x0e, 0x86, 0x34, - 0xf1, 0xae, 0xd1, 0xcd, 0xb1, 0xad, 0x4d, 0x3b, 0xb8, 0xd0, 0xba, 0xb8, 0xd1, 0x13, 0x0b, 0xe9, - 0xc1, 0x4b, 0x0a, 0x4d, 0xd4, 0x6f, 0xea, 0xb7, 0x31, 0x87, 0x83, 0xd1, 0xeb, 0x2d, 0x84, 0xfc, - 0x9e, 0xdf, 0x1b, 0x1e, 0x00, 0x5f, 0x1b, 0x78, 0x95, 0x65, 0xee, 0x9d, 0xb6, 0x72, 0x58, 0x42, - 0xb1, 0x7e, 0x12, 0x56, 0xbf, 0x49, 0x1f, 0x32, 0xb7, 0x81, 0x00, 0xa4, 0x1d, 0xf3, 0xde, 0x57, - 0x82, 0xc9, 0xbc, 0xe6, 0x10, 0x82, 0x93, 0xd9, 0x98, 0xa2, 0x14, 0xb1, 0xd6, 0x30, 0x19, 0x20, - 0x9e, 0xcc, 0xc6, 0xa4, 0x87, 0xbb, 0x93, 0x6d, 0x59, 0x4e, 0x74, 0x29, 0xa7, 0xc2, 0x2b, 0x9a, - 0xa4, 0x88, 0x75, 0xf9, 0x1f, 0x46, 0x18, 0x3e, 0xad, 0xf6, 0x23, 0xb5, 0xb5, 0xeb, 0x0a, 0x48, - 0x4f, 0x5b, 0x51, 0xfb, 0x8f, 0xc9, 0x03, 0xa6, 0xf5, 0x73, 0xa7, 0x52, 0xaf, 0x54, 0xa0, 0xfb, - 0x29, 0x62, 0xc7, 0xbc, 0x31, 0x27, 0x97, 0xf8, 0x68, 0xa4, 0x84, 0xb6, 0x8b, 0x0f, 0x27, 0x69, - 0x3b, 0x45, 0xac, 0xcd, 0x7f, 0x01, 0x79, 0xc6, 0x17, 0xf5, 0xc9, 0xa8, 0x76, 0x52, 0xc4, 0x4e, - 0xee, 0xae, 0xb2, 0x78, 0xf5, 0x6c, 0xb7, 0xc4, 0x1b, 0x86, 0xc9, 0x00, 0x9f, 0x3f, 0xbe, 0x3b, - 0xbd, 0x11, 0x41, 0x83, 0x5d, 0x68, 0x23, 0x7d, 0x10, 0xc6, 0xd1, 0x83, 0xea, 0x8d, 0xf8, 0xae, - 0xa8, 0x7f, 0xd3, 0x54, 0x84, 0x74, 0xf1, 0xe1, 0xdc, 0x0a, 0xe7, 0x15, 0x84, 0xb3, 0xbd, 0x61, - 0xff, 0x85, 0xad, 0x74, 0x50, 0xdb, 0x65, 0x56, 0x80, 0xc9, 0x3f, 0x01, 0x96, 0xc5, 0xcf, 0x7a, - 0x5b, 0xc0, 0x46, 0xe6, 0x05, 0x18, 0x03, 0x36, 0x8f, 0xa5, 0x97, 0x9d, 0xf8, 0x7b, 0xf7, 0xdf, - 0x01, 0x00, 0x00, 0xff, 0xff, 0x46, 0x52, 0x0d, 0xe2, 0xdf, 0x01, 0x00, 0x00, + // 306 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x91, 0xcf, 0x4f, 0xc2, 0x30, + 0x1c, 0xc5, 0xed, 0x10, 0xc4, 0x06, 0x7f, 0xa4, 0x26, 0xda, 0x83, 0x9a, 0x85, 0x83, 0x69, 0x48, + 0xdc, 0x88, 0x5e, 0x8c, 0x47, 0x40, 0x02, 0x07, 0x2f, 0x05, 0x2f, 0xde, 0xc6, 0xfc, 0xc2, 0x1a, + 0xb6, 0x7e, 0x97, 0xb5, 0x24, 0xea, 0xbf, 0xe1, 0x3f, 0x6c, 0x56, 0x25, 0x44, 0x61, 0x97, 0xa6, + 0x79, 0x9f, 0xf7, 0xd2, 0x97, 0x57, 0x7a, 0x9d, 0xe1, 0x1b, 0xa4, 0xa1, 0xc9, 0x95, 0x86, 0x5e, + 0x8a, 0xf1, 0xf2, 0x39, 0xd2, 0x6a, 0x0e, 0xc6, 0x06, 0x79, 0x81, 0x16, 0x59, 0xdd, 0xf1, 0xf6, + 0x57, 0x8d, 0xb2, 0xc9, 0x96, 0x87, 0x31, 0xea, 0x8d, 0x07, 0x9c, 0xf8, 0x44, 0xd4, 0x7a, 0x5e, + 0x97, 0x48, 0x6f, 0x3c, 0x60, 0x6d, 0xda, 0x1a, 0xae, 0xd2, 0x74, 0xa8, 0x52, 0x18, 0x45, 0x26, + 0xe1, 0x9e, 0x4f, 0x44, 0x4b, 0xfe, 0xd1, 0x98, 0xa0, 0x27, 0xe5, 0xbd, 0x9f, 0xac, 0xf4, 0xb2, + 0x14, 0xc0, 0xf0, 0x9a, 0xb3, 0xfd, 0x97, 0xd9, 0x03, 0xbd, 0x58, 0xbf, 0x26, 0x61, 0x0e, 0x05, + 0xe8, 0x18, 0x46, 0xa0, 0x16, 0x89, 0xe5, 0xfb, 0x3e, 0x11, 0x47, 0xb2, 0x0a, 0xb3, 0x47, 0xca, + 0xd7, 0x68, 0xd3, 0xfc, 0x37, 0x5a, 0x77, 0xd1, 0x4a, 0xce, 0x2e, 0xe9, 0x61, 0x3f, 0x89, 0x94, + 0x9e, 0x7e, 0xe4, 0xc0, 0x1b, 0x3e, 0x11, 0x75, 0xb9, 0x11, 0xd8, 0x0b, 0x3d, 0xdf, 0xde, 0xc2, + 0x59, 0x0f, 0x7c, 0x22, 0x8e, 0xef, 0xae, 0x02, 0x37, 0x5a, 0xb0, 0xdb, 0x24, 0x2b, 0xc2, 0xac, + 0x4b, 0xcf, 0x9e, 0xde, 0x73, 0x55, 0x44, 0x56, 0xa1, 0x9e, 0xaa, 0x0c, 0x8c, 0x8d, 0xb2, 0x9c, + 0x37, 0xcb, 0x75, 0xe5, 0x2e, 0xd4, 0xb9, 0xa9, 0x2a, 0xc2, 0x5a, 0xb4, 0x39, 0xd1, 0x51, 0x6e, + 0x12, 0xb4, 0xa7, 0x7b, 0xbd, 0xce, 0xab, 0x58, 0x28, 0x9b, 0xac, 0x66, 0x41, 0x8c, 0x59, 0xf8, + 0x89, 0x38, 0x8b, 0x7f, 0xce, 0xdb, 0x18, 0x0b, 0x08, 0x63, 0xcc, 0x32, 0xd4, 0xa1, 0x2b, 0x3d, + 0x6b, 0xb8, 0x7f, 0xbf, 0xff, 0x0e, 0x00, 0x00, 0xff, 0xff, 0xe9, 0x9f, 0x65, 0x53, 0x19, 0x02, + 0x00, 0x00, } From 4e4b6bfb52765519ba787a58e7ec0f493c117946 Mon Sep 17 00:00:00 2001 From: andy-shi88 Date: Tue, 21 Apr 2020 13:53:45 +0800 Subject: [PATCH 02/17] rename spine block manifest column and add indesx on block height column --- common/database/migration.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/common/database/migration.go b/common/database/migration.go index 634e1af96..01f5714f2 100644 --- a/common/database/migration.go +++ b/common/database/migration.go @@ -269,9 +269,10 @@ func (m *Migration) Init() error { "full_file_hash" BLOB, -- hash of the (snapshot) file content "file_chunk_hashes" BLOB, -- sorted sequence file chunks hashes referenced by the spine_block_manifest "manifest_reference_height" INTEGER NOT NULL, -- height at which the snapshot was taken on the (main)chain + "manifest_spine_block_height" INTEGER NOT NULL, -- height at which the snapshot was taken on the (main)chain "chain_type" INTEGER NOT NULL, -- chain type this spine_block_manifest reference to "manifest_type" INTEGER NOT NULL, -- type of spine_block_manifest (as of now only snapshot) - "manifest_timestamp" INTEGER NOT NULL, -- timestamp that marks the end of file chunks processing + "expiration_timestamp" INTEGER NOT NULL, -- timestamp that marks the end of file chunks processing PRIMARY KEY("id") UNIQUE("id") ) @@ -345,6 +346,12 @@ func (m *Migration) Init() error { ` CREATE INDEX "published_receipt_datum_hash_idx" ON "published_receipt" ("datum_hash") `, + ` + CREATE INDEX "spine_block_manifest_spine_block_height_idx" ON "spine_block_manifest" ("manifest_spine_block_height") + `, + ` + CREATE INDEX "spine_block_manifest_reference_height_idx" ON "spine_block_manifest" ("manifest_reference_height") + `, } return nil } From 880f220e56f0f57914eba73687d624cc8f2e578e Mon Sep 17 00:00:00 2001 From: andy-shi88 Date: Tue, 21 Apr 2020 13:55:06 +0800 Subject: [PATCH 03/17] rename spine_block_manifest column field --- common/query/spineBlockManifestQuery.go | 35 +++++++++++++++---- .../blockchainOrchestratorService.go | 4 +-- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/common/query/spineBlockManifestQuery.go b/common/query/spineBlockManifestQuery.go index 062202749..d25dc1325 100644 --- a/common/query/spineBlockManifestQuery.go +++ b/common/query/spineBlockManifestQuery.go @@ -3,9 +3,10 @@ package query import ( "database/sql" "fmt" - "github.com/zoobc/zoobc-core/common/constant" "strings" + "github.com/zoobc/zoobc-core/common/constant" + "github.com/zoobc/zoobc-core/common/chaintype" "github.com/zoobc/zoobc-core/common/model" @@ -15,6 +16,8 @@ type ( SpineBlockManifestQueryInterface interface { InsertSpineBlockManifest(spineBlockManifest *model.SpineBlockManifest) (str string, args []interface{}) GetSpineBlockManifestTimeInterval(fromTimestamp, toTimestamp int64) string + GetManifestBySpineBlockHeight(spineBlockHeight uint32) string + GetManifestsFromSpineBlockHeight(spineBlockHeight uint32) string GetLastSpineBlockManifest(ct chaintype.ChainType, mbType model.SpineBlockManifestType) string ExtractModel(mb *model.SpineBlockManifest) []interface{} BuildModel(spineBlockManifests []*model.SpineBlockManifest, rows *sql.Rows) ([]*model.SpineBlockManifest, error) @@ -34,9 +37,10 @@ func NewSpineBlockManifestQuery() *SpineBlockManifestQuery { "full_file_hash", "file_chunk_hashes", "manifest_reference_height", + "manifest_spine_block_height", "chain_type", "manifest_type", - "manifest_timestamp", + "expiration_timestamp", }, TableName: "spine_block_manifest", } @@ -72,19 +76,34 @@ func (mbl *SpineBlockManifestQuery) GetLastSpineBlockManifest(ct chaintype.Chain // GetSpineBlockManifestTimeInterval retrieve all spineBlockManifests within a time frame // Note: it is used to get all entities that have expired between spine blocks func (mbl *SpineBlockManifestQuery) GetSpineBlockManifestTimeInterval(fromTimestamp, toTimestamp int64) string { - query := fmt.Sprintf("SELECT %s FROM %s WHERE manifest_timestamp > %d AND manifest_timestamp <= %d "+ + query := fmt.Sprintf("SELECT %s FROM %s WHERE expiration_timestamp > %d AND expiration_timestamp <= %d "+ "ORDER BY manifest_type, chain_type, manifest_reference_height", strings.Join(mbl.Fields, ", "), mbl.getTableName(), fromTimestamp, toTimestamp) return query } +// GetManifestBySpineBlockHeight retrieve manifests of binded to a spineblock height +func (mbl *SpineBlockManifestQuery) GetManifestBySpineBlockHeight(spineBlockHeight uint32) string { + query := fmt.Sprintf("SELECT %s FROM %s WHERE manifest_spine_block_height = %d "+ + "ORDER BY manifest_type, chain_type, manifest_reference_height", + strings.Join(mbl.Fields, ", "), mbl.getTableName(), spineBlockHeight) + return query +} + +func (mbl *SpineBlockManifestQuery) GetManifestsFromSpineBlockHeight(spineBlockHeight uint32) string { + query := fmt.Sprintf("SELECT %s FROM %s WHERE manifest_spine_block_height > %d ", + strings.Join(mbl.Fields, ", "), mbl.getTableName(), spineBlockHeight) + return query +} + // ExtractModel extract the model struct fields to the order of SpineBlockManifestQuery.Fields func (mbl *SpineBlockManifestQuery) ExtractModel(mb *model.SpineBlockManifest) []interface{} { return []interface{}{ mb.ID, mb.FullFileHash, mb.FileChunkHashes, - mb.SpineBlockManifestHeight, + mb.ManifestReferenceHeight, + mb.ManifestSpineBlockHeight, mb.ChainType, mb.SpineBlockManifestType, mb.ExpirationTimestamp, @@ -106,7 +125,8 @@ func (mbl *SpineBlockManifestQuery) BuildModel( &mb.ID, &mb.FullFileHash, &mb.FileChunkHashes, - &mb.SpineBlockManifestHeight, + &mb.ManifestReferenceHeight, + &mb.ManifestSpineBlockHeight, &mb.ChainType, &mb.SpineBlockManifestType, &mb.ExpirationTimestamp, @@ -125,7 +145,8 @@ func (mbl *SpineBlockManifestQuery) Scan(mb *model.SpineBlockManifest, row *sql. &mb.ID, &mb.FullFileHash, &mb.FileChunkHashes, - &mb.SpineBlockManifestHeight, + &mb.ManifestReferenceHeight, + &mb.ManifestSpineBlockHeight, &mb.ChainType, &mb.SpineBlockManifestType, &mb.ExpirationTimestamp, @@ -142,7 +163,7 @@ func (mbl *SpineBlockManifestQuery) Scan(mb *model.SpineBlockManifest, row *sql. func (mbl *SpineBlockManifestQuery) Rollback(height uint32) (multiQueries [][]interface{}) { return [][]interface{}{ { - fmt.Sprintf("DELETE FROM %s WHERE manifest_reference_height > ?", mbl.getTableName()), + fmt.Sprintf("DELETE FROM %s WHERE manifest_spine_block_height > ?", mbl.getTableName()), height - constant.MinRollbackBlocks, }, } diff --git a/core/blockchainsync/blockchainOrchestratorService.go b/core/blockchainsync/blockchainOrchestratorService.go index 9c216fe5c..5c38c0fea 100644 --- a/core/blockchainsync/blockchainOrchestratorService.go +++ b/core/blockchainsync/blockchainOrchestratorService.go @@ -103,14 +103,14 @@ func (bos *BlockchainOrchestratorService) DownloadSnapshot(ct chaintype.ChainTyp } bos.Logger.Infof("found a Snapshot Spine Block Manifest for chaintype %s, "+ "at height is %d. Start downloading...\n", ct.GetName(), - lastSpineBlockManifest.SpineBlockManifestHeight) + lastSpineBlockManifest.ManifestReferenceHeight) snapshotFileInfo, err := bos.FileDownloader.DownloadSnapshot(ct, lastSpineBlockManifest) if err != nil { bos.Logger.Warning(err) return err } else if err := bos.MainchainSnapshotBlockServices.ImportSnapshotFile(snapshotFileInfo); err != nil { bos.Logger.Warningf("error importing snapshot file for chaintype %s at height %d: %s\n", ct.GetName(), - lastSpineBlockManifest.SpineBlockManifestHeight, err.Error()) + lastSpineBlockManifest.ManifestReferenceHeight, err.Error()) return err } From 9d01d6d897f27e1a05516c2be8f3d0635aa40af8 Mon Sep 17 00:00:00 2001 From: andy-shi88 Date: Tue, 21 Apr 2020 13:55:31 +0800 Subject: [PATCH 04/17] move spine_block_manifest to spinechain derived query category --- common/query/query.go | 2 +- common/query/query_test.go | 2 +- common/query/spineBlockManifestQuery_test.go | 20 ++++++++++---------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/common/query/query.go b/common/query/query.go index 155c3029a..f4006f944 100644 --- a/common/query/query.go +++ b/common/query/query.go @@ -35,12 +35,12 @@ func GetDerivedQuery(ct chaintype.ChainType) (derivedQuery []DerivedQuery) { NewPendingTransactionQuery(), NewPendingSignatureQuery(), NewMultisignatureInfoQuery(), - NewSpineBlockManifestQuery(), } derivedQuery = append(derivedQuery, mainchainDerivedQuery...) case *chaintype.SpineChain: spinechainDerivedQuery := []DerivedQuery{ NewSpinePublicKeyQuery(), + NewSpineBlockManifestQuery(), } derivedQuery = append(derivedQuery, spinechainDerivedQuery...) } diff --git a/common/query/query_test.go b/common/query/query_test.go index 779f0567c..a18855cc9 100644 --- a/common/query/query_test.go +++ b/common/query/query_test.go @@ -36,7 +36,6 @@ func TestGetDerivedQuery(t *testing.T) { NewPendingTransactionQuery(), NewPendingSignatureQuery(), NewMultisignatureInfoQuery(), - NewSpineBlockManifestQuery(), }, }, { @@ -45,6 +44,7 @@ func TestGetDerivedQuery(t *testing.T) { want: []DerivedQuery{ NewBlockQuery(spinechain), NewSpinePublicKeyQuery(), + NewSpineBlockManifestQuery(), }, }, } diff --git a/common/query/spineBlockManifestQuery_test.go b/common/query/spineBlockManifestQuery_test.go index 7173f2d9c..cfc98ef76 100644 --- a/common/query/spineBlockManifestQuery_test.go +++ b/common/query/spineBlockManifestQuery_test.go @@ -18,8 +18,8 @@ func TestSpineBlockManifestQuery_InsertSpineBlockManifest(t *testing.T) { } mb1 := &model.SpineBlockManifest{ - FullFileHash: make([]byte, 64), // sha3-512 - SpineBlockManifestHeight: 720, + FullFileHash: make([]byte, 64), // sha3-512 + ManifestReferenceHeight: 720, } tests := []struct { name string @@ -37,8 +37,8 @@ func TestSpineBlockManifestQuery_InsertSpineBlockManifest(t *testing.T) { spineBlockManifest: mb1, }, want: "INSERT OR REPLACE INTO spine_block_manifest (id,full_file_hash,file_chunk_hashes,manifest_reference_height," + - "chain_type,manifest_type," + - "manifest_timestamp) VALUES(? , ?, ?, ?, ?, ?, ?)", + "manifest_spine_block_height,chain_type,manifest_type," + + "expiration_timestamp) VALUES(? , ?, ?, ?, ?, ?, ?, ?)", }, } for _, tt := range tests { @@ -79,9 +79,9 @@ func TestSpineBlockManifestQuery_GetLastSpineBlockManifest(t *testing.T) { ct: &chaintype.MainChain{}, mbType: model.SpineBlockManifestType_Snapshot, }, - want: "SELECT id, full_file_hash, file_chunk_hashes, manifest_reference_height, chain_type, manifest_type, " + - "manifest_timestamp FROM spine_block_manifest WHERE chain_type = 0 AND manifest_type = 0 ORDER BY manifest_reference_height" + - " DESC LIMIT 1", + want: "SELECT id, full_file_hash, file_chunk_hashes, manifest_reference_height, manifest_spine_block_height, " + + "chain_type, manifest_type, expiration_timestamp FROM spine_block_manifest WHERE chain_type = 0 AND " + + "manifest_type = 0 ORDER BY manifest_reference_height DESC LIMIT 1", }, } for _, tt := range tests { @@ -122,9 +122,9 @@ func TestSpineBlockManifestQuery_GetSpineBlockManifestsInTimeInterval(t *testing fromTimestamp: 10, toTimestamp: 20, }, - want: "SELECT id, full_file_hash, file_chunk_hashes, manifest_reference_height, chain_type, manifest_type, " + - "manifest_timestamp FROM spine_block_manifest WHERE manifest_timestamp > 10 AND manifest_timestamp <= 20 ORDER" + - " BY manifest_type, chain_type, manifest_reference_height", + want: "SELECT id, full_file_hash, file_chunk_hashes, manifest_reference_height, manifest_spine_block_height, " + + "chain_type, manifest_type, expiration_timestamp FROM spine_block_manifest WHERE expiration_timestamp > 10 " + + "AND expiration_timestamp <= 20 ORDER BY manifest_type, chain_type, manifest_reference_height", }, } for _, tt := range tests { From 255592ddc85abb06725796cc7f048217e876db4f Mon Sep 17 00:00:00 2001 From: andy-shi88 Date: Tue, 21 Apr 2020 13:56:04 +0800 Subject: [PATCH 05/17] clean snapshot file on rollbacked --- core/service/blockSpineService.go | 38 +++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/core/service/blockSpineService.go b/core/service/blockSpineService.go index d8608fa8d..0a9876f8b 100644 --- a/core/service/blockSpineService.go +++ b/core/service/blockSpineService.go @@ -43,6 +43,7 @@ type ( SpinePublicKeyService BlockSpinePublicKeyServiceInterface SpineBlockManifestService SpineBlockManifestServiceInterface BlocksmithService BlocksmithServiceInterface + SnapshotMainBlockService SnapshotBlockServiceInterface } ) @@ -58,6 +59,7 @@ func NewBlockSpineService( logger *log.Logger, megablockQuery query.SpineBlockManifestQueryInterface, blocksmithService BlocksmithServiceInterface, + snapshotMainblockService SnapshotBlockServiceInterface, ) *BlockSpineService { return &BlockSpineService{ Chaintype: ct, @@ -80,7 +82,8 @@ func NewBlockSpineService( spineBlockQuery, logger, ), - BlocksmithService: blocksmithService, + BlocksmithService: blocksmithService, + SnapshotMainBlockService: snapshotMainblockService, } } @@ -497,12 +500,11 @@ func (bs *BlockSpineService) PopulateBlockData(block *model.Block) error { return blocker.NewBlocker(blocker.BlockErr, "error getting block spine public keys") } block.SpinePublicKeys = spinePublicKeys - spineBlockManifests, err := bs.SpineBlockManifestService.GetSpineBlockManifestsForSpineBlock(block.Height, block.Timestamp) + spineBlockManifests, err := bs.SpineBlockManifestService.GetSpineBlockManifestBySpineBlockHeight(block.Height) if err != nil { return blocker.NewBlocker(blocker.BlockErr, "error getting block spineBlockManifests") } block.SpineBlockManifests = spineBlockManifests - return nil } @@ -563,7 +565,10 @@ func (bs *BlockSpineService) GenerateBlock( if err != nil { return nil, err } - + // assign spine block height to every manifests + for _, spm := range spineBlockManifests { + spm.ManifestSpineBlockHeight = newBlockHeight + } // loop through transaction to build block hash digest.Reset() // reset the digest if _, err := digest.Write(previousBlock.GetBlockSeed()); err != nil { @@ -805,7 +810,10 @@ func (bs *BlockSpineService) PopOffToBlock(commonBlock *model.Block) ([]*model.B return []*model.Block{}, blocker.NewBlocker(blocker.BlockNotFoundErr, fmt.Sprintf("the common block is not found %v", commonBlock.ID)) } - var poppedBlocks []*model.Block + var ( + poppedBlocks []*model.Block + poppedManifests []*model.SpineBlockManifest + ) block := lastBlock for block.ID != commonBlock.ID && block.ID != bs.Chaintype.GetGenesisBlockID() { @@ -826,11 +834,27 @@ func (bs *BlockSpineService) PopOffToBlock(commonBlock *model.Block) ([]*model.B queries := dQuery.Rollback(commonBlock.Height) err = bs.QueryExecutor.ExecuteTransactions(queries) if err != nil { - _ = bs.QueryExecutor.RollbackTx() + rollbackErr := bs.QueryExecutor.RollbackTx() + if rollbackErr != nil { + bs.Logger.Warnf("spineblock-rollback-err: %v", rollbackErr) + } return []*model.Block{}, err } } - + // post rollback action: + // - clean snapshot data + poppedManifests, err = bs.SpineBlockManifestService.GetSpineBlockManifestsFromSpineBlockHeight(commonBlock.Height) + if err != nil { + rollbackErr := bs.QueryExecutor.RollbackTx() + if rollbackErr != nil { + bs.Logger.Warn(rollbackErr) + } + return []*model.Block{}, err + } + for _, manifest := range poppedManifests { + // ignore error, file deletion can fail + _ = bs.SnapshotMainBlockService.DeleteFileByChunkHashes(manifest.FileChunkHashes) + } err = bs.QueryExecutor.CommitTx() if err != nil { return nil, err From 68c9bdaf0786ad9dacbba25367291da5c3e6af8d Mon Sep 17 00:00:00 2001 From: andy-shi88 Date: Tue, 21 Apr 2020 13:57:28 +0800 Subject: [PATCH 06/17] add delete snapshot file function --- core/service/snapshotBasicChunkStrategy.go | 15 +++++++++++++-- core/service/snapshotBlockService.go | 1 + core/service/snapshotChunkStrategy.go | 1 + core/service/snapshotMainBlockService.go | 5 +++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/core/service/snapshotBasicChunkStrategy.go b/core/service/snapshotBasicChunkStrategy.go index 7bd129fd8..104ca9734 100644 --- a/core/service/snapshotBasicChunkStrategy.go +++ b/core/service/snapshotBasicChunkStrategy.go @@ -2,12 +2,13 @@ package service import ( "bytes" + "io/ioutil" + "path/filepath" + "github.com/zoobc/zoobc-core/common/blocker" "github.com/zoobc/zoobc-core/common/model" "github.com/zoobc/zoobc-core/common/util" "golang.org/x/crypto/sha3" - "io/ioutil" - "path/filepath" ) type ( @@ -109,3 +110,13 @@ func (ss *SnapshotBasicChunkStrategy) BuildSnapshotFromChunks(fullHash []byte, f } return snapshotPayload, nil } + +// DeleteFileByChunkHashes take in the concatenated file hashes (file name) and delete them. +func (ss *SnapshotBasicChunkStrategy) DeleteFileByChunkHashes(fileChunkHashes []byte, filePath string) error { + fileChunks := util.SplitByteSliceByChunkSize(fileChunkHashes, ss.ChunkSize) + err := ss.FileService.DeleteFilesByHash(filePath, fileChunks) + if err != nil { + return err + } + return nil +} diff --git a/core/service/snapshotBlockService.go b/core/service/snapshotBlockService.go index 3619b0f60..f8c4a4d84 100644 --- a/core/service/snapshotBlockService.go +++ b/core/service/snapshotBlockService.go @@ -9,5 +9,6 @@ type ( NewSnapshotFile(block *model.Block) (*model.SnapshotFileInfo, error) ImportSnapshotFile(snapshotFileInfo *model.SnapshotFileInfo) error IsSnapshotHeight(height uint32) bool + DeleteFileByChunkHashes(fileChunkHashes []byte) error } ) diff --git a/core/service/snapshotChunkStrategy.go b/core/service/snapshotChunkStrategy.go index 19c347f1f..397d24c6d 100644 --- a/core/service/snapshotChunkStrategy.go +++ b/core/service/snapshotChunkStrategy.go @@ -9,5 +9,6 @@ type ( GenerateSnapshotChunks(snapshotPayload *model.SnapshotPayload, filePath string) (fullHash []byte, fileChunkHashes [][]byte, err error) BuildSnapshotFromChunks(fullHash []byte, fileChunkHashes [][]byte, filePath string) (*model.SnapshotPayload, error) + DeleteFileByChunkHashes(concatenatedFileChunks []byte, filePath string) error } ) diff --git a/core/service/snapshotMainBlockService.go b/core/service/snapshotMainBlockService.go index a2277dfe1..5fe09ce5f 100644 --- a/core/service/snapshotMainBlockService.go +++ b/core/service/snapshotMainBlockService.go @@ -319,3 +319,8 @@ func (ss *SnapshotMainBlockService) InsertSnapshotPayloadToDB(payload *model.Sna } return nil } + +// DeleteFileByChunkHashes delete the files included in the file chunk hashes. +func (ss *SnapshotMainBlockService) DeleteFileByChunkHashes(fileChunkHashes []byte) error { + return ss.SnapshotBasicChunkStrategy.DeleteFileByChunkHashes(fileChunkHashes, ss.SnapshotPath) +} From 71f021d08e049c06ff52d86805eb0ce47463d757 Mon Sep 17 00:00:00 2001 From: andy-shi88 Date: Tue, 21 Apr 2020 13:57:50 +0800 Subject: [PATCH 07/17] update tests --- core/service/blockSpineService_test.go | 158 +++++++++++++++--- .../service/spineBlockManifestService_test.go | 34 ++-- 2 files changed, 149 insertions(+), 43 deletions(-) diff --git a/core/service/blockSpineService_test.go b/core/service/blockSpineService_test.go index fb905ba2c..bf1831f68 100644 --- a/core/service/blockSpineService_test.go +++ b/core/service/blockSpineService_test.go @@ -859,12 +859,12 @@ func TestBlockSpineService_PushBlock(t *testing.T) { SpineBlockManifestService: &mockSpineBlockManifestService{ ResSpineBlockManifests: []*model.SpineBlockManifest{ { - ID: 1, - FullFileHash: make([]byte, 64), - FileChunkHashes: make([]byte, 0), - SpineBlockManifestHeight: 720, - SpineBlockManifestType: model.SpineBlockManifestType_Snapshot, - ExpirationTimestamp: int64(1000), + ID: 1, + FullFileHash: make([]byte, 64), + FileChunkHashes: make([]byte, 0), + ManifestReferenceHeight: 720, + SpineBlockManifestType: model.SpineBlockManifestType_Snapshot, + ExpirationTimestamp: int64(1000), }, }, }, @@ -927,12 +927,12 @@ func TestBlockSpineService_PushBlock(t *testing.T) { SpineBlockManifestService: &mockSpineBlockManifestService{ ResSpineBlockManifests: []*model.SpineBlockManifest{ { - ID: 1, - FullFileHash: make([]byte, 64), - FileChunkHashes: make([]byte, 0), - SpineBlockManifestHeight: 720, - SpineBlockManifestType: model.SpineBlockManifestType_Snapshot, - ExpirationTimestamp: int64(1000), + ID: 1, + FullFileHash: make([]byte, 64), + FileChunkHashes: make([]byte, 0), + ManifestReferenceHeight: 720, + SpineBlockManifestType: model.SpineBlockManifestType_Snapshot, + ExpirationTimestamp: int64(1000), }, }, }, @@ -1313,6 +1313,22 @@ func (ss *mockSpineBlockManifestService) GetSpineBlockManifestsForSpineBlock( return spineBlockManifests, err } +func (ss *mockSpineBlockManifestService) GetSpineBlockManifestBySpineBlockHeight( + spineHeight uint32, +) ([]*model.SpineBlockManifest, error) { + var ( + spineBlockManifests = make([]*model.SpineBlockManifest, 0) + err error + ) + if ss.ResSpineBlockManifests != nil { + spineBlockManifests = ss.ResSpineBlockManifests + } + if ss.ResError != nil { + err = ss.ResError + } + return spineBlockManifests, err +} + func (*mockSpineReceiptServiceReturnEmpty) SelectReceipts(int64, uint32, uint32) ([]*model.PublishedReceipt, error) { return []*model.PublishedReceipt{}, nil } @@ -1448,12 +1464,12 @@ func TestBlockSpineService_GenerateBlock(t *testing.T) { SpineBlockManifestService: &mockSpineBlockManifestService{ ResSpineBlockManifests: []*model.SpineBlockManifest{ { - ID: 1, - FullFileHash: make([]byte, 64), - FileChunkHashes: make([]byte, 0), - SpineBlockManifestHeight: 720, - SpineBlockManifestType: model.SpineBlockManifestType_Snapshot, - ExpirationTimestamp: int64(1000), + ID: 1, + FullFileHash: make([]byte, 64), + FileChunkHashes: make([]byte, 0), + ManifestReferenceHeight: 720, + SpineBlockManifestType: model.SpineBlockManifestType_Snapshot, + ExpirationTimestamp: int64(1000), }, }, }, @@ -3382,6 +3398,57 @@ func (*mockSpineExecutorBlockPopSuccess) ExecuteSelectRow(qStr string, tx bool, return db.QueryRow(qStr), nil } +type ( + // mock PopOffToBlock + mockSnapshotMainBlockServiceDeleteFail struct { + SnapshotMainBlockService + } + mockSnapshotMainBlockServiceDeleteSuccess struct { + SnapshotMainBlockService + } + + mockSpineBlockManifestServiceFailGetManifestFromHeight struct { + SpineBlockManifestServiceInterface + } + + mockSpineBlockManifestServiceSuccesGetManifestFromHeight struct { + SpineBlockManifestServiceInterface + } +) + +func (*mockSnapshotMainBlockServiceDeleteFail) DeleteFileByChunkHashes([]byte) error { + return errors.New("mockedError") +} + +func (*mockSnapshotMainBlockServiceDeleteSuccess) DeleteFileByChunkHashes([]byte) error { + return nil +} + +func (*mockSpineBlockManifestServiceFailGetManifestFromHeight) GetSpineBlockManifestsFromSpineBlockHeight( + uint32, +) ([]*model.SpineBlockManifest, error) { + return []*model.SpineBlockManifest{}, errors.New("mockedError") +} + +func (*mockSpineBlockManifestServiceFailGetManifestFromHeight) GetSpineBlockManifestBySpineBlockHeight(uint32) ( + []*model.SpineBlockManifest, error, +) { + return make([]*model.SpineBlockManifest, 0), nil +} + +func (*mockSpineBlockManifestServiceSuccesGetManifestFromHeight) GetSpineBlockManifestsFromSpineBlockHeight( + uint32, +) ([]*model.SpineBlockManifest, error) { + return []*model.SpineBlockManifest{ + ssMockSpineBlockManifest, + }, nil +} + +func (*mockSpineBlockManifestServiceSuccesGetManifestFromHeight) GetSpineBlockManifestBySpineBlockHeight(uint32) ( + []*model.SpineBlockManifest, error, +) { + return make([]*model.SpineBlockManifest, 0), nil +} func TestBlockSpineService_PopOffToBlock(t *testing.T) { type fields struct { RWMutex sync.RWMutex @@ -3407,6 +3474,7 @@ func TestBlockSpineService_PopOffToBlock(t *testing.T) { Logger *log.Logger SpinePublicKeyService BlockSpinePublicKeyServiceInterface SpineBlockManifestService SpineBlockManifestServiceInterface + SnapshotMainBlockService SnapshotBlockServiceInterface } type args struct { commonBlock *model.Block @@ -3526,7 +3594,7 @@ func TestBlockSpineService_PopOffToBlock(t *testing.T) { wantErr: true, }, { - name: "Success", + name: "GetManifestFromSpineBlockHeight-Fail", fields: fields{ RWMutex: sync.RWMutex{}, Chaintype: &chaintype.SpineChain{}, @@ -3553,7 +3621,44 @@ func TestBlockSpineService_PopOffToBlock(t *testing.T) { Signature: nil, SpinePublicKeyQuery: query.NewSpinePublicKeyQuery(), }, - SpineBlockManifestService: &mockSpineBlockManifestService{}, + SpineBlockManifestService: &mockSpineBlockManifestServiceFailGetManifestFromHeight{}, + }, + args: args{ + commonBlock: mockSpineGoodCommonBlock, + }, + want: []*model.Block{}, + wantErr: true, + }, + { + name: "GetManifestFromSpineBlockHeight-Success", + fields: fields{ + RWMutex: sync.RWMutex{}, + Chaintype: &chaintype.SpineChain{}, + KVExecutor: nil, + QueryExecutor: &mockSpineExecutorBlockPopSuccess{}, + BlockQuery: query.NewBlockQuery(&chaintype.SpineChain{}), + MempoolQuery: nil, + TransactionQuery: query.NewTransactionQuery(&chaintype.SpineChain{}), + MerkleTreeQuery: nil, + PublishedReceiptQuery: nil, + SkippedBlocksmithQuery: nil, + Signature: nil, + MempoolService: &mockSpineMempoolServiceBlockPopSuccess{}, + ReceiptService: &mockSpineReceiptSuccess{}, + ActionTypeSwitcher: nil, + AccountBalanceQuery: nil, + ParticipationScoreQuery: nil, + Observer: nil, + Logger: log.New(), + SpinePublicKeyService: &BlockSpinePublicKeyService{ + Logger: log.New(), + NodeRegistrationQuery: query.NewNodeRegistrationQuery(), + QueryExecutor: &mockSpineExecutorBlockPopSuccess{}, + Signature: nil, + SpinePublicKeyQuery: query.NewSpinePublicKeyQuery(), + }, + SpineBlockManifestService: &mockSpineBlockManifestServiceSuccesGetManifestFromHeight{}, + SnapshotMainBlockService: &mockSnapshotMainBlockServiceDeleteSuccess{}, }, args: args{ commonBlock: mockSpineGoodCommonBlock, @@ -3574,6 +3679,7 @@ func TestBlockSpineService_PopOffToBlock(t *testing.T) { Logger: tt.fields.Logger, SpinePublicKeyService: tt.fields.SpinePublicKeyService, SpineBlockManifestService: tt.fields.SpineBlockManifestService, + SnapshotMainBlockService: tt.fields.SnapshotMainBlockService, } got, err := bs.PopOffToBlock(tt.args.commonBlock) if (err != nil) != tt.wantErr { @@ -3810,12 +3916,12 @@ func TestBlockSpineService_ValidateSpineBlockManifest(t *testing.T) { ResSpineBlockManifestBytes: []byte{1, 1, 1, 1, 1, 1, 1, 1}, ResSpineBlockManifests: []*model.SpineBlockManifest{ { - ID: 12345678, - FullFileHash: make([]byte, 64), - FileChunkHashes: make([]byte, 0), - SpineBlockManifestHeight: 720, - SpineBlockManifestType: model.SpineBlockManifestType_Snapshot, - ExpirationTimestamp: int64(1000), + ID: 12345678, + FullFileHash: make([]byte, 64), + FileChunkHashes: make([]byte, 0), + ManifestReferenceHeight: 720, + SpineBlockManifestType: model.SpineBlockManifestType_Snapshot, + ExpirationTimestamp: int64(1000), }, }, }, diff --git a/core/service/spineBlockManifestService_test.go b/core/service/spineBlockManifestService_test.go index fe63a27dc..c9510ec4f 100644 --- a/core/service/spineBlockManifestService_test.go +++ b/core/service/spineBlockManifestService_test.go @@ -20,13 +20,13 @@ type ( var ( ssMockSpineBlockManifest = &model.SpineBlockManifest{ - ID: 1, - FullFileHash: ssMockFullHash, - SpineBlockManifestHeight: 720, - FileChunkHashes: []byte{}, - ChainType: 0, - SpineBlockManifestType: model.SpineBlockManifestType_Snapshot, - ExpirationTimestamp: 1000, + ID: 1, + FullFileHash: ssMockFullHash, + ManifestReferenceHeight: 720, + FileChunkHashes: []byte{}, + ChainType: 0, + SpineBlockManifestType: model.SpineBlockManifestType_Snapshot, + ExpirationTimestamp: 1000, } ) @@ -98,13 +98,13 @@ func TestBlockSpineSnapshotService_CreateSpineBlockManifest(t *testing.T) { }, wantErr: false, want: &model.SpineBlockManifest{ - ID: int64(-446833005029263155), - FullFileHash: make([]byte, 32), - SpineBlockManifestHeight: ssMockMainBlock.Height, - ExpirationTimestamp: int64(1562117286), - FileChunkHashes: make([]byte, 0), - SpineBlockManifestType: model.SpineBlockManifestType_Snapshot, - ChainType: 0, + ID: int64(-4442731824309358759), + FullFileHash: make([]byte, 32), + ManifestReferenceHeight: ssMockMainBlock.Height, + ExpirationTimestamp: int64(1562117286), + FileChunkHashes: make([]byte, 0), + SpineBlockManifestType: model.SpineBlockManifestType_Snapshot, + ChainType: 0, }, }, } @@ -151,9 +151,9 @@ func TestSnapshotService_GetSpineBlockManifestBytes(t *testing.T) { args: args{ spineBlockManifest: ssMockSpineBlockManifest, }, - want: []byte{1, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 208, 2, 0, 0, 0, 0, 0, 0, 232, 3, 0, 0, 0, 0, 0, 0}, + want: []byte{1, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 208, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 3, 0, 0, 0, 0, 0, 0}, }, } for _, tt := range tests { From 073038de08ed6ed550ed6755989db91803228cd2 Mon Sep 17 00:00:00 2001 From: andy-shi88 Date: Tue, 21 Apr 2020 13:58:37 +0800 Subject: [PATCH 08/17] remove ErrNoRows check as that error only occur on sql.Row not sql.Rows --- core/service/snapshotService.go | 5 +- core/service/spineBlockManifestService.go | 67 +++++++++++++++++++---- 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/core/service/snapshotService.go b/core/service/snapshotService.go index f81437c4d..530ef45e0 100644 --- a/core/service/snapshotService.go +++ b/core/service/snapshotService.go @@ -2,13 +2,14 @@ package service import ( "fmt" + "time" + log "github.com/sirupsen/logrus" "github.com/zoobc/zoobc-core/common/blocker" "github.com/zoobc/zoobc-core/common/chaintype" "github.com/zoobc/zoobc-core/common/constant" "github.com/zoobc/zoobc-core/common/model" "github.com/zoobc/zoobc-core/observer" - "time" ) type ( @@ -79,7 +80,7 @@ func (ss *SnapshotService) StopSnapshotGeneration(ct chaintype.ChainType) error } stopSnapshotGeneration[ct.GetTypeInt()] <- true // TODO implement error handling for abrupt snapshot termination. for now we just wait a few seconds and return - time.Sleep(2 * time.Second) + time.Sleep(2 * time.Second) // todo: move the constant return nil } diff --git a/core/service/spineBlockManifestService.go b/core/service/spineBlockManifestService.go index 7f2792e22..13db02ed9 100644 --- a/core/service/spineBlockManifestService.go +++ b/core/service/spineBlockManifestService.go @@ -2,7 +2,6 @@ package service import ( "bytes" - "database/sql" log "github.com/sirupsen/logrus" "github.com/zoobc/zoobc-core/common/blocker" @@ -22,6 +21,12 @@ type ( ct chaintype.ChainType, mbType model.SpineBlockManifestType) (*model.SpineBlockManifest, error) GetSpineBlockManifestBytes(spineBlockManifest *model.SpineBlockManifest) []byte InsertSpineBlockManifest(spineBlockManifest *model.SpineBlockManifest) error + GetSpineBlockManifestBySpineBlockHeight(spineBlockHeight uint32) ( + []*model.SpineBlockManifest, error, + ) + GetSpineBlockManifestsFromSpineBlockHeight(spineBlockHeight uint32) ( + []*model.SpineBlockManifest, error, + ) } SpineBlockManifestService struct { @@ -46,6 +51,46 @@ func NewSpineBlockManifestService( } } +// GetSpineBlockManifestBySpineBlockHeight return all manifests published in spine block +func (ss *SpineBlockManifestService) GetSpineBlockManifestBySpineBlockHeight(spineBlockHeight uint32) ( + []*model.SpineBlockManifest, error, +) { + var ( + spineBlockManifests = make([]*model.SpineBlockManifest, 0) + ) + qry := ss.SpineBlockManifestQuery.GetManifestBySpineBlockHeight(spineBlockHeight) + rows, err := ss.QueryExecutor.ExecuteSelect(qry, false) + if err != nil { + return nil, err + } + defer rows.Close() + spineBlockManifests, err = ss.SpineBlockManifestQuery.BuildModel(spineBlockManifests, rows) + if err != nil { + return nil, blocker.NewBlocker(blocker.DBErr, err.Error()) + } + return spineBlockManifests, err +} + +// GetSpineBlockManifestsFromSpineBlockHeight return all manifest where height > spineBlockHeight +func (ss *SpineBlockManifestService) GetSpineBlockManifestsFromSpineBlockHeight(spineBlockHeight uint32) ( + []*model.SpineBlockManifest, error, +) { + var ( + spineBlockManifests = make([]*model.SpineBlockManifest, 0) + ) + qry := ss.SpineBlockManifestQuery.GetManifestsFromSpineBlockHeight(spineBlockHeight) + rows, err := ss.QueryExecutor.ExecuteSelect(qry, false) + if err != nil { + return nil, err + } + defer rows.Close() + spineBlockManifests, err = ss.SpineBlockManifestQuery.BuildModel(spineBlockManifests, rows) + if err != nil { + return nil, blocker.NewBlocker(blocker.DBErr, err.Error()) + } + return spineBlockManifests, err +} + // GetSpineBlockManifestsForSpineBlock retrieve all spineBlockManifests for a given spine height // if there are no spineBlockManifest at this height, return nil // spineHeight height of the spine block we want to fetch the spineBlockManifests for @@ -79,10 +124,7 @@ func (ss *SpineBlockManifestService) GetSpineBlockManifestsForSpineBlock(spineHe defer rows.Close() spineBlockManifests, err = ss.SpineBlockManifestQuery.BuildModel(spineBlockManifests, rows) if err != nil { - if err != sql.ErrNoRows { - return nil, blocker.NewBlocker(blocker.DBErr, err.Error()) - } - return nil, nil + return nil, blocker.NewBlocker(blocker.DBErr, err.Error()) } return spineBlockManifests, nil @@ -134,12 +176,12 @@ func (ss *SpineBlockManifestService) CreateSpineBlockManifest(fullFileHash []byt spineBlockManifest := &model.SpineBlockManifest{ // we store SpineBlockManifest ID as little endian of fullFileHash so that we can join the spineBlockManifest and // FileChunks tables if needed - FullFileHash: fullFileHash, - FileChunkHashes: megablockFileHashes, - SpineBlockManifestHeight: megablockHeight, - ChainType: ct.GetTypeInt(), - SpineBlockManifestType: mbType, - ExpirationTimestamp: expirationTimestamp, + FullFileHash: fullFileHash, + FileChunkHashes: megablockFileHashes, + ManifestReferenceHeight: megablockHeight, + ChainType: ct.GetTypeInt(), + SpineBlockManifestType: mbType, + ExpirationTimestamp: expirationTimestamp, } megablockID, err := ss.GetSpineBlockManifestID(spineBlockManifest) if err != nil { @@ -184,7 +226,8 @@ func (ss *SpineBlockManifestService) GetSpineBlockManifestBytes(spineBlockManife buffer.Write(spineBlockManifest.FullFileHash) // spineBlockManifest payload = all file chunks' entities bytes buffer.Write(spineBlockManifest.FileChunkHashes) - buffer.Write(util.ConvertUint32ToBytes(spineBlockManifest.SpineBlockManifestHeight)) + buffer.Write(util.ConvertUint32ToBytes(spineBlockManifest.ManifestReferenceHeight)) + buffer.Write(util.ConvertUint32ToBytes(spineBlockManifest.ManifestSpineBlockHeight)) buffer.Write(util.ConvertUint32ToBytes(uint32(spineBlockManifest.ChainType))) buffer.Write(util.ConvertUint64ToBytes(uint64(spineBlockManifest.ExpirationTimestamp))) return buffer.Bytes() From 87c72564537076976ca84bb10efcd481e3e87269 Mon Sep 17 00:00:00 2001 From: andy-shi88 Date: Tue, 21 Apr 2020 13:59:09 +0800 Subject: [PATCH 09/17] inject snapshot block service to spine block service --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 861efc766..d79e909b9 100644 --- a/main.go +++ b/main.go @@ -648,6 +648,7 @@ func startSpinechain() { loggerCoreService, query.NewSpineBlockManifestQuery(), spinechainBlocksmithService, + snapshotBlockServices[mainchain.GetTypeInt()], ) blockServices[spinechain.GetTypeInt()] = spinechainBlockService From 958f7305b82323047b977c6233c5a0569da0bc48 Mon Sep 17 00:00:00 2001 From: andy-shi88 Date: Tue, 21 Apr 2020 13:59:41 +0800 Subject: [PATCH 10/17] snapshot manifest field rename --- p2p/fileDownloader.go | 5 +++-- p2p/p2p.go | 1 + p2p/p2p_test.go | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/p2p/fileDownloader.go b/p2p/fileDownloader.go index e1c52eab0..9f0e262ba 100644 --- a/p2p/fileDownloader.go +++ b/p2p/fileDownloader.go @@ -2,6 +2,8 @@ package p2p import ( "fmt" + "sync" + log "github.com/sirupsen/logrus" "github.com/zoobc/zoobc-core/common/blocker" "github.com/zoobc/zoobc-core/common/chaintype" @@ -9,7 +11,6 @@ import ( "github.com/zoobc/zoobc-core/common/model" "github.com/zoobc/zoobc-core/core/service" "golang.org/x/crypto/sha3" - "sync" ) type ( @@ -92,7 +93,7 @@ func (ss *FileDownloader) DownloadSnapshot(ct chaintype.ChainType, spineBlockMan SnapshotFileHash: spineBlockManifest.GetFullFileHash(), FileChunksHashes: fileChunkHashes, ChainType: ct.GetTypeInt(), - Height: spineBlockManifest.SpineBlockManifestHeight, + Height: spineBlockManifest.ManifestReferenceHeight, ProcessExpirationTimestamp: spineBlockManifest.ExpirationTimestamp, SpineBlockManifestType: model.SpineBlockManifestType_Snapshot, }, nil diff --git a/p2p/p2p.go b/p2p/p2p.go index d8cbd4b75..c52aebe34 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -321,6 +321,7 @@ func (s *Peer2PeerService) DownloadFilesFromPeer(fileChunksNames []string, maxRe // download the files fileDownloadResponse, err := s.PeerServiceClient.RequestDownloadFile(peer, fileChunksToDownload) if err != nil { + log.Warnf("error download: %v\nchunks: %v\npeer: %v\n", err, fileChunksToDownload, peer) return nil, err } diff --git a/p2p/p2p_test.go b/p2p/p2p_test.go index d6807787a..276488d42 100644 --- a/p2p/p2p_test.go +++ b/p2p/p2p_test.go @@ -2,10 +2,11 @@ package p2p import ( "bytes" - "github.com/pkg/errors" "reflect" "testing" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" "github.com/zoobc/zoobc-core/common/model" "github.com/zoobc/zoobc-core/common/transaction" From 0d533d329882c867fce0df6632513d830ea9e1f8 Mon Sep 17 00:00:00 2001 From: andy-shi88 Date: Thu, 23 Apr 2020 09:54:38 +0800 Subject: [PATCH 11/17] fix new rollback query for manifest --- common/query/spineBlockManifestQuery.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/common/query/spineBlockManifestQuery.go b/common/query/spineBlockManifestQuery.go index d25dc1325..f3a63b488 100644 --- a/common/query/spineBlockManifestQuery.go +++ b/common/query/spineBlockManifestQuery.go @@ -5,8 +5,6 @@ import ( "fmt" "strings" - "github.com/zoobc/zoobc-core/common/constant" - "github.com/zoobc/zoobc-core/common/chaintype" "github.com/zoobc/zoobc-core/common/model" @@ -164,7 +162,7 @@ func (mbl *SpineBlockManifestQuery) Rollback(height uint32) (multiQueries [][]in return [][]interface{}{ { fmt.Sprintf("DELETE FROM %s WHERE manifest_spine_block_height > ?", mbl.getTableName()), - height - constant.MinRollbackBlocks, + height, }, } } From 74534d04a04d3bc1ede327ed5b689247c523d8b3 Mon Sep 17 00:00:00 2001 From: andy-shi88 Date: Thu, 23 Apr 2020 09:55:16 +0800 Subject: [PATCH 12/17] use parse instead of splitting chunks manually --- core/service/snapshotBasicChunkStrategy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/service/snapshotBasicChunkStrategy.go b/core/service/snapshotBasicChunkStrategy.go index 104ca9734..c2323f95a 100644 --- a/core/service/snapshotBasicChunkStrategy.go +++ b/core/service/snapshotBasicChunkStrategy.go @@ -113,8 +113,8 @@ func (ss *SnapshotBasicChunkStrategy) BuildSnapshotFromChunks(fullHash []byte, f // DeleteFileByChunkHashes take in the concatenated file hashes (file name) and delete them. func (ss *SnapshotBasicChunkStrategy) DeleteFileByChunkHashes(fileChunkHashes []byte, filePath string) error { - fileChunks := util.SplitByteSliceByChunkSize(fileChunkHashes, ss.ChunkSize) - err := ss.FileService.DeleteFilesByHash(filePath, fileChunks) + fileChunks, err := ss.FileService.ParseFileChunkHashes(fileChunkHashes, sha3.New256().Size()) + err = ss.FileService.DeleteFilesByHash(filePath, fileChunks) if err != nil { return err } From 70c58f51c496c73f66a0bda65262a65abd983ecc Mon Sep 17 00:00:00 2001 From: andy-shi88 Date: Thu, 23 Apr 2020 09:55:36 +0800 Subject: [PATCH 13/17] log the deletion error --- core/service/blockSpineService.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/service/blockSpineService.go b/core/service/blockSpineService.go index 0a9876f8b..b1a170ee7 100644 --- a/core/service/blockSpineService.go +++ b/core/service/blockSpineService.go @@ -853,7 +853,10 @@ func (bs *BlockSpineService) PopOffToBlock(commonBlock *model.Block) ([]*model.B } for _, manifest := range poppedManifests { // ignore error, file deletion can fail - _ = bs.SnapshotMainBlockService.DeleteFileByChunkHashes(manifest.FileChunkHashes) + deleteErr := bs.SnapshotMainBlockService.DeleteFileByChunkHashes(manifest.FileChunkHashes) + if deleteErr != nil { + log.Warnf("fail deleting snapshot during rollback: %v\n", deleteErr) + } } err = bs.QueryExecutor.CommitTx() if err != nil { From 4b9540fb830d954cbe8eed3682b3f45942dfec89 Mon Sep 17 00:00:00 2001 From: andy-shi88 Date: Thu, 23 Apr 2020 10:28:21 +0800 Subject: [PATCH 14/17] handle error --- core/service/snapshotBasicChunkStrategy.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/service/snapshotBasicChunkStrategy.go b/core/service/snapshotBasicChunkStrategy.go index c2323f95a..9987054e2 100644 --- a/core/service/snapshotBasicChunkStrategy.go +++ b/core/service/snapshotBasicChunkStrategy.go @@ -114,6 +114,9 @@ func (ss *SnapshotBasicChunkStrategy) BuildSnapshotFromChunks(fullHash []byte, f // DeleteFileByChunkHashes take in the concatenated file hashes (file name) and delete them. func (ss *SnapshotBasicChunkStrategy) DeleteFileByChunkHashes(fileChunkHashes []byte, filePath string) error { fileChunks, err := ss.FileService.ParseFileChunkHashes(fileChunkHashes, sha3.New256().Size()) + if err != nil { + return err + } err = ss.FileService.DeleteFilesByHash(filePath, fileChunks) if err != nil { return err From 9b57d809634259134bab8b90c6b3bbe20e03bfba Mon Sep 17 00:00:00 2001 From: andy-shi88 Date: Thu, 23 Apr 2020 11:06:43 +0800 Subject: [PATCH 15/17] update schema --- common/schema | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/schema b/common/schema index 4475b68cc..36246d29c 160000 --- a/common/schema +++ b/common/schema @@ -1 +1 @@ -Subproject commit 4475b68cca2ce4463c4bbdce6d93b8828d3d37da +Subproject commit 36246d29cbd983feb8dc001660dc37c196b210d8 From 3fe4829728bff636b1e02e54903680de85d2b1d1 Mon Sep 17 00:00:00 2001 From: andy-shi88 Date: Thu, 23 Apr 2020 12:12:47 +0800 Subject: [PATCH 16/17] add extra comment for testing --- common/constant/blockchainSync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/constant/blockchainSync.go b/common/constant/blockchainSync.go index ebd421057..f4e6ce297 100644 --- a/common/constant/blockchainSync.go +++ b/common/constant/blockchainSync.go @@ -11,7 +11,7 @@ const ( PeerGetBlocksLimit uint32 = 1440 CommonMilestoneBlockIdsLimit int32 = 10 SafeBlockGap = MinRollbackBlocks / 2 - // @iltoga change this to 2 for testing snapshots + // @iltoga change this to 2 for testing snapshots to maintain functionality keep this value above PriorityStrategyBuildScrambleNodesGap MinRollbackBlocks uint32 = 720 // production 720 MaxCommonMilestoneRequestTrial = MinRollbackBlocks/uint32(CommonMilestoneBlockIdsLimit) + 1 MinimumPeersBlocksToDownload int32 = 2 From 0692cefc7a6ac4c6547e5914867d3d38e3283082 Mon Sep 17 00:00:00 2001 From: andy-shi88 Date: Thu, 23 Apr 2020 14:11:20 +0800 Subject: [PATCH 17/17] move file deletion to after db commit and as goroutine --- core/service/blockSpineService.go | 35 +++++++++++++------------ core/service/blockSpineService_test.go | 36 -------------------------- 2 files changed, 18 insertions(+), 53 deletions(-) diff --git a/core/service/blockSpineService.go b/core/service/blockSpineService.go index b1a170ee7..0dd694c6e 100644 --- a/core/service/blockSpineService.go +++ b/core/service/blockSpineService.go @@ -841,27 +841,28 @@ func (bs *BlockSpineService) PopOffToBlock(commonBlock *model.Block) ([]*model.B return []*model.Block{}, err } } - // post rollback action: - // - clean snapshot data - poppedManifests, err = bs.SpineBlockManifestService.GetSpineBlockManifestsFromSpineBlockHeight(commonBlock.Height) - if err != nil { - rollbackErr := bs.QueryExecutor.RollbackTx() - if rollbackErr != nil { - bs.Logger.Warn(rollbackErr) - } - return []*model.Block{}, err - } - for _, manifest := range poppedManifests { - // ignore error, file deletion can fail - deleteErr := bs.SnapshotMainBlockService.DeleteFileByChunkHashes(manifest.FileChunkHashes) - if deleteErr != nil { - log.Warnf("fail deleting snapshot during rollback: %v\n", deleteErr) - } - } err = bs.QueryExecutor.CommitTx() if err != nil { return nil, err } + go func() { + // post rollback action: + // - clean snapshot data + poppedManifests, err = bs.SpineBlockManifestService.GetSpineBlockManifestsFromSpineBlockHeight(commonBlock.Height) + if err != nil { + rollbackErr := bs.QueryExecutor.RollbackTx() + if rollbackErr != nil { + bs.Logger.Warn(rollbackErr) + } + } + for _, manifest := range poppedManifests { + // ignore error, file deletion can fail + deleteErr := bs.SnapshotMainBlockService.DeleteFileByChunkHashes(manifest.FileChunkHashes) + if deleteErr != nil { + log.Warnf("fail deleting snapshot during rollback: %v\n", deleteErr) + } + } + }() return poppedBlocks, nil } diff --git a/core/service/blockSpineService_test.go b/core/service/blockSpineService_test.go index bc2d007cc..03c68d7ee 100644 --- a/core/service/blockSpineService_test.go +++ b/core/service/blockSpineService_test.go @@ -3607,42 +3607,6 @@ func TestBlockSpineService_PopOffToBlock(t *testing.T) { want: make([]*model.Block, 0), wantErr: true, }, - { - name: "GetManifestFromSpineBlockHeight-Fail", - fields: fields{ - RWMutex: sync.RWMutex{}, - Chaintype: &chaintype.SpineChain{}, - KVExecutor: nil, - QueryExecutor: &mockSpineExecutorBlockPopSuccess{}, - BlockQuery: query.NewBlockQuery(&chaintype.SpineChain{}), - MempoolQuery: nil, - TransactionQuery: query.NewTransactionQuery(&chaintype.SpineChain{}), - MerkleTreeQuery: nil, - PublishedReceiptQuery: nil, - SkippedBlocksmithQuery: nil, - Signature: nil, - MempoolService: &mockSpineMempoolServiceBlockPopSuccess{}, - ReceiptService: &mockSpineReceiptSuccess{}, - ActionTypeSwitcher: nil, - AccountBalanceQuery: nil, - ParticipationScoreQuery: nil, - Observer: nil, - Logger: log.New(), - SpinePublicKeyService: &BlockSpinePublicKeyService{ - Logger: log.New(), - NodeRegistrationQuery: query.NewNodeRegistrationQuery(), - QueryExecutor: &mockSpineExecutorBlockPopSuccess{}, - Signature: nil, - SpinePublicKeyQuery: query.NewSpinePublicKeyQuery(), - }, - SpineBlockManifestService: &mockSpineBlockManifestServiceFailGetManifestFromHeight{}, - }, - args: args{ - commonBlock: mockSpineGoodCommonBlock, - }, - want: []*model.Block{}, - wantErr: true, - }, { name: "GetManifestFromSpineBlockHeight-Success", fields: fields{