Spine block manifest and spine public keys validation when downloading blocks from peers

common/constant/blockchainSync.go

@@ -11,6 +11,7 @@ const (
 	PeerGetBlocksLimit               uint32        = 1440
 	CommonMilestoneBlockIdsLimit     int32         = 10
 	SafeBlockGap                     uint32        = 1440
-	MinRollbackBlocks                uint32        = 720
-	MaxCommonMilestoneRequestTrial   uint32        = MinRollbackBlocks/uint32(CommonMilestoneBlockIdsLimit) + 1
+	// @iltoga change this to 2 for testing snapshots
+	MinRollbackBlocks              uint32 = 720 // production 720
+	MaxCommonMilestoneRequestTrial uint32 = MinRollbackBlocks/uint32(CommonMilestoneBlockIdsLimit) + 1
 )

common/constant/genesis.go

@@ -3,7 +3,7 @@
 package constant
 
 const (
-	MainchainGenesisBlockID int64 = 3638197708165910115
+	MainchainGenesisBlockID int64 = 1070983609761144356
 )
 
 type (

common/constant/snapshot.go

@@ -12,5 +12,5 @@ const (
 	// @iltoga reduce to 1 for testing locally
 	SnapshotChunkSize int = int(100 * 1024) // 10 KB
 	// DownloadSnapshotNumberOfRetries number of times to retry downloading failed snapshot file chunks from other peers
-	DownloadSnapshotNumberOfRetries uint32 = 3
+	DownloadSnapshotNumberOfRetries uint32 = 5
 )

common/monitoring/metricsMonitoring.go

@@ -48,7 +48,7 @@ const (
 	P2pGetCommonMilestoneBlockIDsServer = "P2pGetCommonMilestoneBlockIDsServer"
 	P2pGetNextBlockIDsServer            = "P2pGetNextBlockIDsServer"
 	P2pGetNextBlocksServer              = "P2pGetNextBlocksServer"
-	P2pRequestFileDownloadServer 		= "P2pRequestFileDownloadServer"
+	P2pRequestFileDownloadServer        = "P2pRequestFileDownloadServer"
 
 	P2pGetPeerInfoClient                = "P2pGetPeerInfoClient"
 	P2pGetMorePeersClient               = "P2pGetMorePeersClient"
@@ -60,7 +60,7 @@ const (
 	P2pGetCommonMilestoneBlockIDsClient = "P2pGetCommonMilestoneBlockIDsClient"
 	P2pGetNextBlockIDsClient            = "P2pGetNextBlockIDsClient"
 	P2pGetNextBlocksClient              = "P2pGetNextBlocksClient"
-	P2pRequestFileDownloadClient 		= "P2pRequestFileDownloadClient"
+	P2pRequestFileDownloadClient        = "P2pRequestFileDownloadClient"
 )
 
 func SetMonitoringActive(isActive bool) {

common/query/blockQuery.go

@@ -15,9 +15,10 @@ type (
 		GetBlocks(height, size uint32) string
 		GetLastBlock() string
 		GetGenesisBlock() string
-		GetBlockByID(int64) string
-		GetBlockByHeight(uint32) string
-		GetBlockFromHeight(uint32, uint32) string
+		GetBlockByID(id int64) string
+		GetBlockByHeight(height uint32) string
+		GetBlockFromHeight(startHeight, limit uint32) string
+		GetBlockFromTimestamp(startTimestamp int64, limit uint32) string
 		InsertBlock(block *model.Block) (str string, args []interface{})
 		ExtractModel(block *model.Block) []interface{}
 		BuildModel(blocks []*model.Block, rows *sql.Rows) ([]*model.Block, error)
@@ -93,10 +94,16 @@ func (bq *BlockQuery) GetBlockByHeight(height uint32) string {
 
 // GetBlockFromHeight returns query string to get blocks from a certain height
 func (bq *BlockQuery) GetBlockFromHeight(startHeight, limit uint32) string {
-	return fmt.Sprintf("SELECT %s FROM %s WHERE HEIGHT >= %d ORDER BY HEIGHT LIMIT %d",
+	return fmt.Sprintf("SELECT %s FROM %s WHERE height >= %d ORDER BY height LIMIT %d",
 		strings.Join(bq.Fields, ", "), bq.getTableName(), startHeight, limit)
 }
 
+// GetBlockFromTimestamp returns query string to get blocks from a certain block timestamp
+func (bq *BlockQuery) GetBlockFromTimestamp(startTimestamp int64, limit uint32) string {
+	return fmt.Sprintf("SELECT %s FROM %s WHERE timestamp >= %d ORDER BY timestamp LIMIT %d",
+		strings.Join(bq.Fields, ", "), bq.getTableName(), startTimestamp, limit)
+}
+
 // ExtractModel extract the model struct fields to the order of BlockQuery.Fields
 func (*BlockQuery) ExtractModel(block *model.Block) []interface{} {
 	return []interface{}{

common/query/blockQuery_test.go

@@ -5,10 +5,8 @@ import (
 	"testing"
 
 	"github.com/DATA-DOG/go-sqlmock"
-
-	"github.com/zoobc/zoobc-core/common/model"
-
 	"github.com/zoobc/zoobc-core/common/chaintype"
+	"github.com/zoobc/zoobc-core/common/model"
 )
 
 var (
@@ -249,3 +247,87 @@ func TestBlockQuery_Rollback(t *testing.T) {
 		})
 	}
 }
+
+func TestBlockQuery_GetBlockFromHeight(t *testing.T) {
+	type fields struct {
+		Fields    []string
+		TableName string
+		ChainType chaintype.ChainType
+	}
+	type args struct {
+		startHeight uint32
+		limit       uint32
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   string
+	}{
+		{
+			name:   "GetBlockFromHeight:success",
+			fields: fields(*mockBlockQuery),
+			args: args{
+				limit:       1,
+				startHeight: 1,
+			},
+			want: "SELECT id, block_hash, previous_block_hash, height, timestamp, block_seed, block_signature, " +
+				"cumulative_difficulty, payload_length, payload_hash, blocksmith_public_key, total_amount, total_fee, " +
+				"total_coinbase, version FROM main_block WHERE height >= 1 ORDER BY height LIMIT 1",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			bq := &BlockQuery{
+				Fields:    tt.fields.Fields,
+				TableName: tt.fields.TableName,
+				ChainType: tt.fields.ChainType,
+			}
+			if got := bq.GetBlockFromHeight(tt.args.startHeight, tt.args.limit); got != tt.want {
+				t.Errorf("BlockQuery.GetBlockFromHeight() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestBlockQuery_GetBlockFromTimestamp(t *testing.T) {
+	type fields struct {
+		Fields    []string
+		TableName string
+		ChainType chaintype.ChainType
+	}
+	type args struct {
+		startTimestamp int64
+		limit          uint32
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   string
+	}{
+		{
+			name:   "GetBlockFromTimestamp:success",
+			fields: fields(*mockBlockQuery),
+			args: args{
+				limit:          1,
+				startTimestamp: 15875392,
+			},
+			want: "SELECT id, block_hash, previous_block_hash, height, timestamp, block_seed, block_signature, " +
+				"cumulative_difficulty, payload_length, payload_hash, blocksmith_public_key, total_amount, total_fee, " +
+				"total_coinbase, version FROM main_block WHERE timestamp >= 15875392 ORDER BY timestamp LIMIT 1",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			bq := &BlockQuery{
+				Fields:    tt.fields.Fields,
+				TableName: tt.fields.TableName,
+				ChainType: tt.fields.ChainType,
+			}
+			if got := bq.GetBlockFromTimestamp(tt.args.startTimestamp, tt.args.limit); got != tt.want {
+				t.Errorf("BlockQuery.GetBlockFromTimestamp() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

core/service/blockCoreService.go

@@ -22,6 +22,8 @@ type (
 			timestamp int64,
 		) (*model.Block, error)
 		ValidateBlock(block, previousLastBlock *model.Block, curTime int64) error
+		ValidatePayloadHash(block *model.Block) error
+		GetPayloadHashAndLength(block *model.Block) (payloadHash []byte, payloadLength uint32, err error)
 		PushBlock(previousBlock, block *model.Block, broadcast, persist bool) error
 		GetBlockByID(id int64, withAttachedData bool) (*model.Block, error)
 		GetBlockByHeight(uint32) (*model.Block, error)

core/service/blockMainService.go

@@ -33,6 +33,7 @@ import (
 )
 
 type (
+	// BlockServiceMainInterface interface that contains methods specific of BlockService
 	BlockServiceMainInterface interface {
 		NewMainBlock(
 			version uint32,
@@ -163,10 +164,12 @@ func (bs *BlockService) NewMainBlock(
 	totalCoinBase int64,
 	transactions []*model.Transaction,
 	publishedReceipts []*model.PublishedReceipt,
-	payloadHash []byte,
-	payloadLength uint32,
 	secretPhrase string,
 ) (*model.Block, error) {
+	var (
+		err error
+	)
+
 	block := &model.Block{
 		Version:             version,
 		PreviousBlockHash:   previousBlockHash,
@@ -179,9 +182,13 @@ func (bs *BlockService) NewMainBlock(
 		TotalCoinBase:       totalCoinBase,
 		Transactions:        transactions,
 		PublishedReceipts:   publishedReceipts,
-		PayloadHash:         payloadHash,
-		PayloadLength:       payloadLength,
 	}
+
+	// compute block's payload hash and length and add it to block struct
+	if block.PayloadHash, block.PayloadLength, err = bs.GetPayloadHashAndLength(block); err != nil {
+		return nil, err
+	}
+
 	blockUnsignedByte, err := util.GetBlockByte(block, false, bs.Chaintype)
 	if err != nil {
 		bs.Logger.Error(err.Error())
@@ -258,7 +265,19 @@ func (bs *BlockService) NewGenesisBlock(
 	return block, nil
 }
 
-// PreValidateBlock valdiate block without it's transactions
+// ValidatePayloadHash validate (computed) block's payload data hash against block's payload hash
+func (bs *BlockService) ValidatePayloadHash(block *model.Block) error {
+	hash, length, err := bs.GetPayloadHashAndLength(block)
+	if err != nil {
+		return err
+	}
+	if length != block.GetPayloadLength() || !bytes.Equal(hash, block.GetPayloadHash()) {
+		return blocker.NewBlocker(blocker.ValidationErr, "InvalidBlockPayload")
+	}
+	return nil
+}
+
+// PreValidateBlock validate block without it's transactions
 func (bs *BlockService) PreValidateBlock(block, previousLastBlock *model.Block) error {
 	// check if blocksmith can smith at the time
 	blocksmithsMap := bs.BlocksmithStrategy.GetSortedBlocksmithsMap(previousLastBlock)
@@ -276,6 +295,10 @@ func (bs *BlockService) PreValidateBlock(block, previousLastBlock *model.Block)
 
 // ValidateBlock validate block to be pushed into the blockchain
 func (bs *BlockService) ValidateBlock(block, previousLastBlock *model.Block, curTime int64) error {
+	if err := bs.ValidatePayloadHash(block); err != nil {
+		return err
+	}
+
 	// check block timestamp
 	if block.GetTimestamp() > curTime+constant.GenerateBlockTimeoutSec {
 		return blocker.NewBlocker(blocker.BlockErr, "InvalidTimestamp")
@@ -896,6 +919,33 @@ func (bs *BlockService) RemoveMempoolTransactions(transactions []*model.Transact
 	return nil
 }
 
+func (bs *BlockService) GetPayloadHashAndLength(block *model.Block) (payloadHash []byte, payloadLength uint32, err error) {
+	var (
+		digest = sha3.New256()
+	)
+	for _, tx := range block.GetTransactions() {
+		if _, err := digest.Write(tx.GetTransactionHash()); err != nil {
+			return nil, 0, err
+		}
+		txType, err := bs.ActionTypeSwitcher.GetTransactionType(tx)
+		if err != nil {
+			return nil, 0, err
+		}
+		payloadLength += txType.GetSize()
+	}
+	// filter only good receipt
+	for _, br := range block.GetPublishedReceipts() {
+		brBytes := bs.ReceiptUtil.GetSignedBatchReceiptBytes(br.BatchReceipt)
+		_, err = digest.Write(brBytes)
+		if err != nil {
+			return nil, 0, err
+		}
+		payloadLength += uint32(len(brBytes))
+	}
+	payloadHash = digest.Sum([]byte{})
+	return
+}
+
 // GenerateBlock generate block from transactions in mempool
 func (bs *BlockService) GenerateBlock(
 	previousBlock *model.Block,
@@ -904,11 +954,9 @@ func (bs *BlockService) GenerateBlock(
 ) (*model.Block, error) {
 	var (
 		totalAmount, totalFee, totalCoinbase int64
-		payloadLength                        uint32
 		// only for mainchain
 		sortedTransactions  []*model.Transaction
 		publishedReceipts   []*model.PublishedReceipt
-		payloadHash         []byte
 		err                 error
 		digest              = sha3.New256()
 		blockSmithPublicKey = crypto.NewEd25519Signature().GetPublicKeyFromSeed(secretPhrase)
@@ -922,44 +970,29 @@ func (bs *BlockService) GenerateBlock(
 	}
 	// select transactions from mempool to be added to the block
 	for _, tx := range sortedTransactions {
-		if _, err = digest.Write(tx.TransactionHash); err != nil {
-			return nil, err
-		}
 		txType, errType := bs.ActionTypeSwitcher.GetTransactionType(tx)
 		if errType != nil {
 			return nil, err
 		}
 		totalAmount += txType.GetAmount()
 		totalFee += tx.Fee
-		payloadLength += txType.GetSize()
 	}
 	// select published receipts to be added to the block
 	publishedReceipts, err = bs.ReceiptService.SelectReceipts(
 		timestamp, bs.ReceiptUtil.GetNumberOfMaxReceipts(
 			len(bs.BlocksmithStrategy.GetSortedBlocksmiths(previousBlock))),
 		previousBlock.Height,
 	)
-	// FIXME: add published receipts to block payload length
-
 	if err != nil {
 		return nil, err
 	}
-	// filter only good receipt
-	for _, br := range publishedReceipts {
-		_, err = digest.Write(bs.ReceiptUtil.GetSignedBatchReceiptBytes(br.BatchReceipt))
-		if err != nil {
-			return nil, err
-		}
-	}
 
-	payloadHash = digest.Sum([]byte{})
 	// loop through transaction to build block hash
-	digest.Reset() // reset the digest
 	if _, err = digest.Write(previousBlock.GetBlockSeed()); err != nil {
 		return nil, err
 	}
-
 	previousSeedHash := digest.Sum([]byte{})
+
 	blockSeed := bs.Signature.SignByNode(previousSeedHash, secretPhrase)
 	digest.Reset() // reset the digest
 	// compute the previous block hash
@@ -979,8 +1012,6 @@ func (bs *BlockService) GenerateBlock(
 		totalCoinbase,
 		sortedTransactions,
 		publishedReceipts,
-		payloadHash,
-		payloadLength,
 		secretPhrase,
 	)
 	if err != nil {