decimal: add support decimal type in msgpack

This patch provides decimal support for all space operations and as
function return result. Decimal type was introduced in Tarantool 2.2.
See more about decimal type in [1] and [2].

According to BCD encoding/decoding specification sign is encoded by
letters: '0x0a', '0x0c', '0x0e', '0x0f' stands for plus, and '0x0b' and
'0x0d' for minus. Tarantool always uses '0x0c' for plus and '0x0d' for
minus. Implementation in Golang follows the same rule and in all test
samples sign encoded by '0x0d' and '0x0c' for simplification.

Because 'c' used by Tarantool.

To use decimal with github.com/shopspring/decimal in msgpack, import
tarantool/decimal submodule.

1. https://www.tarantool.io/en/doc/latest/book/box/data_model/
2. https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-decimal-type
3. https://github.com/douglascrockford/DEC64/blob/663f562a5f0621021b98bfdd4693571993316174/dec64_test.c#L62-L104
4. https://github.com/shopspring/decimal/blob/v1.3.1/decimal_test.go#L27-L64
5. https://github.com/tarantool/tarantool/blob/60fe9d14c1c7896aa7d961e4b68649eddb4d2d6c/test/unit/decimal.c#L154-L171

Lua snippet for encoding number to MsgPack representation:

local decimal = require('decimal')

local function mp_encode_dec(num)
    local dec = msgpack.encode(decimal.new(num))
    return dec:gsub('.', function (c)
        return string.format('%02x', string.byte(c))
    end)
end
print(mp_encode_dec(-12.34)) -- 0xd6010201234d

Follows up tarantool/tarantool#692
Part of #96

Author: ligurio

CHANGELOG.md

@@ -17,6 +17,7 @@ Versioning](http://semver.org/spec/v2.0.0.html) except to the first release.
 - Go modules support (#91)
 - queue-utube handling (#85)
 - Master discovery (#113)
+- Support decimal type in msgpack (#96)
 
 ### Fixed
 

Makefile

@@ -45,6 +45,12 @@ test-main:
 	go clean -testcache
 	go test . -v -p 1
 
+.PHONY: test-decimal
+test-decimal:
+	@echo "Running tests in decimal package"
+	go clean -testcache
+	go test ./decimal/ -v -p 1
+
 .PHONY: coverage
 coverage:
 	go clean -testcache

decimal/bcd.go

@@ -0,0 +1,210 @@
+// Package decimal implements methods to encode and decode BCD.
+//
+// BCD (Binary-Coded Decimal) is a sequence of bytes representing decimal
+// digits of the encoded number (each byte has two decimal digits each encoded
+// using 4-bit nibbles), so byte >> 4 is the first digit and byte & 0x0f is the
+// second digit. The leftmost digit in the array is the most significant. The
+// rightmost digit in the array is the least significant.
+//
+// The first byte of the BCD array contains the first digit of the number,
+// represented as follows:
+//
+//     |  4 bits           |  4 bits           |
+//        = 0x                = the 1st digit
+//
+// (The first nibble contains 0 if the decimal number has an even number of
+// digits). The last byte of the BCD array contains the last digit of the
+// number and the final nibble, represented as follows:
+//
+//     |  4 bits           |  4 bits           |
+//        = the last digit    = nibble
+//
+// The final nibble represents the number's sign: 0x0a, 0x0c, 0x0e, 0x0f stand
+// for plus, 0x0b and 0x0d stand for minus.
+//
+// Examples:
+//
+// The decimal -12.34 will be encoded as 0xd6, 0x01, 0x02, 0x01, 0x23, 0x4d:
+//
+// |MP_EXT (fixext 4) | MP_DECIMAL | scale |  1   |  2,3 |  4 (minus) |
+// |       0xd6       |    0x01    | 0x02  | 0x01 | 0x23 | 0x4d       |
+//
+// The decimal 0.000000000000000000000000000000000010 will be encoded as
+// 0xc7, 0x03, 0x01, 0x24, 0x01, 0x0c:
+//
+// | MP_EXT (ext 8) | length | MP_DECIMAL | scale |  1   | 0 (plus) |
+// |      0xc7      |  0x03  |    0x01    | 0x24  | 0x01 | 0x0c     |
+//
+// See also:
+//
+// * MessagePack extensions https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/
+//
+// * An implementation in C language https://github.com/tarantool/decNumber/blob/master/decPacked.c
+package decimal
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"unicode"
+)
+
+var mapRuneToByteSign = map[rune]byte{
+	'+': 0x0c,
+	'-': 0x0d,
+}
+
+var isNegative = map[byte]bool{
+	0x0a: false,
+	0x0b: true,
+	0x0c: false,
+	0x0d: true,
+	0x0e: false,
+	0x0f: false,
+}
+
+// EncodeStringToBCD converts a string buffer to BCD Packed Decimal.
+//
+// The number is converted to a BCD packed decimal byte array, right aligned in
+// the BCD array, whose length is indicated by the second parameter. The final
+// 4-bit nibble in the array will be a sign nibble, 0x0a for "+" and 0x0b for
+// "-". Unused bytes and nibbles to the left of the number are set to 0. scale
+// is set to the scale of the number (this is the exponent, negated).
+func EncodeStringToBCD(buf string) ([]byte, error) {
+	sign := '+' // By default number is positive.
+	if res := strings.HasPrefix(buf, "-"); res {
+		sign = '-'
+	}
+
+	// Calculate a number of digits in the decimal number. Leading and
+	// trailing zeros do not count.
+	// Examples:
+	// 0.0000000000000001 - 1 digit
+	// 00012.34           - 4 digits
+	// 0.340              - 2 digits
+	s := strings.ReplaceAll(buf, "-", "") // Remove a sign.
+	s = strings.ReplaceAll(s, "+", "")    // Remove a sign.
+	s = strings.ReplaceAll(s, ".", "")    // Remove a dot.
+	s = strings.TrimLeft(s, "0")          // Remove leading zeros.
+	c := len(s)
+
+	// Fix a case with a single 0.
+	if c == 0 {
+		s = "0" //nolint
+		c = 1
+	}
+
+	// The first nibble should contain 0, if the decimal number has an even
+	// number of digits. Therefore highNibble is false when decimal number
+	// is even.
+	highNibble := true
+	if c%2 == 0 {
+		highNibble = false
+	}
+	scale := 0 // By default decimal number is integer.
+	var byteBuf []byte
+	for i, ch := range buf {
+		// Skip leading zeros.
+		if (len(byteBuf) == 0) && ch == '0' {
+			continue
+		}
+		if (i == 0) && (ch == '-' || ch == '+') {
+			continue
+		}
+		// Calculate a number of digits after the decimal point.
+		if ch == '.' {
+			scale = len(buf) - i - 1
+			continue
+		}
+		if !unicode.IsDigit(ch) {
+			return nil, fmt.Errorf("Symbol in position %d is not a digit: %c", i, ch)
+		}
+
+		d, err := strconv.Atoi(string(ch))
+		if err != nil {
+			return nil, fmt.Errorf("Failed to convert symbol '%c' to a digit: %s", ch, err)
+		}
+		digit := byte(d)
+		if highNibble {
+			// Add a digit to a high nibble.
+			digit = digit << 4
+			byteBuf = append(byteBuf, digit)
+			highNibble = false
+		} else {
+			if len(byteBuf) == 0 {
+				byteBuf = make([]byte, 1)
+			}
+			// Add a digit to a low nibble.
+			lowByteIdx := len(byteBuf) - 1
+			byteBuf[lowByteIdx] = byteBuf[lowByteIdx] | digit
+			highNibble = true
+		}
+	}
+	if highNibble {
+		// Put a sign to a high nibble.
+		byteBuf = append(byteBuf, mapRuneToByteSign[sign])
+	} else {
+		// Put a sign to a low nibble.
+		lowByteIdx := len(byteBuf) - 1
+		byteBuf[lowByteIdx] = byteBuf[lowByteIdx] | mapRuneToByteSign[sign]
+	}
+	byteBuf = append([]byte{byte(scale)}, byteBuf...)
+
+	return byteBuf, nil
+}
+
+// DecodeStringFromBCD converts a BCD Packed Decimal to a string buffer.
+//
+// The BCD packed decimal byte array, together with an associated scale, is
+// converted to a string. The BCD array is assumed full of digits, and must be
+// ended by a 4-bit sign nibble in the least significant four bits of the final
+// byte. The scale is used (negated) as the exponent of the decimal number.
+// Note that zeros may have a sign and/or a scale.
+func DecodeStringFromBCD(bcdBuf []byte) (string, error) {
+	const scaleIdx = 0 // Index of a byte with scale.
+	scale := int(bcdBuf[scaleIdx])
+	// Get a BCD buffer without a byte with scale.
+	bcdBuf = bcdBuf[scaleIdx+1:]
+	length := len(bcdBuf)
+	var digits []string
+	for i, bcdByte := range bcdBuf {
+		highNibble := int(bcdByte >> 4)
+		if !(len(digits) == 0 && highNibble == 0) || len(bcdBuf) == 1 {
+			digits = append(digits, strconv.Itoa(highNibble))
+		}
+		lowNibble := int(bcdByte & 0x0f)
+		if !(len(digits) == 0 && lowNibble == 0) && i != length-1 {
+			digits = append(digits, strconv.Itoa(int(lowNibble)))
+		}
+	}
+
+	// Add missing zeros when scale is less than current length.
+	l := len(digits)
+	if scale >= l {
+		var zeros []string
+		for i := 0; i <= scale-l; i++ {
+			zeros = append(zeros, "0")
+			digits = append(zeros, digits...)
+		}
+	}
+
+	// Add a dot when number is fractional.
+	if scale != 0 {
+		idx := len(digits) - scale
+		digits = append(digits, "X")       // [1 2 3 X]
+		copy(digits[idx:], digits[idx-1:]) // [1 2 2 3]
+		digits[idx] = "."                  // [1 . 2 3]
+	}
+
+	// Add a sign, it is encoded in a low nibble of a last byte.
+	lastByte := bcdBuf[length-1]
+	sign := lastByte & 0x0f
+	if isNegative[sign] {
+		digits = append([]string{"-"}, digits...)
+	}
+
+	// Merge slice to a single string.
+	str := strings.Join(digits, "")
+
+	return str, nil
+}

decimal/config.lua

@@ -0,0 +1,41 @@
+local decimal = require('decimal')
+local msgpack = require('msgpack')
+
+-- Do not set listen for now so connector won't be
+-- able to send requests until everything is configured.
+box.cfg{
+    work_dir = os.getenv("TEST_TNT_WORK_DIR"),
+}
+
+box.schema.user.create('test', { password = 'test' , if_not_exists = true })
+box.schema.user.grant('test', 'execute', 'universe', nil, { if_not_exists = true })
+
+local decimal_msgpack_supported = pcall(msgpack.encode, decimal.new(1))
+if not decimal_msgpack_supported then
+    error('Decimal unsupported, use Tarantool 2.2 or newer')
+end
+
+local s = box.schema.space.create('testDecimal', {
+    id = 524,
+    if_not_exists = true,
+})
+s:create_index('primary', {
+    type = 'TREE',
+    parts = {
+        {
+            field = 1,
+            type = 'decimal',
+        },
+    },
+    if_not_exists = true
+})
+s:truncate()
+
+box.schema.user.grant('test', 'read,write', 'space', 'testDecimal', { if_not_exists = true })
+
+-- Set listen only when every other thing is configured.
+box.cfg{
+    listen = os.getenv("TEST_TNT_LISTEN"),
+}
+
+require('console').start()

decimal/decimal.go

@@ -0,0 +1,87 @@
+// Package decimal with support of Tarantool's decimal data type.
+//
+// Decimal data type supported in Tarantool since 2.2.
+//
+// Since: 1.6
+//
+// See also:
+//
+// * Tarantool MessagePack extensions https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-decimal-type
+//
+// * Tarantool data model https://www.tarantool.io/en/doc/latest/book/box/data_model/
+//
+// * Tarantool issue for support decimal type https://github.com/tarantool/tarantool/issues/692
+package decimal
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/shopspring/decimal"
+	"gopkg.in/vmihailenco/msgpack.v2"
+)
+
+// Decimal external type.
+const decimalExtID = 1
+
+func encodeDecimal(e *msgpack.Encoder, v reflect.Value) error {
+	number := v.Interface().(decimal.Decimal)
+	strBuf := number.String()
+	bcdBuf, err := EncodeStringToBCD(strBuf)
+	if err != nil {
+		return fmt.Errorf("msgpack: can't encode string (%s) to a BCD buffer: %w", strBuf, err)
+	}
+	if _, err = e.Writer().Write(bcdBuf); err != nil {
+		return fmt.Errorf("msgpack: can't write bytes to encoder writer: %w", err)
+	}
+
+	return nil
+}
+
+// Decimal values can be encoded to fixext MessagePack, where buffer
+// has a fixed length encoded by first byte, and ext MessagePack, where
+// buffer length is not fixed and encoded by a number in a separate
+// field:
+//
+// +--------+-------------------+------------+===============+
+// | MP_EXT | length (optional) | MP_DECIMAL | PackedDecimal |
+// +--------+-------------------+------------+===============+
+//
+// Before reading a buffer with encoded decimal number (PackedDecimal) we need
+// to allocate it, but before reading we don't know it's exact size.
+// msgpack.Decoder in msgpack v2 package pass a buffer with PackedDecimal bytes
+// and there is no possibility to read length in advance. to obtain MessagePack
+// length in advance. For example on attempt to decode MessagePack buffer
+// c7030100088c (88) msgpack pass 00088a to decodeDecimal() and length field
+// value (03) is unavailable nor in buffer nor via decoder interface.
+//
+// To solve a problem we allocate a buffer with maximum size (it is 32 bytes
+// and it is corresponds to ext32 MessagePack), then read encoded decimal to a
+// buffer and finally cut off unused part.
+func decodeDecimal(d *msgpack.Decoder, v reflect.Value) error {
+	var maxBytesCount int = 32
+	b := make([]byte, maxBytesCount)
+
+	readBytes, err := d.Buffered().Read(b)
+	if err != nil {
+		return fmt.Errorf("msgpack: can't read bytes on decimal decode: %w", err)
+	}
+	b = b[:readBytes]
+	digits, err := DecodeStringFromBCD(b)
+	if err != nil {
+		return fmt.Errorf("msgpack: can't decode string from BCD buffer (%x): %w", b, err)
+	}
+	dec, err := decimal.NewFromString(digits)
+	if err != nil {
+		return fmt.Errorf("msgpack: can't encode string (%s) to a decimal number: %w", digits, err)
+	}
+
+	v.Set(reflect.ValueOf(dec))
+
+	return nil
+}
+
+func init() {
+	msgpack.Register(reflect.TypeOf((*decimal.Decimal)(nil)).Elem(), encodeDecimal, decodeDecimal)
+	msgpack.RegisterExt(decimalExtID, (*decimal.Decimal)(nil))
+}

decimal/decimal_test.go

@@ -0,0 +1,56 @@
+// Run Tarantool instance before example execution:
+//
+// Terminal 1:
+// $ cd decimal
+// $ TEST_TNT_LISTEN=3013 TEST_TNT_WORK_DIR=$(mktemp -d -t 'tarantool.XXX') tarantool config.lua
+//
+// Terminal 2:
+// $ go test -v example_test.go
+package decimal_test
+
+import (
+	"log"
+	"time"
+
+	"github.com/shopspring/decimal"
+	"github.com/tarantool/go-tarantool"
+	_ "github.com/tarantool/go-tarantool/decimal"
+)
+
+// To enable support of decimal in msgpack with
+// https://github.com/shopspring/decimal,
+// import tarantool/decimal submodule.
+func Example() {
+	server := "127.0.0.1:3013"
+	opts := tarantool.Opts{
+		Timeout:       500 * time.Millisecond,
+		Reconnect:     1 * time.Second,
+		MaxReconnects: 3,
+		User:          "test",
+		Pass:          "test",
+	}
+	client, err := tarantool.Connect(server, opts)
+	if err != nil {
+		log.Fatalf("Failed to connect: %s", err.Error())
+	}
+
+	spaceNo := uint32(524)
+
+	number, err := decimal.NewFromString("-22.804")
+	if err != nil {
+		log.Fatalf("Failed to prepare test decimal: %s", err)
+	}
+
+	resp, err := client.Replace(spaceNo, []interface{}{number})
+	if err != nil {
+		log.Fatalf("Decimal replace failed: %s", err)
+	}
+	if resp == nil {
+		log.Fatalf("Response is nil after Replace")
+	}
+
+	log.Println("Decimal tuple replace")
+	log.Println("Error", err)
+	log.Println("Code", resp.Code)
+	log.Println("Data", resp.Data)
+}

decimal/example_test.go

@@ -3,10 +3,14 @@ module github.com/tarantool/go-tarantool
 go 1.11
 
 require (
+	github.com/google/go-cmp v0.5.7 // indirect
 	github.com/google/uuid v1.3.0
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/stretchr/testify v1.7.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/shopspring/decimal v1.3.1
 	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 	gopkg.in/vmihailenco/msgpack.v2 v2.9.2
+	gotest.tools v2.2.0+incompatible
 )

go.mod

@@ -2,6 +2,8 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -14,13 +16,18 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
+github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -30,3 +37,5 @@ gopkg.in/vmihailenco/msgpack.v2 v2.9.2 h1:gjPqo9orRVlSAH/065qw3MsFCDpH7fa1KpiizX
 gopkg.in/vmihailenco/msgpack.v2 v2.9.2/go.mod h1:/3Dn1Npt9+MYyLpYYXjInO/5jvMLamn+AEGwNEOatn8=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
+gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=