From 1e1b1204c210b2da1604fe8cd888631e423a4222 Mon Sep 17 00:00:00 2001 From: Sergey Bronnikov Date: Wed, 2 Feb 2022 18:57:45 +0300 Subject: [PATCH 1/2] decimal: add support decimal type in msgpack This patch provides decimal support for all space operations and as function return result. Decimal type was introduced in Tarantool 2.2. See more about decimal type in [1] and [2]. According to BCD encoding/decoding specification sign is encoded by letters: '0x0a', '0x0c', '0x0e', '0x0f' stands for plus, and '0x0b' and '0x0d' for minus. Tarantool always uses '0x0c' for plus and '0x0d' for minus. Implementation in Golang follows the same rule and in all test samples sign encoded by '0x0d' and '0x0c' for simplification. Because 'c' used by Tarantool. To use decimal with github.com/shopspring/decimal in msgpack, import tarantool/decimal submodule. 1. https://www.tarantool.io/en/doc/latest/book/box/data_model/ 2. https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-decimal-type 3. https://github.com/douglascrockford/DEC64/blob/663f562a5f0621021b98bfdd4693571993316174/dec64_test.c#L62-L104 4. https://github.com/shopspring/decimal/blob/v1.3.1/decimal_test.go#L27-L64 5. https://github.com/tarantool/tarantool/blob/60fe9d14c1c7896aa7d961e4b68649eddb4d2d6c/test/unit/decimal.c#L154-L171 Lua snippet for encoding number to MsgPack representation: local decimal = require('decimal') local msgpack = require('msgpack') local function mp_encode_dec(num) local dec = msgpack.encode(decimal.new(num)) return dec:gsub('.', function (c) return string.format('%02x', string.byte(c)) end) end print(mp_encode_dec(-12.34)) -- 0xd6010201234d Follows up https://github.com/tarantool/tarantool/issues/692 Part of #96 Co-authored-by: Oleg Jukovec --- CHANGELOG.md | 1 + Makefile | 8 +- decimal/bcd.go | 257 ++++++++++++++++ decimal/config.lua | 41 +++ decimal/decimal.go | 105 +++++++ decimal/decimal_test.go | 644 ++++++++++++++++++++++++++++++++++++++++ decimal/example_test.go | 55 ++++ decimal/export_test.go | 17 ++ go.mod | 4 + go.sum | 29 ++ 10 files changed, 1160 insertions(+), 1 deletion(-) create mode 100644 decimal/bcd.go create mode 100644 decimal/config.lua create mode 100644 decimal/decimal.go create mode 100644 decimal/decimal_test.go create mode 100644 decimal/example_test.go create mode 100644 decimal/export_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 6664a1cd4..2e4c72d53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Versioning](http://semver.org/spec/v2.0.0.html) except to the first release. - SSL support (#155) - IPROTO_PUSH messages support (#67) - Public API with request object types (#126) +- Support decimal type in msgpack (#96) ### Changed diff --git a/Makefile b/Makefile index 06aa01189..c9daf85ae 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,12 @@ test-connection-pool: go clean -testcache go test -tags "$(TAGS)" ./connection_pool/ -v -p 1 +.PHONY: test-decimal +test-decimal: + @echo "Running tests in decimal package" + go clean -testcache + go test -tags "$(TAGS)" ./decimal/ -v -p 1 + .PHONY: test-multi test-multi: @echo "Running tests in multiconnection package" @@ -75,7 +81,7 @@ test-main: coverage: go clean -testcache go get golang.org/x/tools/cmd/cover - go test -tags "$(TAGS)" ./... -v -p 1 -covermode=atomic -coverprofile=$(COVERAGE_FILE) -coverpkg=./... + go test -tags "$(TAGS)" ./... -v -p 1 -covermode=atomic -coverprofile=$(COVERAGE_FILE) go tool cover -func=$(COVERAGE_FILE) .PHONY: coveralls diff --git a/decimal/bcd.go b/decimal/bcd.go new file mode 100644 index 000000000..4cc57aa45 --- /dev/null +++ b/decimal/bcd.go @@ -0,0 +1,257 @@ +// Package decimal implements methods to encode and decode BCD. +// +// BCD (Binary-Coded Decimal) is a sequence of bytes representing decimal +// digits of the encoded number (each byte has two decimal digits each encoded +// using 4-bit nibbles), so byte >> 4 is the first digit and byte & 0x0f is the +// second digit. The leftmost digit in the array is the most significant. The +// rightmost digit in the array is the least significant. +// +// The first byte of the BCD array contains the first digit of the number, +// represented as follows: +// +// | 4 bits | 4 bits | +// = 0x = the 1st digit +// +// (The first nibble contains 0 if the decimal number has an even number of +// digits). The last byte of the BCD array contains the last digit of the +// number and the final nibble, represented as follows: +// +// | 4 bits | 4 bits | +// = the last digit = nibble +// +// The final nibble represents the number's sign: 0x0a, 0x0c, 0x0e, 0x0f stand +// for plus, 0x0b and 0x0d stand for minus. +// +// Examples: +// +// The decimal -12.34 will be encoded as 0xd6, 0x01, 0x02, 0x01, 0x23, 0x4d: +// +// | MP_EXT (fixext 4) | MP_DECIMAL | scale | 1 | 2,3 | 4 (minus) | +// | 0xd6 | 0x01 | 0x02 | 0x01 | 0x23 | 0x4d | +// +// The decimal 0.000000000000000000000000000000000010 will be encoded as +// 0xc7, 0x03, 0x01, 0x24, 0x01, 0x0c: +// +// | MP_EXT (ext 8) | length | MP_DECIMAL | scale | 1 | 0 (plus) | +// | 0xc7 | 0x03 | 0x01 | 0x24 | 0x01 | 0x0c | +// +// See also: +// +// * MessagePack extensions https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/ +// +// * An implementation in C language https://github.com/tarantool/decNumber/blob/master/decPacked.c +package decimal + +import ( + "fmt" + "strings" +) + +const ( + bytePlus = byte(0x0c) + byteMinus = byte(0x0d) +) + +var isNegative = [256]bool{ + 0x0a: false, + 0x0b: true, + 0x0c: false, + 0x0d: true, + 0x0e: false, + 0x0f: false, +} + +// Calculate a number of digits in a buffer with decimal number. +// +// Plus, minus, point and leading zeroes do not count. +// Contains a quirk for a zero - returns 1. +// +// Examples (see more examples in tests): +// +// - 0.0000000000000001 - 1 digit +// +// - 00012.34 - 4 digits +// +// - 0.340 - 3 digits +// +// - 0 - 1 digit +func getNumberLength(buf string) int { + if len(buf) == 0 { + return 0 + } + n := 0 + for _, ch := range []byte(buf) { + if ch >= '1' && ch <= '9' { + n += 1 + } else if ch == '0' && n != 0 { + n += 1 + } + } + + // Fix a case with a single 0. + if n == 0 { + n = 1 + } + + return n +} + +// encodeStringToBCD converts a string buffer to BCD Packed Decimal. +// +// The number is converted to a BCD packed decimal byte array, right aligned in +// the BCD array, whose length is indicated by the second parameter. The final +// 4-bit nibble in the array will be a sign nibble, 0x0c for "+" and 0x0d for +// "-". Unused bytes and nibbles to the left of the number are set to 0. scale +// is set to the scale of the number (this is the exponent, negated). +func encodeStringToBCD(buf string) ([]byte, error) { + if len(buf) == 0 { + return nil, fmt.Errorf("Length of number is zero") + } + signByte := bytePlus // By default number is positive. + if buf[0] == '-' { + signByte = byteMinus + } + + // The first nibble should contain 0, if the decimal number has an even + // number of digits. Therefore highNibble is false when decimal number + // is even. + highNibble := true + l := GetNumberLength(buf) + if l%2 == 0 { + highNibble = false + } + scale := 0 // By default decimal number is integer. + var byteBuf []byte + for i, ch := range []byte(buf) { + // Skip leading zeroes. + if (len(byteBuf) == 0) && ch == '0' { + continue + } + if (i == 0) && (ch == '-' || ch == '+') { + continue + } + // Calculate a number of digits after the decimal point. + if ch == '.' { + if scale != 0 { + return nil, fmt.Errorf("Number contains more than one point") + } + scale = len(buf) - i - 1 + continue + } + + if ch < '0' || ch > '9' { + return nil, fmt.Errorf("Failed to convert symbol '%c' to a digit", ch) + } + digit := byte(ch - '0') + if highNibble { + // Add a digit to a high nibble. + digit = digit << 4 + byteBuf = append(byteBuf, digit) + highNibble = false + } else { + if len(byteBuf) == 0 { + byteBuf = make([]byte, 1) + } + // Add a digit to a low nibble. + lowByteIdx := len(byteBuf) - 1 + byteBuf[lowByteIdx] = byteBuf[lowByteIdx] | digit + highNibble = true + } + } + if len(byteBuf) == 0 { + // a special case: -0 + signByte = bytePlus + } + if highNibble { + // Put a sign to a high nibble. + byteBuf = append(byteBuf, signByte) + } else { + // Put a sign to a low nibble. + lowByteIdx := len(byteBuf) - 1 + byteBuf[lowByteIdx] = byteBuf[lowByteIdx] | signByte + } + byteBuf = append([]byte{byte(scale)}, byteBuf...) + + return byteBuf, nil +} + +// decodeStringFromBCD converts a BCD Packed Decimal to a string buffer. +// +// The BCD packed decimal byte array, together with an associated scale, is +// converted to a string. The BCD array is assumed full of digits, and must be +// ended by a 4-bit sign nibble in the least significant four bits of the final +// byte. The scale is used (negated) as the exponent of the decimal number. +// Note that zeroes may have a sign and/or a scale. +func decodeStringFromBCD(bcdBuf []byte) (string, error) { + // Index of a byte with scale. + const scaleIdx = 0 + scale := int(bcdBuf[scaleIdx]) + + // Get a BCD buffer without scale. + bcdBuf = bcdBuf[scaleIdx+1:] + bufLen := len(bcdBuf) + + // Every nibble contains a digit, and the last low nibble contains a + // sign. + ndigits := bufLen*2 - 1 + + // The first nibble contains 0 if the decimal number has an even number of + // digits. Decrease a number of digits if so. + if bcdBuf[0]&0xf0 == 0 { + ndigits -= 1 + } + + // Reserve bytes for dot and sign. + numLen := ndigits + 2 + // Reserve bytes for zeroes. + if scale >= ndigits { + numLen += scale - ndigits + } + + var bld strings.Builder + bld.Grow(numLen) + + // Add a sign, it is encoded in a low nibble of a last byte. + lastByte := bcdBuf[bufLen-1] + sign := lastByte & 0x0f + if isNegative[sign] { + bld.WriteByte('-') + } + + // Add missing zeroes to the left side when scale is bigger than a + // number of digits and a single missed zero to the right side when + // equal. + if scale > ndigits { + bld.WriteByte('0') + bld.WriteByte('.') + for diff := scale - ndigits; diff > 0; diff-- { + bld.WriteByte('0') + } + } else if scale == ndigits { + bld.WriteByte('0') + } + + const MaxDigit = 0x09 + // Builds a buffer with symbols of decimal number (digits, dot and sign). + processNibble := func(nibble byte) { + if nibble <= MaxDigit { + if ndigits == scale { + bld.WriteByte('.') + } + bld.WriteByte(nibble + '0') + ndigits-- + } + } + + for i, bcdByte := range bcdBuf { + highNibble := bcdByte >> 4 + lowNibble := bcdByte & 0x0f + // Skip a first high nibble as no digit there. + if i != 0 || highNibble != 0 { + processNibble(highNibble) + } + processNibble(lowNibble) + } + + return bld.String(), nil +} diff --git a/decimal/config.lua b/decimal/config.lua new file mode 100644 index 000000000..58b038958 --- /dev/null +++ b/decimal/config.lua @@ -0,0 +1,41 @@ +local decimal = require('decimal') +local msgpack = require('msgpack') + +-- Do not set listen for now so connector won't be +-- able to send requests until everything is configured. +box.cfg{ + work_dir = os.getenv("TEST_TNT_WORK_DIR"), +} + +box.schema.user.create('test', { password = 'test' , if_not_exists = true }) +box.schema.user.grant('test', 'execute', 'universe', nil, { if_not_exists = true }) + +local decimal_msgpack_supported = pcall(msgpack.encode, decimal.new(1)) +if not decimal_msgpack_supported then + error('Decimal unsupported, use Tarantool 2.2 or newer') +end + +local s = box.schema.space.create('testDecimal', { + id = 524, + if_not_exists = true, +}) +s:create_index('primary', { + type = 'TREE', + parts = { + { + field = 1, + type = 'decimal', + }, + }, + if_not_exists = true +}) +s:truncate() + +box.schema.user.grant('test', 'read,write', 'space', 'testDecimal', { if_not_exists = true }) + +-- Set listen only when every other thing is configured. +box.cfg{ + listen = os.getenv("TEST_TNT_LISTEN"), +} + +require('console').start() diff --git a/decimal/decimal.go b/decimal/decimal.go new file mode 100644 index 000000000..e6513af29 --- /dev/null +++ b/decimal/decimal.go @@ -0,0 +1,105 @@ +// Package decimal with support of Tarantool's decimal data type. +// +// Decimal data type supported in Tarantool since 2.2. +// +// Since: 1.7.0 +// +// See also: +// +// * Tarantool MessagePack extensions https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-decimal-type +// +// * Tarantool data model https://www.tarantool.io/en/doc/latest/book/box/data_model/ +// +// * Tarantool issue for support decimal type https://github.com/tarantool/tarantool/issues/692 +// +// * Tarantool module decimal https://www.tarantool.io/en/doc/latest/reference/reference_lua/decimal/ +package decimal + +import ( + "fmt" + + "github.com/shopspring/decimal" + "gopkg.in/vmihailenco/msgpack.v2" +) + +// Decimal numbers have 38 digits of precision, that is, the total +// number of digits before and after the decimal point can be 38. +// A decimal operation will fail if overflow happens (when a number is +// greater than 10^38 - 1 or less than -10^38 - 1). +// +// See also: +// +// * Tarantool module decimal https://www.tarantool.io/en/doc/latest/reference/reference_lua/decimal/ + +const ( + // Decimal external type. + decimalExtID = 1 + decimalPrecision = 38 +) + +type DecNumber struct { + decimal.Decimal +} + +// NewDecNumber creates a new DecNumber from a decimal.Decimal. +func NewDecNumber(decimal decimal.Decimal) *DecNumber { + return &DecNumber{Decimal: decimal} +} + +// NewDecNumberFromString creates a new DecNumber from a string. +func NewDecNumberFromString(src string) (result *DecNumber, err error) { + dec, err := decimal.NewFromString(src) + if err != nil { + return + } + result = NewDecNumber(dec) + return +} + +var _ msgpack.Marshaler = (*DecNumber)(nil) +var _ msgpack.Unmarshaler = (*DecNumber)(nil) + +func (decNum *DecNumber) MarshalMsgpack() ([]byte, error) { + one := decimal.NewFromInt(1) + maxSupportedDecimal := decimal.New(1, DecimalPrecision).Sub(one) // 10^DecimalPrecision - 1 + minSupportedDecimal := maxSupportedDecimal.Neg().Sub(one) // -10^DecimalPrecision - 1 + if decNum.GreaterThan(maxSupportedDecimal) { + return nil, fmt.Errorf("msgpack: decimal number is bigger than maximum supported number (10^%d - 1)", DecimalPrecision) + } + if decNum.LessThan(minSupportedDecimal) { + return nil, fmt.Errorf("msgpack: decimal number is lesser than minimum supported number (-10^%d - 1)", DecimalPrecision) + } + + strBuf := decNum.String() + bcdBuf, err := encodeStringToBCD(strBuf) + if err != nil { + return nil, fmt.Errorf("msgpack: can't encode string (%s) to a BCD buffer: %w", strBuf, err) + } + return bcdBuf, nil +} + +// Decimal values can be encoded to fixext MessagePack, where buffer +// has a fixed length encoded by first byte, and ext MessagePack, where +// buffer length is not fixed and encoded by a number in a separate +// field: +// +// +--------+-------------------+------------+===============+ +// | MP_EXT | length (optional) | MP_DECIMAL | PackedDecimal | +// +--------+-------------------+------------+===============+ +func (decNum *DecNumber) UnmarshalMsgpack(b []byte) error { + digits, err := decodeStringFromBCD(b) + if err != nil { + return fmt.Errorf("msgpack: can't decode string from BCD buffer (%x): %w", b, err) + } + dec, err := decimal.NewFromString(digits) + *decNum = *NewDecNumber(dec) + if err != nil { + return fmt.Errorf("msgpack: can't encode string (%s) to a decimal number: %w", digits, err) + } + + return nil +} + +func init() { + msgpack.RegisterExt(decimalExtID, &DecNumber{}) +} diff --git a/decimal/decimal_test.go b/decimal/decimal_test.go new file mode 100644 index 000000000..68bf299e0 --- /dev/null +++ b/decimal/decimal_test.go @@ -0,0 +1,644 @@ +package decimal_test + +import ( + "encoding/hex" + "fmt" + "log" + "os" + "reflect" + "testing" + "time" + + "github.com/shopspring/decimal" + . "github.com/tarantool/go-tarantool" + . "github.com/tarantool/go-tarantool/decimal" + "github.com/tarantool/go-tarantool/test_helpers" + "gopkg.in/vmihailenco/msgpack.v2" +) + +var isDecimalSupported = false + +var server = "127.0.0.1:3013" +var opts = Opts{ + Timeout: 500 * time.Millisecond, + User: "test", + Pass: "test", +} + +func skipIfDecimalUnsupported(t *testing.T) { + t.Helper() + + if isDecimalSupported == false { + t.Skip("Skipping test for Tarantool without datetime support in msgpack") + } +} + +var space = "testDecimal" +var index = "primary" + +type TupleDecimal struct { + number DecNumber +} + +func (t *TupleDecimal) EncodeMsgpack(e *msgpack.Encoder) error { + if err := e.EncodeSliceLen(1); err != nil { + return err + } + return e.EncodeValue(reflect.ValueOf(&t.number)) +} + +func (t *TupleDecimal) DecodeMsgpack(d *msgpack.Decoder) error { + var err error + var l int + if l, err = d.DecodeSliceLen(); err != nil { + return err + } + if l != 1 { + return fmt.Errorf("Array length doesn't match: %d", l) + } + + res, err := d.DecodeInterface() + if err != nil { + return err + } + t.number = res.(DecNumber) + + return nil +} + +var benchmarkSamples = []struct { + numString string + mpBuf string + fixExt bool +}{ + {"0.7", "d501017c", true}, + {"0.3", "d501013c", true}, + {"0.00000000000000000000000000000000000001", "d501261c", true}, + {"0.00000000000000000000000000000000000009", "d501269c", true}, + {"-18.34", "d6010201834d", true}, + {"-108.123456789", "d701090108123456789d", true}, + {"-11111111111111111111111111111111111111", "c7150100011111111111111111111111111111111111111d", false}, +} + +var correctnessSamples = []struct { + numString string + mpBuf string + fixExt bool +}{ + {"100", "c7030100100c", false}, + {"0.1", "d501011c", true}, + {"-0.1", "d501011d", true}, + {"0.0000000000000000000000000000000000001", "d501251c", true}, + {"-0.0000000000000000000000000000000000001", "d501251d", true}, + {"0.00000000000000000000000000000000000001", "d501261c", true}, + {"-0.00000000000000000000000000000000000001", "d501261d", true}, + {"1", "d501001c", true}, + {"-1", "d501001d", true}, + {"0", "d501000c", true}, + {"-0", "d501000c", true}, + {"0.01", "d501021c", true}, + {"0.001", "d501031c", true}, + {"99999999999999999999999999999999999999", "c7150100099999999999999999999999999999999999999c", false}, + {"-99999999999999999999999999999999999999", "c7150100099999999999999999999999999999999999999d", false}, + {"-12.34", "d6010201234d", true}, + {"12.34", "d6010201234c", true}, + {"1.4", "c7030101014c", false}, + {"2.718281828459045", "c70a010f02718281828459045c", false}, + {"-2.718281828459045", "c70a010f02718281828459045d", false}, + {"3.141592653589793", "c70a010f03141592653589793c", false}, + {"-3.141592653589793", "c70a010f03141592653589793d", false}, + {"1234567891234567890.0987654321987654321", "c7150113012345678912345678900987654321987654321c", false}, + {"-1234567891234567890.0987654321987654321", "c7150113012345678912345678900987654321987654321d", false}, +} + +// There is a difference between encoding result from a raw string and from +// decimal.Decimal. It's expected because decimal.Decimal simplifies decimals: +// 0.00010000 -> 0.0001 + +var rawSamples = []struct { + numString string + mpBuf string + fixExt bool +}{ + {"0.000000000000000000000000000000000010", "c7030124010c", false}, + {"0.010", "c7030103010c", false}, + {"123.456789000000000", "c70b010f0123456789000000000c", false}, +} + +var decimalSamples = []struct { + numString string + mpBuf string + fixExt bool +}{ + {"0.000000000000000000000000000000000010", "d501231c", true}, + {"0.010", "d501021c", true}, + {"123.456789000000000", "c7060106123456789c", false}, +} + +func TestMPEncodeDecode(t *testing.T) { + for _, testcase := range benchmarkSamples { + t.Run(testcase.numString, func(t *testing.T) { + decNum, err := NewDecNumberFromString(testcase.numString) + if err != nil { + t.Fatal(err) + } + var buf []byte + tuple := TupleDecimal{number: *decNum} + if buf, err = msgpack.Marshal(&tuple); err != nil { + t.Fatalf("Failed to encode decimal number '%s' to a MessagePack buffer: %s", testcase.numString, err) + } + var v TupleDecimal + if err = msgpack.Unmarshal(buf, &v); err != nil { + t.Fatalf("Failed to decode MessagePack buffer '%x' to a decimal number: %s", buf, err) + } + if !decNum.Equal(v.number.Decimal) { + fmt.Println(decNum) + fmt.Println(v.number) + t.Fatal("Decimal numbers are not equal") + } + }) + } +} + +var lengthSamples = []struct { + numString string + length int +}{ + {"0.010", 2}, + {"0.01", 1}, + {"-0.1", 1}, + {"0.1", 1}, + {"0", 1}, + {"00.1", 1}, + {"100", 3}, + {"0100", 3}, + {"+1", 1}, + {"-1", 1}, + {"1", 1}, + {"-12.34", 4}, + {"123.456789000000000", 18}, +} + +func TestGetNumberLength(t *testing.T) { + for _, testcase := range lengthSamples { + t.Run(testcase.numString, func(t *testing.T) { + l := GetNumberLength(testcase.numString) + if l != testcase.length { + t.Fatalf("Length is wrong: correct %d, incorrect %d", testcase.length, l) + } + }) + } + + if l := GetNumberLength(""); l != 0 { + t.Fatalf("Length is wrong: correct 0, incorrect %d", l) + } + + if l := GetNumberLength("0"); l != 1 { + t.Fatalf("Length is wrong: correct 0, incorrect %d", l) + } + + if l := GetNumberLength("10"); l != 2 { + t.Fatalf("Length is wrong: correct 0, incorrect %d", l) + } +} + +func TestEncodeStringToBCDIncorrectNumber(t *testing.T) { + referenceErrMsg := "Number contains more than one point" + var numString = "0.1.0" + buf, err := EncodeStringToBCD(numString) + if err == nil { + t.Fatalf("no error on encoding a string with incorrect number") + } + if buf != nil { + t.Fatalf("buf is not nil on encoding of a string with double points") + } + if err.Error() != referenceErrMsg { + t.Fatalf("wrong error message on encoding of a string double points") + } + + referenceErrMsg = "Length of number is zero" + numString = "" + buf, err = EncodeStringToBCD(numString) + if err == nil { + t.Fatalf("no error on encoding of an empty string") + } + if buf != nil { + t.Fatalf("buf is not nil on encoding of an empty string") + } + if err.Error() != referenceErrMsg { + t.Fatalf("wrong error message on encoding of an empty string") + } + + referenceErrMsg = "Failed to convert symbol 'a' to a digit" + numString = "0.1a" + buf, err = EncodeStringToBCD(numString) + if err == nil { + t.Fatalf("no error on encoding of a string number with non-digit symbol") + } + if buf != nil { + t.Fatalf("buf is not nil on encoding of a string number with non-digit symbol") + } + if err.Error() != referenceErrMsg { + t.Fatalf("wrong error message on encoding of a string number with non-digit symbol") + } +} + +func TestEncodeMaxNumber(t *testing.T) { + referenceErrMsg := "msgpack: decimal number is bigger than maximum supported number (10^38 - 1)" + decNum := decimal.New(1, DecimalPrecision) // // 10^DecimalPrecision + tuple := TupleDecimal{number: *NewDecNumber(decNum)} + _, err := msgpack.Marshal(&tuple) + if err == nil { + t.Fatalf("It is possible to encode a number unsupported by Tarantool") + } + if err.Error() != referenceErrMsg { + t.Fatalf("Incorrect error message on attempt to encode number unsupported by Tarantool") + } +} + +func TestEncodeMinNumber(t *testing.T) { + referenceErrMsg := "msgpack: decimal number is lesser than minimum supported number (-10^38 - 1)" + two := decimal.NewFromInt(2) + decNum := decimal.New(1, DecimalPrecision).Neg().Sub(two) // -10^DecimalPrecision - 2 + tuple := TupleDecimal{number: *NewDecNumber(decNum)} + _, err := msgpack.Marshal(&tuple) + if err == nil { + t.Fatalf("It is possible to encode a number unsupported by Tarantool") + } + if err.Error() != referenceErrMsg { + fmt.Println("Actual message: ", err.Error()) + fmt.Println("Expected message: ", referenceErrMsg) + t.Fatalf("Incorrect error message on attempt to encode number unsupported by Tarantool") + } +} + +func benchmarkMPEncodeDecode(b *testing.B, src decimal.Decimal, dst interface{}) { + b.ResetTimer() + + var v TupleDecimal + var buf []byte + var err error + for i := 0; i < b.N; i++ { + tuple := TupleDecimal{number: *NewDecNumber(src)} + if buf, err = msgpack.Marshal(&tuple); err != nil { + b.Fatal(err) + } + if err = msgpack.Unmarshal(buf, &v); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkMPEncodeDecodeDecimal(b *testing.B) { + for _, testcase := range benchmarkSamples { + b.Run(testcase.numString, func(b *testing.B) { + dec, err := decimal.NewFromString(testcase.numString) + if err != nil { + b.Fatal(err) + } + benchmarkMPEncodeDecode(b, dec, &dec) + }) + } +} + +func BenchmarkMPEncodeDecimal(b *testing.B) { + for _, testcase := range benchmarkSamples { + b.Run(testcase.numString, func(b *testing.B) { + decNum, err := NewDecNumberFromString(testcase.numString) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + msgpack.Marshal(decNum) + } + }) + } +} + +func BenchmarkMPDecodeDecimal(b *testing.B) { + for _, testcase := range benchmarkSamples { + b.Run(testcase.numString, func(b *testing.B) { + decNum, err := NewDecNumberFromString(testcase.numString) + if err != nil { + b.Fatal(err) + } + var buf []byte + if buf, err = msgpack.Marshal(decNum); err != nil { + b.Fatal(err) + } + b.ResetTimer() + var v TupleDecimal + for i := 0; i < b.N; i++ { + msgpack.Unmarshal(buf, &v) + } + + }) + } +} + +func connectWithValidation(t *testing.T) *Connection { + t.Helper() + + conn, err := Connect(server, opts) + if err != nil { + t.Fatalf("Failed to connect: %s", err.Error()) + } + if conn == nil { + t.Fatalf("conn is nil after Connect") + } + return conn +} + +func tupleValueIsDecimal(t *testing.T, tuples []interface{}, number decimal.Decimal) { + if len(tuples) != 1 { + t.Fatalf("Response Data len (%d) != 1", len(tuples)) + } + + if tpl, ok := tuples[0].([]interface{}); !ok { + t.Fatalf("Unexpected return value body") + } else { + if len(tpl) != 1 { + t.Fatalf("Unexpected return value body (tuple len)") + } + if val, ok := tpl[0].(DecNumber); !ok || !val.Equal(number) { + t.Fatalf("Unexpected return value body (tuple 0 field)") + } + } +} + +func trimMPHeader(mpBuf []byte, fixExt bool) []byte { + mpHeaderLen := 2 + if fixExt == false { + mpHeaderLen = 3 + } + return mpBuf[mpHeaderLen:] +} + +func TestEncodeStringToBCD(t *testing.T) { + samples := append(correctnessSamples, rawSamples...) + samples = append(samples, benchmarkSamples...) + for _, testcase := range samples { + t.Run(testcase.numString, func(t *testing.T) { + buf, err := EncodeStringToBCD(testcase.numString) + if err != nil { + t.Fatalf("Failed to encode decimal '%s' to BCD: %s", testcase.numString, err) + + } + b, _ := hex.DecodeString(testcase.mpBuf) + bcdBuf := trimMPHeader(b, testcase.fixExt) + if reflect.DeepEqual(buf, bcdBuf) != true { + t.Fatalf("Failed to encode decimal '%s' to BCD: expected '%x', actual '%x'", testcase.numString, bcdBuf, buf) + } + }) + } +} + +func TestDecodeStringFromBCD(t *testing.T) { + samples := append(correctnessSamples, rawSamples...) + samples = append(samples, benchmarkSamples...) + for _, testcase := range samples { + t.Run(testcase.numString, func(t *testing.T) { + b, _ := hex.DecodeString(testcase.mpBuf) + bcdBuf := trimMPHeader(b, testcase.fixExt) + s, err := DecodeStringFromBCD(bcdBuf) + if err != nil { + t.Fatalf("Failed to decode BCD '%x' to decimal: %s", bcdBuf, err) + } + + decActual, err := decimal.NewFromString(s) + if err != nil { + t.Fatalf("Failed to encode string ('%s') to decimal", s) + } + decExpected, err := decimal.NewFromString(testcase.numString) + if err != nil { + t.Fatalf("Failed to encode string ('%s') to decimal", testcase.numString) + } + if !decExpected.Equal(decActual) { + t.Fatalf("Decoded decimal from BCD ('%x') is incorrect: expected '%s', actual '%s'", bcdBuf, testcase.numString, s) + } + }) + } +} + +func TestMPEncode(t *testing.T) { + samples := append(correctnessSamples, decimalSamples...) + samples = append(samples, benchmarkSamples...) + for _, testcase := range samples { + t.Run(testcase.numString, func(t *testing.T) { + dec, err := NewDecNumberFromString(testcase.numString) + if err != nil { + t.Fatalf("NewDecNumberFromString() failed: %s", err.Error()) + } + buf, err := msgpack.Marshal(dec) + if err != nil { + t.Fatalf("Marshalling failed: %s", err.Error()) + } + refBuf, _ := hex.DecodeString(testcase.mpBuf) + if reflect.DeepEqual(buf, refBuf) != true { + t.Fatalf("Failed to encode decimal '%s', actual %x, expected %x", + testcase.numString, + buf, + refBuf) + } + }) + } +} + +func TestMPDecode(t *testing.T) { + samples := append(correctnessSamples, decimalSamples...) + samples = append(samples, benchmarkSamples...) + for _, testcase := range samples { + t.Run(testcase.numString, func(t *testing.T) { + mpBuf, err := hex.DecodeString(testcase.mpBuf) + if err != nil { + t.Fatalf("hex.DecodeString() failed: %s", err) + } + var v interface{} + err = msgpack.Unmarshal(mpBuf, &v) + if err != nil { + t.Fatalf("Unmarshalling failed: %s", err.Error()) + } + decActual := v.(DecNumber) + + decExpected, err := decimal.NewFromString(testcase.numString) + if err != nil { + t.Fatalf("decimal.NewFromString() failed: %s", err.Error()) + } + if !decExpected.Equal(decActual.Decimal) { + t.Fatalf("Decoded decimal ('%s') is incorrect", testcase.mpBuf) + } + }) + } +} + +func BenchmarkEncodeStringToBCD(b *testing.B) { + for _, testcase := range benchmarkSamples { + b.Run(testcase.numString, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + EncodeStringToBCD(testcase.numString) + } + }) + } +} + +func BenchmarkDecodeStringFromBCD(b *testing.B) { + for _, testcase := range benchmarkSamples { + b.Run(testcase.numString, func(b *testing.B) { + buf, _ := hex.DecodeString(testcase.mpBuf) + bcdBuf := trimMPHeader(buf, testcase.fixExt) + b.ResetTimer() + for n := 0; n < b.N; n++ { + DecodeStringFromBCD(bcdBuf) + } + }) + } +} + +func TestSelect(t *testing.T) { + skipIfDecimalUnsupported(t) + + conn := connectWithValidation(t) + defer conn.Close() + + number, err := decimal.NewFromString("-12.34") + if err != nil { + t.Fatalf("Failed to prepare test decimal: %s", err) + } + + resp, err := conn.Insert(space, []interface{}{NewDecNumber(number)}) + if err != nil { + t.Fatalf("Decimal insert failed: %s", err) + } + if resp == nil { + t.Fatalf("Response is nil after Replace") + } + tupleValueIsDecimal(t, resp.Data, number) + + var offset uint32 = 0 + var limit uint32 = 1 + resp, err = conn.Select(space, index, offset, limit, IterEq, []interface{}{NewDecNumber(number)}) + if err != nil { + t.Fatalf("Decimal select failed: %s", err.Error()) + } + if resp == nil { + t.Fatalf("Response is nil after Select") + } + tupleValueIsDecimal(t, resp.Data, number) + + resp, err = conn.Delete(space, index, []interface{}{NewDecNumber(number)}) + if err != nil { + t.Fatalf("Decimal delete failed: %s", err) + } + tupleValueIsDecimal(t, resp.Data, number) +} + +func assertInsert(t *testing.T, conn *Connection, numString string) { + number, err := decimal.NewFromString(numString) + if err != nil { + t.Fatalf("Failed to prepare test decimal: %s", err) + } + + resp, err := conn.Insert(space, []interface{}{NewDecNumber(number)}) + if err != nil { + t.Fatalf("Decimal insert failed: %s", err) + } + if resp == nil { + t.Fatalf("Response is nil after Replace") + } + tupleValueIsDecimal(t, resp.Data, number) + + resp, err = conn.Delete(space, index, []interface{}{NewDecNumber(number)}) + if err != nil { + t.Fatalf("Decimal delete failed: %s", err) + } + tupleValueIsDecimal(t, resp.Data, number) +} + +func TestInsert(t *testing.T) { + skipIfDecimalUnsupported(t) + + conn := connectWithValidation(t) + defer conn.Close() + + samples := append(correctnessSamples, benchmarkSamples...) + for _, testcase := range samples { + t.Run(testcase.numString, func(t *testing.T) { + assertInsert(t, conn, testcase.numString) + }) + } +} + +func TestReplace(t *testing.T) { + skipIfDecimalUnsupported(t) + + conn := connectWithValidation(t) + defer conn.Close() + + number, err := decimal.NewFromString("-12.34") + if err != nil { + t.Fatalf("Failed to prepare test decimal: %s", err) + } + + respRep, errRep := conn.Replace(space, []interface{}{NewDecNumber(number)}) + if errRep != nil { + t.Fatalf("Decimal replace failed: %s", errRep) + } + if respRep == nil { + t.Fatalf("Response is nil after Replace") + } + tupleValueIsDecimal(t, respRep.Data, number) + + respSel, errSel := conn.Select(space, index, 0, 1, IterEq, []interface{}{NewDecNumber(number)}) + if errSel != nil { + t.Fatalf("Decimal select failed: %s", errSel) + } + if respSel == nil { + t.Fatalf("Response is nil after Select") + } + tupleValueIsDecimal(t, respSel.Data, number) +} + +// runTestMain is a body of TestMain function +// (see https://pkg.go.dev/testing#hdr-Main). +// Using defer + os.Exit is not works so TestMain body +// is a separate function, see +// https://stackoverflow.com/questions/27629380/how-to-exit-a-go-program-honoring-deferred-calls +func runTestMain(m *testing.M) int { + isLess, err := test_helpers.IsTarantoolVersionLess(2, 2, 0) + if err != nil { + log.Fatalf("Failed to extract Tarantool version: %s", err) + } + + if isLess { + log.Println("Skipping decimal tests...") + isDecimalSupported = false + return m.Run() + } else { + isDecimalSupported = true + } + + instance, err := test_helpers.StartTarantool(test_helpers.StartOpts{ + InitScript: "config.lua", + Listen: server, + WorkDir: "work_dir", + User: opts.User, + Pass: opts.Pass, + WaitStart: 100 * time.Millisecond, + ConnectRetry: 3, + RetryTimeout: 500 * time.Millisecond, + }) + defer test_helpers.StopTarantoolWithCleanup(instance) + + if err != nil { + log.Fatalf("Failed to prepare test Tarantool: %s", err) + } + + return m.Run() +} + +func TestMain(m *testing.M) { + code := runTestMain(m) + os.Exit(code) +} diff --git a/decimal/example_test.go b/decimal/example_test.go new file mode 100644 index 000000000..f509b2492 --- /dev/null +++ b/decimal/example_test.go @@ -0,0 +1,55 @@ +// Run Tarantool instance before example execution: +// +// Terminal 1: +// $ cd decimal +// $ TEST_TNT_LISTEN=3013 TEST_TNT_WORK_DIR=$(mktemp -d -t 'tarantool.XXX') tarantool config.lua +// +// Terminal 2: +// $ go test -v example_test.go +package decimal_test + +import ( + "log" + "time" + + "github.com/tarantool/go-tarantool" + . "github.com/tarantool/go-tarantool/decimal" +) + +// To enable support of decimal in msgpack with +// https://github.com/shopspring/decimal, +// import tarantool/decimal submodule. +func Example() { + server := "127.0.0.1:3013" + opts := tarantool.Opts{ + Timeout: 500 * time.Millisecond, + Reconnect: 1 * time.Second, + MaxReconnects: 3, + User: "test", + Pass: "test", + } + client, err := tarantool.Connect(server, opts) + if err != nil { + log.Fatalf("Failed to connect: %s", err.Error()) + } + + spaceNo := uint32(524) + + number, err := NewDecNumberFromString("-22.804") + if err != nil { + log.Fatalf("Failed to prepare test decimal: %s", err) + } + + resp, err := client.Replace(spaceNo, []interface{}{number}) + if err != nil { + log.Fatalf("Decimal replace failed: %s", err) + } + if resp == nil { + log.Fatalf("Response is nil after Replace") + } + + log.Println("Decimal tuple replace") + log.Println("Error", err) + log.Println("Code", resp.Code) + log.Println("Data", resp.Data) +} diff --git a/decimal/export_test.go b/decimal/export_test.go new file mode 100644 index 000000000..c43a812c6 --- /dev/null +++ b/decimal/export_test.go @@ -0,0 +1,17 @@ +package decimal + +func EncodeStringToBCD(buf string) ([]byte, error) { + return encodeStringToBCD(buf) +} + +func DecodeStringFromBCD(bcdBuf []byte) (string, error) { + return decodeStringFromBCD(bcdBuf) +} + +func GetNumberLength(buf string) int { + return getNumberLength(buf) +} + +const ( + DecimalPrecision = decimalPrecision +) diff --git a/go.mod b/go.mod index 6dcaee974..306725105 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,15 @@ module github.com/tarantool/go-tarantool go 1.11 require ( + github.com/google/go-cmp v0.5.7 // indirect github.com/google/uuid v1.3.0 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/shopspring/decimal v1.3.1 github.com/stretchr/testify v1.7.1 // indirect github.com/tarantool/go-openssl v0.0.8-0.20220419150948-be4921aa2f87 google.golang.org/appengine v1.6.7 // indirect gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/vmihailenco/msgpack.v2 v2.9.2 + gotest.tools/v3 v3.2.0 // indirect ) diff --git a/go.sum b/go.sum index 1af7f9933..f9ffc6b0a 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,9 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -11,24 +14,48 @@ github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tarantool/go-openssl v0.0.8-0.20220419150948-be4921aa2f87 h1:JGzuBxNBq5saVtPUcuu5Y4+kbJON6H02//OT+RNqGts= github.com/tarantool/go-openssl v0.0.8-0.20220419150948-be4921aa2f87/go.mod h1:M7H4xYSbzqpW/ZRBMyH0eyqQBsnhAMfsYk5mv0yid7A= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -38,3 +65,5 @@ gopkg.in/vmihailenco/msgpack.v2 v2.9.2 h1:gjPqo9orRVlSAH/065qw3MsFCDpH7fa1KpiizX gopkg.in/vmihailenco/msgpack.v2 v2.9.2/go.mod h1:/3Dn1Npt9+MYyLpYYXjInO/5jvMLamn+AEGwNEOatn8= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.2.0 h1:I0DwBVMGAx26dttAj1BtJLAkVGncrkkUXfJLC4Flt/I= +gotest.tools/v3 v3.2.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A= From b699fd91264d887b4659dfd54ba54b778bcc7438 Mon Sep 17 00:00:00 2001 From: Sergey Bronnikov Date: Thu, 28 Apr 2022 11:52:25 +0300 Subject: [PATCH 2/2] decimal: add fuzzing test Fuzzing tests in Golang, see [1] and [2], requires Go 1.18+. However in CI we use Go 1.13 that fails on running fuzzing tests. To avoid this fuzzing test has been moved to a separate file an marked with build tag. 1. https://go.dev/doc/tutorial/fuzz 2. https://go.dev/doc/fuzz/ Closes #96 Co-authored-by: Oleg Jukovec --- .github/workflows/testing.yml | 24 ++++++++++++++++-- CONTRIBUTING.md | 5 ++++ Makefile | 6 +++++ README.md | 9 +++++-- decimal/fuzzing_test.go | 46 +++++++++++++++++++++++++++++++++++ 5 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 decimal/fuzzing_test.go diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 5c3c84eae..09d6528a4 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -26,12 +26,23 @@ jobs: strategy: fail-fast: false matrix: + golang: + - 1.13 tarantool: - '1.10' - '2.8' - '2.9' - '2.x-latest' coveralls: [false] + fuzzing: [false] + include: + - tarantool: '2.x-latest' + coveralls: true + golang: 1.13 + - tarantool: '2.x-latest' + fuzzing: true + golang: 1.18 + coveralls: false steps: - name: Clone the connector @@ -52,17 +63,21 @@ jobs: - name: Setup golang for the connector and tests uses: actions/setup-go@v2 with: - go-version: 1.13 + go-version: ${{ matrix.golang }} - name: Install test dependencies run: make deps - - name: Run tests + - name: Run regression tests run: make test - name: Run tests with call_17 run: make test TAGS="go_tarantool_call_17" + - name: Run fuzzing tests + if: ${{ matrix.fuzzing }} + run: make fuzzing TAGS="go_tarantool_decimal_fuzzing" + - name: Run tests, collect code coverage data and send to Coveralls if: ${{ matrix.coveralls }} env: @@ -96,6 +111,7 @@ jobs: - '1.10.11-0-gf0b0e7ecf-r470' - '2.8.3-21-g7d35cd2be-r470' coveralls: [false] + fuzzing: [false] ssl: [false] include: - sdk-version: '2.10.0-1-gfa775b383-r486-linux-x86_64' @@ -144,6 +160,10 @@ jobs: env: TEST_TNT_SSL: ${{matrix.ssl}} + - name: Run fuzzing tests + if: ${{ matrix.fuzzing }} + run: make fuzzing TAGS="go_tarantool_decimal_fuzzing" + - name: Run tests, collect code coverage data and send to Coveralls if: ${{ matrix.coveralls }} env: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bd9075767..e8d493e40 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,6 +45,11 @@ For example, for running tests in `multi`, `uuid` and `main` packages, call make test-multi test-uuid test-main ``` +To run [fuzz tests](https://go.dev/doc/tutorial/fuzz) for the main package and each subpackage: +```bash +make TAGS="go_tarantool_decimal_fuzzing" fuzzing +``` + To check if the current changes will pass the linter in CI, install golangci-lint from [sources](https://golangci-lint.run/usage/install/) and run it with next command: diff --git a/Makefile b/Makefile index c9daf85ae..1c6e0ce18 100644 --- a/Makefile +++ b/Makefile @@ -118,3 +118,9 @@ bench-diff: ${BENCH_FILES} @echo "Comparing performance between master and the current branch" @echo "'old' is a version in master branch, 'new' is a version in a current branch" benchstat ${BENCH_FILES} | grep -v pkg: + +.PHONY: fuzzing +fuzzing: + @echo "Running fuzzing tests" + go clean -testcache + go test -tags "$(TAGS)" ./... -run=^Fuzz -v -p 1 diff --git a/README.md b/README.md index 7ddf956bf..0d992a344 100644 --- a/README.md +++ b/README.md @@ -62,12 +62,17 @@ This allows us to introduce new features without losing backward compatibility. ``` go_tarantool_ssl_disable ``` -2. to change the default `Call` behavior from `Call16` to `Call17`, you can use the build - tag: +2. To change the default `Call` behavior from `Call16` to `Call17`, you can use + the build tag: ``` go_tarantool_call_17 ``` **Note:** In future releases, `Call17` may be used as default `Call` behavior. +3. To run fuzz tests with decimals, you can use the build tag: + ``` + go_tarantool_decimal_fuzzing + ``` + **Note:** It crashes old Tarantool versions and requires Go 1.18+. ## Documentation diff --git a/decimal/fuzzing_test.go b/decimal/fuzzing_test.go new file mode 100644 index 000000000..c69a68719 --- /dev/null +++ b/decimal/fuzzing_test.go @@ -0,0 +1,46 @@ +//go:build go_tarantool_decimal_fuzzing +// +build go_tarantool_decimal_fuzzing + +package decimal_test + +import ( + "testing" + + "github.com/shopspring/decimal" + . "github.com/tarantool/go-tarantool/decimal" +) + +func strToDecimal(t *testing.T, buf string) decimal.Decimal { + decNum, err := decimal.NewFromString(buf) + if err != nil { + t.Fatal(err) + } + return decNum +} + +func FuzzEncodeDecodeBCD(f *testing.F) { + samples := append(correctnessSamples, benchmarkSamples...) + for _, testcase := range samples { + if len(testcase.numString) > 0 { + f.Add(testcase.numString) // Use f.Add to provide a seed corpus. + } + } + f.Fuzz(func(t *testing.T, orig string) { + if l := GetNumberLength(orig); l > DecimalPrecision { + t.Skip("max number length is exceeded") + } + bcdBuf, err := EncodeStringToBCD(orig) + if err != nil { + t.Skip("Only correct requests are interesting: %w", err) + } + var dec string + dec, err = DecodeStringFromBCD(bcdBuf) + if err != nil { + t.Fatalf("Failed to decode encoded value ('%s')", orig) + } + + if !strToDecimal(t, dec).Equal(strToDecimal(t, orig)) { + t.Fatal("Decimal numbers are not equal") + } + }) +}