Skip to content

Commit 4d33de9

Browse files
fjlfjl
authored
rlp: optimize big.Int decoding for size <= 32 bytes (#22927)
This change grows the static integer buffer in Stream to 32 bytes, making it possible to decode 256bit integers without allocating a temporary buffer. In the recent commit 088da24, Stream struct size decreased from 120 bytes down to 88 bytes. This commit grows the struct to 112 bytes again, but the size change will not degrade performance because Stream instances are internally cached in sync.Pool. name old time/op new time/op delta DecodeBigInts-8 12.2µs ± 0% 8.6µs ± 4% -29.58% (p=0.000 n=9+10) name old speed new speed delta DecodeBigInts-8 230MB/s ± 0% 326MB/s ± 4% +42.04% (p=0.000 n=9+10)
1 parent 017cf71 commit 4d33de9

File tree

3 files changed

+74
-16
lines changed

3 files changed

+74
-16
lines changed

‎rlp/decode.go‎

Lines changed: 47 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -220,20 +220,51 @@ func decodeBigIntNoPtr(s *Stream, val reflect.Value) error {
220220
}
221221

222222
func decodeBigInt(s *Stream, val reflect.Value) error {
223-
b, err := s.Bytes()
224-
if err != nil {
223+
var buffer []byte
224+
kind, size, err := s.Kind()
225+
switch {
226+
case err != nil:
225227
return wrapStreamError(err, val.Type())
228+
case kind == List:
229+
return wrapStreamError(ErrExpectedString, val.Type())
230+
case kind == Byte:
231+
buffer = s.uintbuf[:1]
232+
buffer[0] = s.byteval
233+
s.kind = -1 // re-arm Kind
234+
case size == 0:
235+
// Avoid zero-length read.
236+
s.kind = -1
237+
case size <= uint64(len(s.uintbuf)):
238+
// For integers smaller than s.uintbuf, allocating a buffer
239+
// can be avoided.
240+
buffer = s.uintbuf[:size]
241+
if err := s.readFull(buffer); err != nil {
242+
return wrapStreamError(err, val.Type())
243+
}
244+
// Reject inputs where single byte encoding should have been used.
245+
if size == 1 && buffer[0] < 128 {
246+
return wrapStreamError(ErrCanonSize, val.Type())
247+
}
248+
default:
249+
// For large integers, a temporary buffer is needed.
250+
buffer = make([]byte, size)
251+
if err := s.readFull(buffer); err != nil {
252+
return wrapStreamError(err, val.Type())
253+
}
254+
}
255+
256+
// Reject leading zero bytes.
257+
if len(buffer) > 0 && buffer[0] == 0 {
258+
return wrapStreamError(ErrCanonInt, val.Type())
226259
}
260+
261+
// Set the integer bytes.
227262
i := val.Interface().(*big.Int)
228263
if i == nil {
229264
i = new(big.Int)
230265
val.Set(reflect.ValueOf(i))
231266
}
232-
// Reject leading zero bytes.
233-
if len(b) > 0 && b[0] == 0 {
234-
return wrapStreamError(ErrCanonInt, val.Type())
235-
}
236-
i.SetBytes(b)
267+
i.SetBytes(buffer)
237268
return nil
238269
}
239270

@@ -563,7 +594,7 @@ type Stream struct {
563594
size uint64 // size of value ahead
564595
kinderr error // error from last readKind
565596
stack []uint64 // list sizes
566-
uintbuf [8]byte // auxiliary buffer for integer decoding
597+
uintbuf [32]byte // auxiliary buffer for integer decoding
567598
kind Kind // kind of value ahead
568599
byteval byte // value of single byte in type tag
569600
limited bool // true if input limit is in effect
@@ -817,7 +848,7 @@ func (s *Stream) Reset(r io.Reader, inputLimit uint64) {
817848
s.kind = -1
818849
s.kinderr = nil
819850
s.byteval = 0
820-
s.uintbuf = [8]byte{}
851+
s.uintbuf = [32]byte{}
821852
}
822853

823854
// Kind returns the kind and size of the next value in the
@@ -927,17 +958,20 @@ func (s *Stream) readUint(size byte) (uint64, error) {
927958
b, err := s.readByte()
928959
return uint64(b), err
929960
default:
961+
buffer := s.uintbuf[:8]
962+
for i := range buffer {
963+
buffer[i] = 0
964+
}
930965
start := int(8 - size)
931-
s.uintbuf = [8]byte{}
932-
if err := s.readFull(s.uintbuf[start:]); err != nil {
966+
if err := s.readFull(buffer[start:]); err != nil {
933967
return 0, err
934968
}
935-
if s.uintbuf[start] == 0 {
969+
if buffer[start] == 0 {
936970
// Note: readUint is also used to decode integer values.
937971
// The error needs to be adjusted to become ErrCanonInt in this case.
938972
return 0, ErrCanonSize
939973
}
940-
return binary.BigEndian.Uint64(s.uintbuf[:]), nil
974+
return binary.BigEndian.Uint64(buffer[:]), nil
941975
}
942976
}
943977

‎rlp/decode_test.go‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,11 @@ type recstruct struct {
329329
Child *recstruct `rlp:"nil"`
330330
}
331331

332+
type bigIntStruct struct {
333+
I *big.Int
334+
B string
335+
}
336+
332337
type invalidNilTag struct {
333338
X []byte `rlp:"nil"`
334339
}
@@ -405,10 +410,11 @@ type ignoredField struct {
405410
}
406411

407412
var (
408-
veryBigInt = big.NewInt(0).Add(
413+
veryBigInt = new(big.Int).Add(
409414
big.NewInt(0).Lsh(big.NewInt(0xFFFFFFFFFFFFFF), 16),
410415
big.NewInt(0xFFFF),
411416
)
417+
veryVeryBigInt = new(big.Int).Exp(veryBigInt, big.NewInt(8), nil)
412418
)
413419

414420
var decodeTests = []decodeTest{
@@ -479,12 +485,15 @@ var decodeTests = []decodeTest{
479485
{input: "C0", ptr: new(string), error: "rlp: expected input string or byte for string"},
480486

481487
// big ints
488+
{input: "80", ptr: new(*big.Int), value: big.NewInt(0)},
482489
{input: "01", ptr: new(*big.Int), value: big.NewInt(1)},
483490
{input: "89FFFFFFFFFFFFFFFFFF", ptr: new(*big.Int), value: veryBigInt},
491+
{input: "B848FFFFFFFFFFFFFFFFF800000000000000001BFFFFFFFFFFFFFFFFC8000000000000000045FFFFFFFFFFFFFFFFC800000000000000001BFFFFFFFFFFFFFFFFF8000000000000000001", ptr: new(*big.Int), value: veryVeryBigInt},
484492
{input: "10", ptr: new(big.Int), value: *big.NewInt(16)}, // non-pointer also works
485493
{input: "C0", ptr: new(*big.Int), error: "rlp: expected input string or byte for *big.Int"},
486-
{input: "820001", ptr: new(big.Int), error: "rlp: non-canonical integer (leading zero bytes) for *big.Int"},
487-
{input: "8105", ptr: new(big.Int), error: "rlp: non-canonical size information for *big.Int"},
494+
{input: "00", ptr: new(*big.Int), error: "rlp: non-canonical integer (leading zero bytes) for *big.Int"},
495+
{input: "820001", ptr: new(*big.Int), error: "rlp: non-canonical integer (leading zero bytes) for *big.Int"},
496+
{input: "8105", ptr: new(*big.Int), error: "rlp: non-canonical size information for *big.Int"},
488497

489498
// structs
490499
{
@@ -497,6 +506,13 @@ var decodeTests = []decodeTest{
497506
ptr: new(recstruct),
498507
value: recstruct{1, &recstruct{2, &recstruct{3, nil}}},
499508
},
509+
{
510+
// This checks that empty big.Int works correctly in struct context. It's easy to
511+
// miss the update of s.kind for this case, so it needs its own test.
512+
input: "C58083343434",
513+
ptr: new(bigIntStruct),
514+
value: bigIntStruct{new(big.Int), "444"},
515+
},
500516

501517
// struct errors
502518
{

‎rlp/encode_test.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,14 @@ var encTests = []encTest{
131131
val: big.NewInt(0).SetBytes(unhex("010000000000000000000000000000000000000000000000000000000000000000")),
132132
output: "A1010000000000000000000000000000000000000000000000000000000000000000",
133133
},
134+
{
135+
val: veryBigInt,
136+
output: "89FFFFFFFFFFFFFFFFFF",
137+
},
138+
{
139+
val: veryVeryBigInt,
140+
output: "B848FFFFFFFFFFFFFFFFF800000000000000001BFFFFFFFFFFFFFFFFC8000000000000000045FFFFFFFFFFFFFFFFC800000000000000001BFFFFFFFFFFFFFFFFF8000000000000000001",
141+
},
134142

135143
// non-pointer big.Int
136144
{val: *big.NewInt(0), output: "80"},

0 commit comments

Comments
 (0)