encoding/json: reduce the number of allocations when decoding in streaming mode (Token API)

When the scanner is used by the Token API it always resets the state before so that the scanner behaves as if it was parsing a top-level value,
which causes it to allocate and set the 'err' field because the following character is not a space. This error value is completely unnecessary
because it's dropped by the next invocation of readValue().

Fixes #56299

Author: dop251

src/encoding/json/scanner.go

@@ -73,6 +73,9 @@ type scanner struct {
 	// Reached end of top-level value.
 	endTop bool
 
+	// The scanner is used in streaming mode (i.e. by the Token API)
+	inStream bool
+
 	// Stack of what we're in the middle of - array values, object keys, object values.
 	parseState []int
 
@@ -280,6 +283,15 @@ func stateEndValue(s *scanner, c byte) int {
 	n := len(s.parseState)
 	if n == 0 {
 		// Completed top-level before the current byte.
+		if s.inStream {
+			// If used in streaming mode (i.e. by the Token API) do not allocate and set s.err in case the next
+			// character is non-space (which stateEndTop() would do). The error would be discarded anyway at the next
+			// call to readValue().
+			if s.err != nil {
+				return scanError
+			}
+			return scanEnd
+		}
 		s.step = stateEndTop
 		s.endTop = true
 		return stateEndTop(s, c)

src/encoding/json/stream.go

@@ -369,6 +369,11 @@ func (d Delim) String() string {
 // to mark the start and end of arrays and objects.
 // Commas and colons are elided.
 func (dec *Decoder) Token() (Token, error) {
+	dec.scan.inStream = true
+	defer func() {
+		dec.scan.inStream = false
+	}()
+
 	for {
 		c, err := dec.peek()
 		if err != nil {