fix memory usage

Author: wxiaoguang

modules/git/blob.go

@@ -30,10 +30,8 @@ func (b *Blob) GetBlobContent(limit int64) (string, error) {
 		return "", err
 	}
 	defer dataRc.Close()
-	buf := make([]byte, limit)
-	n, _ := util.ReadAtMost(dataRc, buf)
-	buf = buf[:n]
-	return string(buf), nil
+	buf, err := util.ReadWithLimit(dataRc, int(limit))
+	return string(buf), err
 }
 
 // GetBlobLineCount gets line count of the blob

modules/util/io.go

@@ -4,13 +4,14 @@
 package util
 
 import (
+	"bytes"
 	"errors"
 	"io"
 )
 
 // ReadAtMost reads at most len(buf) bytes from r into buf.
 // It returns the number of bytes copied. n is only less than len(buf) if r provides fewer bytes.
-// If EOF occurs while reading, err will be nil.
+// If EOF or ErrUnexpectedEOF occurs while reading, err will be nil.
 func ReadAtMost(r io.Reader, buf []byte) (n int, err error) {
 	n, err = io.ReadFull(r, buf)
 	if err == io.EOF || err == io.ErrUnexpectedEOF {
@@ -19,6 +20,42 @@ func ReadAtMost(r io.Reader, buf []byte) (n int, err error) {
 	return n, err
 }
 
+// ReadWithLimit reads at most "limit" bytes from r into buf.
+// If EOF or ErrUnexpectedEOF occurs while reading, err will be nil.
+func ReadWithLimit(r io.Reader, n int) (buf []byte, err error) {
+	return readWithLimit(r, 1024, n)
+}
+
+func readWithLimit(r io.Reader, batch, limit int) ([]byte, error) {
+	if limit <= batch {
+		buf := make([]byte, limit)
+		n, err := ReadAtMost(r, buf)
+		if err != nil {
+			return nil, err
+		}
+		return buf[:n], nil
+	}
+	res := bytes.NewBuffer(make([]byte, 0, batch))
+	bufFix := make([]byte, batch)
+	eof := false
+	for res.Len() < limit && !eof {
+		bufTmp := bufFix
+		if res.Len()+batch > limit {
+			bufTmp = bufFix[:limit-res.Len()]
+		}
+		n, err := io.ReadFull(r, bufTmp)
+		if err == io.EOF || err == io.ErrUnexpectedEOF {
+			eof = true
+		} else if err != nil {
+			return nil, err
+		}
+		if _, err = res.Write(bufTmp[:n]); err != nil {
+			return nil, err
+		}
+	}
+	return res.Bytes(), nil
+}
+
 // ErrNotEmpty is an error reported when there is a non-empty reader
 var ErrNotEmpty = errors.New("not-empty")
 

modules/util/io_test.go

@@ -0,0 +1,66 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package util
+
+import (
+	"bytes"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type readerWithError struct {
+	buf *bytes.Buffer
+}
+
+func (r *readerWithError) Read(p []byte) (n int, err error) {
+	if r.buf.Len() < 2 {
+		return 0, errors.New("test error")
+	}
+	return r.buf.Read(p)
+}
+
+func TestReadWithLimit(t *testing.T) {
+	bs := []byte("0123456789abcdef")
+
+	// normal test
+	buf, err := readWithLimit(bytes.NewBuffer(bs), 5, 2)
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("01"), buf)
+
+	buf, err = readWithLimit(bytes.NewBuffer(bs), 5, 5)
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("01234"), buf)
+
+	buf, err = readWithLimit(bytes.NewBuffer(bs), 5, 6)
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("012345"), buf)
+
+	buf, err = readWithLimit(bytes.NewBuffer(bs), 5, len(bs))
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("0123456789abcdef"), buf)
+
+	buf, err = readWithLimit(bytes.NewBuffer(bs), 5, 100)
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("0123456789abcdef"), buf)
+
+	// test with error
+	buf, err = readWithLimit(&readerWithError{bytes.NewBuffer(bs)}, 5, 10)
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("0123456789"), buf)
+
+	buf, err = readWithLimit(&readerWithError{bytes.NewBuffer(bs)}, 5, 100)
+	assert.ErrorContains(t, err, "test error")
+	assert.Empty(t, buf)
+
+	// test public function
+	buf, err = ReadWithLimit(bytes.NewBuffer(bs), 2)
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("01"), buf)
+
+	buf, err = ReadWithLimit(bytes.NewBuffer(bs), 9999999)
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("0123456789abcdef"), buf)
+}