Merge pull request #1634 Prevented create decoder instance until start read a message from topics

Author: rekby

CHANGELOG.md

@@ -1,3 +1,5 @@
+* Prevented create decoder instance until start read a message from topics
+
 ## v3.99.4
 * Fixed bug with wrong context on session closing
 * Fixed goroutine leak on closing `database/sql` driver

internal/topic/topicreadercommon/message.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"time"
 
 	"github.com/ydb-platform/ydb-go-sdk/v3/internal/empty"
@@ -84,14 +85,18 @@ type PublicMessageContentUnmarshaler interface {
 }
 
 func createReader(decoders DecoderMap, codec rawtopiccommon.Codec, rawBytes []byte) oneTimeReader {
-	reader, err := decoders.Decode(codec, bytes.NewReader(rawBytes))
-	if err != nil {
-		reader = errorReader{
-			err: fmt.Errorf("failed to decode message with codec '%v': %w", codec, err),
+	var maker readerMaker = func() io.Reader {
+		reader, err := decoders.Decode(codec, bytes.NewReader(rawBytes))
+		if err != nil {
+			reader = errorReader{
+				err: fmt.Errorf("failed to decode message with codec '%v': %w", codec, err),
+			}
 		}
+
+		return reader
 	}
 
-	return newOneTimeReader(reader)
+	return newOneTimeReader(maker)
 }
 
 type errorReader struct {

internal/topic/topicreadercommon/message_content_pool_test.go

@@ -16,11 +16,11 @@ func BenchmarkConsumeContent(b *testing.B) {
 	b.ReportAllocs()
 	content := []byte("asd")
 	reader := bytes.NewReader(content)
-	msg := PublicMessage{data: newOneTimeReader(reader)}
+	msg := PublicMessage{data: newOneTimeReaderFromReader(reader)}
 	for i := 0; i < b.N; i++ {
 		reader.Reset(content)
 		msg.dataConsumed = false
-		msg.data = newOneTimeReader(reader)
+		msg.data = newOneTimeReaderFromReader(reader)
 		err := msg.UnmarshalTo(emptyConsumer{})
 		if err != nil {
 			b.Fatal()

internal/topic/topicreadercommon/one_time_reader.go

@@ -7,19 +7,35 @@ import (
 type oneTimeReader struct {
 	err    error
 	reader io.Reader
+
+	// prevent early create decoder, because it can consume a lot of memory
+	// https://github.com/ydb-platform/ydb-go-sdk/issues/1341
+	readerMaker readerMaker
 }
 
-func newOneTimeReader(reader io.Reader) oneTimeReader {
+type readerMaker func() io.Reader
+
+func newOneTimeReader(readerMaker readerMaker) oneTimeReader {
 	return oneTimeReader{
-		reader: reader,
+		readerMaker: readerMaker,
 	}
 }
 
+func newOneTimeReaderFromReader(reader io.Reader) oneTimeReader {
+	maker := func() io.Reader { return reader }
+
+	return newOneTimeReader(maker)
+}
+
 func (s *oneTimeReader) Read(p []byte) (n int, err error) {
 	if s.err != nil {
 		return 0, s.err
 	}
 
+	if s.reader == nil {
+		s.reader = s.readerMaker()
+	}
+
 	n, err = s.reader.Read(p)
 	if err != nil {
 		s.err = err

internal/topic/topicreadercommon/one_time_reader_test.go

@@ -12,7 +12,7 @@ import (
 
 func TestOneTimeReader(t *testing.T) {
 	t.Run("FullRead", func(t *testing.T) {
-		r := newOneTimeReader(bytes.NewReader([]byte{1, 2, 3}))
+		r := newOneTimeReaderFromReader(bytes.NewReader([]byte{1, 2, 3}))
 		dstBuf := make([]byte, 3)
 		n, err := r.Read(dstBuf)
 		require.NoError(t, err)
@@ -24,7 +24,7 @@ func TestOneTimeReader(t *testing.T) {
 		require.Equal(t, io.EOF, r.err)
 	})
 	t.Run("DstMoreThenContent", func(t *testing.T) {
-		r := newOneTimeReader(bytes.NewReader([]byte{1, 2, 3}))
+		r := newOneTimeReaderFromReader(bytes.NewReader([]byte{1, 2, 3}))
 		dstBuf := make([]byte, 4)
 		n, err := r.Read(dstBuf)
 		require.NoError(t, err)
@@ -37,7 +37,7 @@ func TestOneTimeReader(t *testing.T) {
 		require.Equal(t, io.EOF, r.err)
 	})
 	t.Run("ReadLess", func(t *testing.T) {
-		r := newOneTimeReader(bytes.NewReader([]byte{1, 2, 3}))
+		r := newOneTimeReaderFromReader(bytes.NewReader([]byte{1, 2, 3}))
 		dstBuf := make([]byte, 2)
 		n, err := r.Read(dstBuf)
 		require.NoError(t, err)
@@ -55,7 +55,7 @@ func TestOneTimeReader(t *testing.T) {
 		require.Equal(t, 0, n)
 	})
 	t.Run("InnerErr", func(t *testing.T) {
-		r := newOneTimeReader(nil)
+		r := newOneTimeReaderFromReader(nil)
 
 		bufSize := 2
 		preparedData := make([]byte, 2*bufSize)