Create payload->blob type system

api/clients/codecs/blob_codec.go

@@ -4,22 +4,24 @@ import (
 	"fmt"
 )
 
-type BlobEncodingVersion byte
+type PayloadEncodingVersion uint8
 
 const (
-	// This minimal blob encoding contains a 32 byte header = [0x00, version byte, uint32 len of data, 0x00, 0x00,...]
+	// PayloadEncodingVersion0 entails a 32 byte header = [0x00, version byte, big-endian uint32 len of payload, 0x00, 0x00,...]
 	// followed by the encoded data [0x00, 31 bytes of data, 0x00, 31 bytes of data,...]
-	DefaultBlobEncoding BlobEncodingVersion = 0x0
+	//
+	// Each group of 32 bytes starts with a 0x00 byte so that they can be parsed as valid bn254 field elements.
+	PayloadEncodingVersion0 PayloadEncodingVersion = 0x0
 )
 
 type BlobCodec interface {
 	DecodeBlob(encodedData []byte) ([]byte, error)
 	EncodeBlob(rawData []byte) ([]byte, error)
 }
 
-func BlobEncodingVersionToCodec(version BlobEncodingVersion) (BlobCodec, error) {
+func BlobEncodingVersionToCodec(version PayloadEncodingVersion) (BlobCodec, error) {
 	switch version {
-	case DefaultBlobEncoding:
+	case PayloadEncodingVersion0:
 		return DefaultBlobCodec{}, nil
 	default:
 		return nil, fmt.Errorf("unsupported blob encoding version: %x", version)
@@ -33,7 +35,7 @@ func GenericDecodeBlob(data []byte) ([]byte, error) {
 	// version byte is stored in [1], because [0] is always 0 to ensure the codecBlobHeader is a valid bn254 element
 	// see https://github.com/Layr-Labs/eigenda/blob/master/api/clients/codecs/default_blob_codec.go#L21
 	// TODO: we should prob be working over a struct with methods such as GetBlobEncodingVersion() to prevent index errors
-	version := BlobEncodingVersion(data[1])
+	version := PayloadEncodingVersion(data[1])
 	codec, err := BlobEncodingVersionToCodec(version)
 	if err != nil {
 		return nil, err
@@ -49,7 +51,7 @@ func GenericDecodeBlob(data []byte) ([]byte, error) {
 
 // CreateCodec creates a new BlobCodec based on the defined polynomial form of payloads, and the desired
 // BlobEncodingVersion
-func CreateCodec(payloadPolynomialForm PolynomialForm, version BlobEncodingVersion) (BlobCodec, error) {
+func CreateCodec(payloadPolynomialForm PolynomialForm, version PayloadEncodingVersion) (BlobCodec, error) {
 	lowLevelCodec, err := BlobEncodingVersionToCodec(version)
 	if err != nil {
 		return nil, fmt.Errorf("create low level codec: %w", err)

api/clients/codecs/default_blob_codec.go

@@ -22,7 +22,7 @@ func (v DefaultBlobCodec) EncodeBlob(rawData []byte) ([]byte, error) {
 	codecBlobHeader := make([]byte, 32)
 	// first byte is always 0 to ensure the codecBlobHeader is a valid bn254 element
 	// encode version byte
-	codecBlobHeader[1] = byte(DefaultBlobEncoding)
+	codecBlobHeader[1] = byte(PayloadEncodingVersion0)
 
 	// encode length as uint32
 	binary.BigEndian.PutUint32(codecBlobHeader[2:6], uint32(len(rawData))) // uint32 should be more than enough to store the length (approx 4gb)

api/clients/config.go

@@ -64,7 +64,7 @@ type EigenDAClientConfig struct {
 	DisableTLS bool
 
 	// The blob encoding version to use when writing blobs from the high level interface.
-	PutBlobEncodingVersion codecs.BlobEncodingVersion
+	PutBlobEncodingVersion codecs.PayloadEncodingVersion
 
 	// Point verification mode does an IFFT on data before it is written, and does an FFT on data after it is read.
 	// This makes it possible to open points on the KZG commitment to prove that the field elements correspond to

api/clients/eigenda_client_test.go

@@ -65,7 +65,7 @@ func TestPutRetrieveBlobIFFTSuccess(t *testing.T) {
 			CustomQuorumIDs:              []uint{},
 			SignerPrivateKeyHex:          "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
 			DisableTLS:                   false,
-			PutBlobEncodingVersion:       codecs.DefaultBlobEncoding,
+			PutBlobEncodingVersion:       codecs.PayloadEncodingVersion0,
 			DisablePointVerificationMode: false,
 			WaitForFinalization:          true,
 		},
@@ -132,7 +132,7 @@ func TestPutRetrieveBlobIFFTNoDecodeSuccess(t *testing.T) {
 			CustomQuorumIDs:              []uint{},
 			SignerPrivateKeyHex:          "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
 			DisableTLS:                   false,
-			PutBlobEncodingVersion:       codecs.DefaultBlobEncoding,
+			PutBlobEncodingVersion:       codecs.PayloadEncodingVersion0,
 			DisablePointVerificationMode: false,
 			WaitForFinalization:          true,
 		},
@@ -202,7 +202,7 @@ func TestPutRetrieveBlobNoIFFTSuccess(t *testing.T) {
 			CustomQuorumIDs:              []uint{},
 			SignerPrivateKeyHex:          "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
 			DisableTLS:                   false,
-			PutBlobEncodingVersion:       codecs.DefaultBlobEncoding,
+			PutBlobEncodingVersion:       codecs.PayloadEncodingVersion0,
 			DisablePointVerificationMode: true,
 			WaitForFinalization:          true,
 		},
@@ -234,7 +234,7 @@ func TestPutBlobFailDispersal(t *testing.T) {
 			CustomQuorumIDs:          []uint{},
 			SignerPrivateKeyHex:      "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
 			DisableTLS:               false,
-			PutBlobEncodingVersion:   codecs.DefaultBlobEncoding,
+			PutBlobEncodingVersion:   codecs.PayloadEncodingVersion0,
 			WaitForFinalization:      true,
 		},
 		Client: disperserClient,
@@ -266,7 +266,7 @@ func TestPutBlobFailureInsufficentSignatures(t *testing.T) {
 			CustomQuorumIDs:          []uint{},
 			SignerPrivateKeyHex:      "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
 			DisableTLS:               false,
-			PutBlobEncodingVersion:   codecs.DefaultBlobEncoding,
+			PutBlobEncodingVersion:   codecs.PayloadEncodingVersion0,
 			WaitForFinalization:      true,
 		},
 		Client: disperserClient,
@@ -298,7 +298,7 @@ func TestPutBlobFailureGeneral(t *testing.T) {
 			CustomQuorumIDs:          []uint{},
 			SignerPrivateKeyHex:      "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
 			DisableTLS:               false,
-			PutBlobEncodingVersion:   codecs.DefaultBlobEncoding,
+			PutBlobEncodingVersion:   codecs.PayloadEncodingVersion0,
 			WaitForFinalization:      true,
 		},
 		Client: disperserClient,
@@ -330,7 +330,7 @@ func TestPutBlobFailureUnknown(t *testing.T) {
 			CustomQuorumIDs:          []uint{},
 			SignerPrivateKeyHex:      "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
 			DisableTLS:               false,
-			PutBlobEncodingVersion:   codecs.DefaultBlobEncoding,
+			PutBlobEncodingVersion:   codecs.PayloadEncodingVersion0,
 			WaitForFinalization:      true,
 		},
 		Client: disperserClient,
@@ -364,7 +364,7 @@ func TestPutBlobFinalizationTimeout(t *testing.T) {
 			CustomQuorumIDs:          []uint{},
 			SignerPrivateKeyHex:      "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
 			DisableTLS:               false,
-			PutBlobEncodingVersion:   codecs.DefaultBlobEncoding,
+			PutBlobEncodingVersion:   codecs.PayloadEncodingVersion0,
 			WaitForFinalization:      true,
 		},
 		Client: disperserClient,
@@ -423,7 +423,7 @@ func TestPutBlobIndividualRequestTimeout(t *testing.T) {
 			CustomQuorumIDs:          []uint{},
 			SignerPrivateKeyHex:      "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
 			DisableTLS:               false,
-			PutBlobEncodingVersion:   codecs.DefaultBlobEncoding,
+			PutBlobEncodingVersion:   codecs.PayloadEncodingVersion0,
 			WaitForFinalization:      true,
 		},
 		Client: disperserClient,
@@ -485,7 +485,7 @@ func TestPutBlobTotalTimeout(t *testing.T) {
 			CustomQuorumIDs:          []uint{},
 			SignerPrivateKeyHex:      "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
 			DisableTLS:               false,
-			PutBlobEncodingVersion:   codecs.DefaultBlobEncoding,
+			PutBlobEncodingVersion:   codecs.PayloadEncodingVersion0,
 			WaitForFinalization:      true,
 		},
 		Client: disperserClient,

api/clients/v2/codecs/blob.go

@@ -0,0 +1,118 @@
+package codecs
+
+import (
+	"fmt"
+
+	"github.com/Layr-Labs/eigenda/api/clients/codecs"
+	"github.com/Layr-Labs/eigenda/encoding/fft"
+	"github.com/Layr-Labs/eigenda/encoding/rs"
+	"github.com/Layr-Labs/eigenda/encoding/utils/codec"
+	"github.com/consensys/gnark-crypto/ecc/bn254/fr"
+)
+
+// Blob is data that is dispersed on eigenDA.
+//
+// A Blob is represented under the hood by an array of field elements, which represent a polynomial in coefficient form
+type Blob struct {
+	coeffPolynomial []fr.Element
+	// blobLengthSymbols must be a power of 2, and should match the blobLength claimed in the BlobCommitment
+	//
+	// This value must be specified, rather than computed from the length of the coeffPolynomial, due to an edge case
+	// illustrated by the following example: imagine a user disperses a very small blob, only 64 bytes, and the last 40
+	// bytes are trailing zeros. When a different user fetches the blob from a relay, it's possible that the relay could
+	// truncate the trailing zeros. If we were to say that blobLengthSymbols = nextPowerOf2(len(coeffPolynomial)), then the
+	// user fetching and reconstructing this blob would determine that the blob length is 1 symbol, when it's actually 2.
+	blobLengthSymbols uint32
+}
+
+// DeserializeBlob initializes a Blob from bytes
+func DeserializeBlob(bytes []byte, blobLengthSymbols uint32) (*Blob, error) {
+	coeffPolynomial, err := rs.ToFrArray(bytes)
+	if err != nil {
+		return nil, fmt.Errorf("bytes to field elements: %w", err)
+	}
+
+	return BlobFromPolynomial(coeffPolynomial, blobLengthSymbols)
+}
+
+// BlobFromPolynomial initializes a blob from a polynomial
+func BlobFromPolynomial(coeffPolynomial []fr.Element, blobLengthSymbols uint32) (*Blob, error) {
+	return &Blob{
+		coeffPolynomial:   coeffPolynomial,
+		blobLengthSymbols: blobLengthSymbols,
+	}, nil
+}
+
+// Serialize gets the raw bytes of the Blob
+func (b *Blob) Serialize() []byte {
+	return rs.SerializeFieldElements(b.coeffPolynomial)
+}
+
+// ToPayload converts the Blob into a Payload
+//
+// The payloadForm indicates how payloads are interpreted. The way that payloads are interpreted dictates what
+// conversion, if any, must be performed when creating a payload from the blob.
+func (b *Blob) ToPayload(payloadForm codecs.PolynomialForm) (*Payload, error) {
+	encodedPayload, err := b.toEncodedPayload(payloadForm)
+	if err != nil {
+		return nil, fmt.Errorf("to encoded payload: %w", err)
+	}
+
+	payload, err := encodedPayload.decode()
+	if err != nil {
+		return nil, fmt.Errorf("decode payload: %w", err)
+	}
+
+	return payload, nil
+}
+
+// toEncodedPayload creates an encodedPayload from the blob
+//
+// The payloadForm indicates how payloads are interpreted. The way that payloads are interpreted dictates what
+// conversion, if any, must be performed when creating an encoded payload from the blob.
+func (b *Blob) toEncodedPayload(payloadForm codecs.PolynomialForm) (*encodedPayload, error) {
+	maxPermissiblePayloadLength, err := codec.GetMaxPermissiblePayloadLength(b.blobLengthSymbols)
+	if err != nil {
+		return nil, fmt.Errorf("get max permissible payload length: %w", err)
+	}
+
+	var payloadElements []fr.Element
+	switch payloadForm {
+	case codecs.PolynomialFormCoeff:
+		// the payload is interpreted as coefficients of the polynomial, so no conversion needs to be done, given that
+		// eigenda also interprets blobs as coefficients
+		payloadElements = b.coeffPolynomial
+	case codecs.PolynomialFormEval:
+		// the payload is interpreted as evaluations of the polynomial, so the coefficient representation contained
+		// in the blob must be converted to the evaluation form
+		payloadElements, err = b.computeEvalPoly()
+		if err != nil {
+			return nil, fmt.Errorf("compute eval poly: %w", err)
+		}
+	default:
+		return nil, fmt.Errorf("invalid polynomial form")
+	}
+
+	encodedPayload, err := encodedPayloadFromElements(payloadElements, maxPermissiblePayloadLength)
+	if err != nil {
+		return nil, fmt.Errorf("encoded payload from elements %w", err)
+	}
+
+	return encodedPayload, nil
+}
+
+// computeEvalPoly converts a blob's coeffPoly to an evalPoly, using the FFT operation
+func (b *Blob) computeEvalPoly() ([]fr.Element, error) {
+	// TODO (litt3): this could conceivably be optimized, so that multiple objects share an instance of FFTSettings,
+	//  which has enough roots of unity for general use. If the following construction of FFTSettings ever proves
+	//  to present a computational burden, consider making this change.
+	fftSettings := fft.FFTSettingsFromBlobLengthSymbols(b.blobLengthSymbols)
+
+	// the FFT method pads to the next power of 2, so we don't need to do that manually
+	fftedElements, err := fftSettings.FFT(b.coeffPolynomial, false)
+	if err != nil {
+		return nil, fmt.Errorf("perform FFT: %w", err)
+	}
+
+	return fftedElements, nil
+}

api/clients/v2/codecs/blob_test.go

@@ -0,0 +1,40 @@
+package codecs
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/Layr-Labs/eigenda/api/clients/codecs"
+	"github.com/stretchr/testify/require"
+)
+
+// TestBlobConversion checks that internal blob conversion methods produce consistent results
+func FuzzBlobConversion(f *testing.F) {
+	for _, seed := range [][]byte{{}, {0x00}, {0xFF}, {0x00, 0x00}, {0xFF, 0xFF}, bytes.Repeat([]byte{0x55}, 1000)} {
+		f.Add(seed)
+	}
+
+	f.Fuzz(
+		func(t *testing.T, originalData []byte) {
+			testBlobConversionForForm(t, originalData, codecs.PolynomialFormEval)
+			testBlobConversionForForm(t, originalData, codecs.PolynomialFormCoeff)
+		})
+
+}
+
+func testBlobConversionForForm(t *testing.T, payloadBytes []byte, payloadForm codecs.PolynomialForm) {
+	blob, err := NewPayload(payloadBytes).ToBlob(payloadForm)
+	require.NoError(t, err)
+
+	blobDeserialized, err := DeserializeBlob(blob.Serialize(), blob.blobLengthSymbols)
+	require.NoError(t, err)
+
+	payloadFromBlob, err := blob.ToPayload(payloadForm)
+	require.NoError(t, err)
+
+	payloadFromDeserializedBlob, err := blobDeserialized.ToPayload(payloadForm)
+	require.NoError(t, err)
+
+	require.Equal(t, payloadFromBlob.Serialize(), payloadFromDeserializedBlob.Serialize())
+	require.Equal(t, payloadBytes, payloadFromBlob.Serialize())
+}