Skip to content

Commit 412342c

Browse files
litt3litt3
authored
Create payload->blob type system (#1120)
Signed-off-by: litt3 <[email protected]>
1 parent 5f060c6 commit 412342c

20 files changed

+836
-47
lines changed

‎api/clients/codecs/blob_codec.go

+9-7
Original file line numberDiff line numberDiff line change
@@ -4,22 +4,24 @@ import (
44
"fmt"
55
)
66

7-
type BlobEncodingVersion byte
7+
type PayloadEncodingVersion uint8
88

99
const (
10-
// This minimal blob encoding contains a 32 byte header = [0x00, version byte, uint32 len of data, 0x00, 0x00,...]
10+
// PayloadEncodingVersion0 entails a 32 byte header = [0x00, version byte, big-endian uint32 len of payload, 0x00, 0x00,...]
1111
// followed by the encoded data [0x00, 31 bytes of data, 0x00, 31 bytes of data,...]
12-
DefaultBlobEncoding BlobEncodingVersion = 0x0
12+
//
13+
// Each group of 32 bytes starts with a 0x00 byte so that they can be parsed as valid bn254 field elements.
14+
PayloadEncodingVersion0 PayloadEncodingVersion = 0x0
1315
)
1416

1517
type BlobCodec interface {
1618
DecodeBlob(encodedData []byte) ([]byte, error)
1719
EncodeBlob(rawData []byte) ([]byte, error)
1820
}
1921

20-
func BlobEncodingVersionToCodec(version BlobEncodingVersion) (BlobCodec, error) {
22+
func BlobEncodingVersionToCodec(version PayloadEncodingVersion) (BlobCodec, error) {
2123
switch version {
22-
case DefaultBlobEncoding:
24+
case PayloadEncodingVersion0:
2325
return DefaultBlobCodec{}, nil
2426
default:
2527
return nil, fmt.Errorf("unsupported blob encoding version: %x", version)
@@ -33,7 +35,7 @@ func GenericDecodeBlob(data []byte) ([]byte, error) {
3335
// version byte is stored in [1], because [0] is always 0 to ensure the codecBlobHeader is a valid bn254 element
3436
// see https://github.com/Layr-Labs/eigenda/blob/master/api/clients/codecs/default_blob_codec.go#L21
3537
// TODO: we should prob be working over a struct with methods such as GetBlobEncodingVersion() to prevent index errors
36-
version := BlobEncodingVersion(data[1])
38+
version := PayloadEncodingVersion(data[1])
3739
codec, err := BlobEncodingVersionToCodec(version)
3840
if err != nil {
3941
return nil, err
@@ -49,7 +51,7 @@ func GenericDecodeBlob(data []byte) ([]byte, error) {
4951

5052
// CreateCodec creates a new BlobCodec based on the defined polynomial form of payloads, and the desired
5153
// BlobEncodingVersion
52-
func CreateCodec(payloadPolynomialForm PolynomialForm, version BlobEncodingVersion) (BlobCodec, error) {
54+
func CreateCodec(payloadPolynomialForm PolynomialForm, version PayloadEncodingVersion) (BlobCodec, error) {
5355
lowLevelCodec, err := BlobEncodingVersionToCodec(version)
5456
if err != nil {
5557
return nil, fmt.Errorf("create low level codec: %w", err)

‎api/clients/codecs/default_blob_codec.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ func (v DefaultBlobCodec) EncodeBlob(rawData []byte) ([]byte, error) {
2222
codecBlobHeader := make([]byte, 32)
2323
// first byte is always 0 to ensure the codecBlobHeader is a valid bn254 element
2424
// encode version byte
25-
codecBlobHeader[1] = byte(DefaultBlobEncoding)
25+
codecBlobHeader[1] = byte(PayloadEncodingVersion0)
2626

2727
// encode length as uint32
2828
binary.BigEndian.PutUint32(codecBlobHeader[2:6], uint32(len(rawData))) // uint32 should be more than enough to store the length (approx 4gb)

‎api/clients/config.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ type EigenDAClientConfig struct {
6464
DisableTLS bool
6565

6666
// The blob encoding version to use when writing blobs from the high level interface.
67-
PutBlobEncodingVersion codecs.BlobEncodingVersion
67+
PutBlobEncodingVersion codecs.PayloadEncodingVersion
6868

6969
// Point verification mode does an IFFT on data before it is written, and does an FFT on data after it is read.
7070
// This makes it possible to open points on the KZG commitment to prove that the field elements correspond to

‎api/clients/eigenda_client_test.go

+10-10
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ func TestPutRetrieveBlobIFFTSuccess(t *testing.T) {
6565
CustomQuorumIDs: []uint{},
6666
SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
6767
DisableTLS: false,
68-
PutBlobEncodingVersion: codecs.DefaultBlobEncoding,
68+
PutBlobEncodingVersion: codecs.PayloadEncodingVersion0,
6969
DisablePointVerificationMode: false,
7070
WaitForFinalization: true,
7171
},
@@ -132,7 +132,7 @@ func TestPutRetrieveBlobIFFTNoDecodeSuccess(t *testing.T) {
132132
CustomQuorumIDs: []uint{},
133133
SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
134134
DisableTLS: false,
135-
PutBlobEncodingVersion: codecs.DefaultBlobEncoding,
135+
PutBlobEncodingVersion: codecs.PayloadEncodingVersion0,
136136
DisablePointVerificationMode: false,
137137
WaitForFinalization: true,
138138
},
@@ -202,7 +202,7 @@ func TestPutRetrieveBlobNoIFFTSuccess(t *testing.T) {
202202
CustomQuorumIDs: []uint{},
203203
SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
204204
DisableTLS: false,
205-
PutBlobEncodingVersion: codecs.DefaultBlobEncoding,
205+
PutBlobEncodingVersion: codecs.PayloadEncodingVersion0,
206206
DisablePointVerificationMode: true,
207207
WaitForFinalization: true,
208208
},
@@ -234,7 +234,7 @@ func TestPutBlobFailDispersal(t *testing.T) {
234234
CustomQuorumIDs: []uint{},
235235
SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
236236
DisableTLS: false,
237-
PutBlobEncodingVersion: codecs.DefaultBlobEncoding,
237+
PutBlobEncodingVersion: codecs.PayloadEncodingVersion0,
238238
WaitForFinalization: true,
239239
},
240240
Client: disperserClient,
@@ -266,7 +266,7 @@ func TestPutBlobFailureInsufficentSignatures(t *testing.T) {
266266
CustomQuorumIDs: []uint{},
267267
SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
268268
DisableTLS: false,
269-
PutBlobEncodingVersion: codecs.DefaultBlobEncoding,
269+
PutBlobEncodingVersion: codecs.PayloadEncodingVersion0,
270270
WaitForFinalization: true,
271271
},
272272
Client: disperserClient,
@@ -298,7 +298,7 @@ func TestPutBlobFailureGeneral(t *testing.T) {
298298
CustomQuorumIDs: []uint{},
299299
SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
300300
DisableTLS: false,
301-
PutBlobEncodingVersion: codecs.DefaultBlobEncoding,
301+
PutBlobEncodingVersion: codecs.PayloadEncodingVersion0,
302302
WaitForFinalization: true,
303303
},
304304
Client: disperserClient,
@@ -330,7 +330,7 @@ func TestPutBlobFailureUnknown(t *testing.T) {
330330
CustomQuorumIDs: []uint{},
331331
SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
332332
DisableTLS: false,
333-
PutBlobEncodingVersion: codecs.DefaultBlobEncoding,
333+
PutBlobEncodingVersion: codecs.PayloadEncodingVersion0,
334334
WaitForFinalization: true,
335335
},
336336
Client: disperserClient,
@@ -364,7 +364,7 @@ func TestPutBlobFinalizationTimeout(t *testing.T) {
364364
CustomQuorumIDs: []uint{},
365365
SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
366366
DisableTLS: false,
367-
PutBlobEncodingVersion: codecs.DefaultBlobEncoding,
367+
PutBlobEncodingVersion: codecs.PayloadEncodingVersion0,
368368
WaitForFinalization: true,
369369
},
370370
Client: disperserClient,
@@ -423,7 +423,7 @@ func TestPutBlobIndividualRequestTimeout(t *testing.T) {
423423
CustomQuorumIDs: []uint{},
424424
SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
425425
DisableTLS: false,
426-
PutBlobEncodingVersion: codecs.DefaultBlobEncoding,
426+
PutBlobEncodingVersion: codecs.PayloadEncodingVersion0,
427427
WaitForFinalization: true,
428428
},
429429
Client: disperserClient,
@@ -485,7 +485,7 @@ func TestPutBlobTotalTimeout(t *testing.T) {
485485
CustomQuorumIDs: []uint{},
486486
SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d",
487487
DisableTLS: false,
488-
PutBlobEncodingVersion: codecs.DefaultBlobEncoding,
488+
PutBlobEncodingVersion: codecs.PayloadEncodingVersion0,
489489
WaitForFinalization: true,
490490
},
491491
Client: disperserClient,

‎api/clients/v2/codecs/blob.go

+118
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
package codecs
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/Layr-Labs/eigenda/api/clients/codecs"
7+
"github.com/Layr-Labs/eigenda/encoding/fft"
8+
"github.com/Layr-Labs/eigenda/encoding/rs"
9+
"github.com/Layr-Labs/eigenda/encoding/utils/codec"
10+
"github.com/consensys/gnark-crypto/ecc/bn254/fr"
11+
)
12+
13+
// Blob is data that is dispersed on eigenDA.
14+
//
15+
// A Blob is represented under the hood by an array of field elements, which represent a polynomial in coefficient form
16+
type Blob struct {
17+
coeffPolynomial []fr.Element
18+
// blobLengthSymbols must be a power of 2, and should match the blobLength claimed in the BlobCommitment
19+
//
20+
// This value must be specified, rather than computed from the length of the coeffPolynomial, due to an edge case
21+
// illustrated by the following example: imagine a user disperses a very small blob, only 64 bytes, and the last 40
22+
// bytes are trailing zeros. When a different user fetches the blob from a relay, it's possible that the relay could
23+
// truncate the trailing zeros. If we were to say that blobLengthSymbols = nextPowerOf2(len(coeffPolynomial)), then the
24+
// user fetching and reconstructing this blob would determine that the blob length is 1 symbol, when it's actually 2.
25+
blobLengthSymbols uint32
26+
}
27+
28+
// DeserializeBlob initializes a Blob from bytes
29+
func DeserializeBlob(bytes []byte, blobLengthSymbols uint32) (*Blob, error) {
30+
coeffPolynomial, err := rs.ToFrArray(bytes)
31+
if err != nil {
32+
return nil, fmt.Errorf("bytes to field elements: %w", err)
33+
}
34+
35+
return BlobFromPolynomial(coeffPolynomial, blobLengthSymbols)
36+
}
37+
38+
// BlobFromPolynomial initializes a blob from a polynomial
39+
func BlobFromPolynomial(coeffPolynomial []fr.Element, blobLengthSymbols uint32) (*Blob, error) {
40+
return &Blob{
41+
coeffPolynomial: coeffPolynomial,
42+
blobLengthSymbols: blobLengthSymbols,
43+
}, nil
44+
}
45+
46+
// Serialize gets the raw bytes of the Blob
47+
func (b *Blob) Serialize() []byte {
48+
return rs.SerializeFieldElements(b.coeffPolynomial)
49+
}
50+
51+
// ToPayload converts the Blob into a Payload
52+
//
53+
// The payloadForm indicates how payloads are interpreted. The way that payloads are interpreted dictates what
54+
// conversion, if any, must be performed when creating a payload from the blob.
55+
func (b *Blob) ToPayload(payloadForm codecs.PolynomialForm) (*Payload, error) {
56+
encodedPayload, err := b.toEncodedPayload(payloadForm)
57+
if err != nil {
58+
return nil, fmt.Errorf("to encoded payload: %w", err)
59+
}
60+
61+
payload, err := encodedPayload.decode()
62+
if err != nil {
63+
return nil, fmt.Errorf("decode payload: %w", err)
64+
}
65+
66+
return payload, nil
67+
}
68+
69+
// toEncodedPayload creates an encodedPayload from the blob
70+
//
71+
// The payloadForm indicates how payloads are interpreted. The way that payloads are interpreted dictates what
72+
// conversion, if any, must be performed when creating an encoded payload from the blob.
73+
func (b *Blob) toEncodedPayload(payloadForm codecs.PolynomialForm) (*encodedPayload, error) {
74+
maxPermissiblePayloadLength, err := codec.GetMaxPermissiblePayloadLength(b.blobLengthSymbols)
75+
if err != nil {
76+
return nil, fmt.Errorf("get max permissible payload length: %w", err)
77+
}
78+
79+
var payloadElements []fr.Element
80+
switch payloadForm {
81+
case codecs.PolynomialFormCoeff:
82+
// the payload is interpreted as coefficients of the polynomial, so no conversion needs to be done, given that
83+
// eigenda also interprets blobs as coefficients
84+
payloadElements = b.coeffPolynomial
85+
case codecs.PolynomialFormEval:
86+
// the payload is interpreted as evaluations of the polynomial, so the coefficient representation contained
87+
// in the blob must be converted to the evaluation form
88+
payloadElements, err = b.computeEvalPoly()
89+
if err != nil {
90+
return nil, fmt.Errorf("compute eval poly: %w", err)
91+
}
92+
default:
93+
return nil, fmt.Errorf("invalid polynomial form")
94+
}
95+
96+
encodedPayload, err := encodedPayloadFromElements(payloadElements, maxPermissiblePayloadLength)
97+
if err != nil {
98+
return nil, fmt.Errorf("encoded payload from elements %w", err)
99+
}
100+
101+
return encodedPayload, nil
102+
}
103+
104+
// computeEvalPoly converts a blob's coeffPoly to an evalPoly, using the FFT operation
105+
func (b *Blob) computeEvalPoly() ([]fr.Element, error) {
106+
// TODO (litt3): this could conceivably be optimized, so that multiple objects share an instance of FFTSettings,
107+
// which has enough roots of unity for general use. If the following construction of FFTSettings ever proves
108+
// to present a computational burden, consider making this change.
109+
fftSettings := fft.FFTSettingsFromBlobLengthSymbols(b.blobLengthSymbols)
110+
111+
// the FFT method pads to the next power of 2, so we don't need to do that manually
112+
fftedElements, err := fftSettings.FFT(b.coeffPolynomial, false)
113+
if err != nil {
114+
return nil, fmt.Errorf("perform FFT: %w", err)
115+
}
116+
117+
return fftedElements, nil
118+
}

‎api/clients/v2/codecs/blob_test.go

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package codecs
2+
3+
import (
4+
"bytes"
5+
"testing"
6+
7+
"github.com/Layr-Labs/eigenda/api/clients/codecs"
8+
"github.com/stretchr/testify/require"
9+
)
10+
11+
// TestBlobConversion checks that internal blob conversion methods produce consistent results
12+
func FuzzBlobConversion(f *testing.F) {
13+
for _, seed := range [][]byte{{}, {0x00}, {0xFF}, {0x00, 0x00}, {0xFF, 0xFF}, bytes.Repeat([]byte{0x55}, 1000)} {
14+
f.Add(seed)
15+
}
16+
17+
f.Fuzz(
18+
func(t *testing.T, originalData []byte) {
19+
testBlobConversionForForm(t, originalData, codecs.PolynomialFormEval)
20+
testBlobConversionForForm(t, originalData, codecs.PolynomialFormCoeff)
21+
})
22+
23+
}
24+
25+
func testBlobConversionForForm(t *testing.T, payloadBytes []byte, payloadForm codecs.PolynomialForm) {
26+
blob, err := NewPayload(payloadBytes).ToBlob(payloadForm)
27+
require.NoError(t, err)
28+
29+
blobDeserialized, err := DeserializeBlob(blob.Serialize(), blob.blobLengthSymbols)
30+
require.NoError(t, err)
31+
32+
payloadFromBlob, err := blob.ToPayload(payloadForm)
33+
require.NoError(t, err)
34+
35+
payloadFromDeserializedBlob, err := blobDeserialized.ToPayload(payloadForm)
36+
require.NoError(t, err)
37+
38+
require.Equal(t, payloadFromBlob.Serialize(), payloadFromDeserializedBlob.Serialize())
39+
require.Equal(t, payloadBytes, payloadFromBlob.Serialize())
40+
}

0 commit comments

Comments
 (0)