diff --git a/api/clients/codecs/blob_codec.go b/api/clients/codecs/blob_codec.go index ec3796a690..3196c3b053 100644 --- a/api/clients/codecs/blob_codec.go +++ b/api/clients/codecs/blob_codec.go @@ -4,12 +4,14 @@ import ( "fmt" ) -type BlobEncodingVersion byte +type PayloadEncodingVersion uint8 const ( - // This minimal blob encoding contains a 32 byte header = [0x00, version byte, uint32 len of data, 0x00, 0x00,...] + // PayloadEncodingVersion0 entails a 32 byte header = [0x00, version byte, big-endian uint32 len of payload, 0x00, 0x00,...] // followed by the encoded data [0x00, 31 bytes of data, 0x00, 31 bytes of data,...] - DefaultBlobEncoding BlobEncodingVersion = 0x0 + // + // Each group of 32 bytes starts with a 0x00 byte so that they can be parsed as valid bn254 field elements. + PayloadEncodingVersion0 PayloadEncodingVersion = 0x0 ) type BlobCodec interface { @@ -17,9 +19,9 @@ type BlobCodec interface { EncodeBlob(rawData []byte) ([]byte, error) } -func BlobEncodingVersionToCodec(version BlobEncodingVersion) (BlobCodec, error) { +func BlobEncodingVersionToCodec(version PayloadEncodingVersion) (BlobCodec, error) { switch version { - case DefaultBlobEncoding: + case PayloadEncodingVersion0: return DefaultBlobCodec{}, nil default: return nil, fmt.Errorf("unsupported blob encoding version: %x", version) @@ -33,7 +35,7 @@ func GenericDecodeBlob(data []byte) ([]byte, error) { // version byte is stored in [1], because [0] is always 0 to ensure the codecBlobHeader is a valid bn254 element // see https://github.com/Layr-Labs/eigenda/blob/master/api/clients/codecs/default_blob_codec.go#L21 // TODO: we should prob be working over a struct with methods such as GetBlobEncodingVersion() to prevent index errors - version := BlobEncodingVersion(data[1]) + version := PayloadEncodingVersion(data[1]) codec, err := BlobEncodingVersionToCodec(version) if err != nil { return nil, err @@ -49,7 +51,7 @@ func GenericDecodeBlob(data []byte) ([]byte, error) { // CreateCodec creates a new BlobCodec based on the defined polynomial form of payloads, and the desired // BlobEncodingVersion -func CreateCodec(payloadPolynomialForm PolynomialForm, version BlobEncodingVersion) (BlobCodec, error) { +func CreateCodec(payloadPolynomialForm PolynomialForm, version PayloadEncodingVersion) (BlobCodec, error) { lowLevelCodec, err := BlobEncodingVersionToCodec(version) if err != nil { return nil, fmt.Errorf("create low level codec: %w", err) diff --git a/api/clients/codecs/default_blob_codec.go b/api/clients/codecs/default_blob_codec.go index 6d3ec29944..4b6fc590c7 100644 --- a/api/clients/codecs/default_blob_codec.go +++ b/api/clients/codecs/default_blob_codec.go @@ -22,7 +22,7 @@ func (v DefaultBlobCodec) EncodeBlob(rawData []byte) ([]byte, error) { codecBlobHeader := make([]byte, 32) // first byte is always 0 to ensure the codecBlobHeader is a valid bn254 element // encode version byte - codecBlobHeader[1] = byte(DefaultBlobEncoding) + codecBlobHeader[1] = byte(PayloadEncodingVersion0) // encode length as uint32 binary.BigEndian.PutUint32(codecBlobHeader[2:6], uint32(len(rawData))) // uint32 should be more than enough to store the length (approx 4gb) diff --git a/api/clients/config.go b/api/clients/config.go index f4d9caa9fb..b3d1d946d6 100644 --- a/api/clients/config.go +++ b/api/clients/config.go @@ -64,7 +64,7 @@ type EigenDAClientConfig struct { DisableTLS bool // The blob encoding version to use when writing blobs from the high level interface. - PutBlobEncodingVersion codecs.BlobEncodingVersion + PutBlobEncodingVersion codecs.PayloadEncodingVersion // Point verification mode does an IFFT on data before it is written, and does an FFT on data after it is read. // This makes it possible to open points on the KZG commitment to prove that the field elements correspond to diff --git a/api/clients/eigenda_client_test.go b/api/clients/eigenda_client_test.go index 84e0bf716d..318c99a8cb 100644 --- a/api/clients/eigenda_client_test.go +++ b/api/clients/eigenda_client_test.go @@ -65,7 +65,7 @@ func TestPutRetrieveBlobIFFTSuccess(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, DisablePointVerificationMode: false, WaitForFinalization: true, }, @@ -132,7 +132,7 @@ func TestPutRetrieveBlobIFFTNoDecodeSuccess(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, DisablePointVerificationMode: false, WaitForFinalization: true, }, @@ -202,7 +202,7 @@ func TestPutRetrieveBlobNoIFFTSuccess(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, DisablePointVerificationMode: true, WaitForFinalization: true, }, @@ -234,7 +234,7 @@ func TestPutBlobFailDispersal(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -266,7 +266,7 @@ func TestPutBlobFailureInsufficentSignatures(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -298,7 +298,7 @@ func TestPutBlobFailureGeneral(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -330,7 +330,7 @@ func TestPutBlobFailureUnknown(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -364,7 +364,7 @@ func TestPutBlobFinalizationTimeout(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -423,7 +423,7 @@ func TestPutBlobIndividualRequestTimeout(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -485,7 +485,7 @@ func TestPutBlobTotalTimeout(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, diff --git a/api/clients/v2/codecs/blob.go b/api/clients/v2/codecs/blob.go new file mode 100644 index 0000000000..c52148f679 --- /dev/null +++ b/api/clients/v2/codecs/blob.go @@ -0,0 +1,118 @@ +package codecs + +import ( + "fmt" + + "github.com/Layr-Labs/eigenda/api/clients/codecs" + "github.com/Layr-Labs/eigenda/encoding/fft" + "github.com/Layr-Labs/eigenda/encoding/rs" + "github.com/Layr-Labs/eigenda/encoding/utils/codec" + "github.com/consensys/gnark-crypto/ecc/bn254/fr" +) + +// Blob is data that is dispersed on eigenDA. +// +// A Blob is represented under the hood by an array of field elements, which represent a polynomial in coefficient form +type Blob struct { + coeffPolynomial []fr.Element + // blobLengthSymbols must be a power of 2, and should match the blobLength claimed in the BlobCommitment + // + // This value must be specified, rather than computed from the length of the coeffPolynomial, due to an edge case + // illustrated by the following example: imagine a user disperses a very small blob, only 64 bytes, and the last 40 + // bytes are trailing zeros. When a different user fetches the blob from a relay, it's possible that the relay could + // truncate the trailing zeros. If we were to say that blobLengthSymbols = nextPowerOf2(len(coeffPolynomial)), then the + // user fetching and reconstructing this blob would determine that the blob length is 1 symbol, when it's actually 2. + blobLengthSymbols uint32 +} + +// DeserializeBlob initializes a Blob from bytes +func DeserializeBlob(bytes []byte, blobLengthSymbols uint32) (*Blob, error) { + coeffPolynomial, err := rs.ToFrArray(bytes) + if err != nil { + return nil, fmt.Errorf("bytes to field elements: %w", err) + } + + return BlobFromPolynomial(coeffPolynomial, blobLengthSymbols) +} + +// BlobFromPolynomial initializes a blob from a polynomial +func BlobFromPolynomial(coeffPolynomial []fr.Element, blobLengthSymbols uint32) (*Blob, error) { + return &Blob{ + coeffPolynomial: coeffPolynomial, + blobLengthSymbols: blobLengthSymbols, + }, nil +} + +// Serialize gets the raw bytes of the Blob +func (b *Blob) Serialize() []byte { + return rs.SerializeFieldElements(b.coeffPolynomial) +} + +// ToPayload converts the Blob into a Payload +// +// The payloadForm indicates how payloads are interpreted. The way that payloads are interpreted dictates what +// conversion, if any, must be performed when creating a payload from the blob. +func (b *Blob) ToPayload(payloadForm codecs.PolynomialForm) (*Payload, error) { + encodedPayload, err := b.toEncodedPayload(payloadForm) + if err != nil { + return nil, fmt.Errorf("to encoded payload: %w", err) + } + + payload, err := encodedPayload.decode() + if err != nil { + return nil, fmt.Errorf("decode payload: %w", err) + } + + return payload, nil +} + +// toEncodedPayload creates an encodedPayload from the blob +// +// The payloadForm indicates how payloads are interpreted. The way that payloads are interpreted dictates what +// conversion, if any, must be performed when creating an encoded payload from the blob. +func (b *Blob) toEncodedPayload(payloadForm codecs.PolynomialForm) (*encodedPayload, error) { + maxPermissiblePayloadLength, err := codec.GetMaxPermissiblePayloadLength(b.blobLengthSymbols) + if err != nil { + return nil, fmt.Errorf("get max permissible payload length: %w", err) + } + + var payloadElements []fr.Element + switch payloadForm { + case codecs.PolynomialFormCoeff: + // the payload is interpreted as coefficients of the polynomial, so no conversion needs to be done, given that + // eigenda also interprets blobs as coefficients + payloadElements = b.coeffPolynomial + case codecs.PolynomialFormEval: + // the payload is interpreted as evaluations of the polynomial, so the coefficient representation contained + // in the blob must be converted to the evaluation form + payloadElements, err = b.computeEvalPoly() + if err != nil { + return nil, fmt.Errorf("compute eval poly: %w", err) + } + default: + return nil, fmt.Errorf("invalid polynomial form") + } + + encodedPayload, err := encodedPayloadFromElements(payloadElements, maxPermissiblePayloadLength) + if err != nil { + return nil, fmt.Errorf("encoded payload from elements %w", err) + } + + return encodedPayload, nil +} + +// computeEvalPoly converts a blob's coeffPoly to an evalPoly, using the FFT operation +func (b *Blob) computeEvalPoly() ([]fr.Element, error) { + // TODO (litt3): this could conceivably be optimized, so that multiple objects share an instance of FFTSettings, + // which has enough roots of unity for general use. If the following construction of FFTSettings ever proves + // to present a computational burden, consider making this change. + fftSettings := fft.FFTSettingsFromBlobLengthSymbols(b.blobLengthSymbols) + + // the FFT method pads to the next power of 2, so we don't need to do that manually + fftedElements, err := fftSettings.FFT(b.coeffPolynomial, false) + if err != nil { + return nil, fmt.Errorf("perform FFT: %w", err) + } + + return fftedElements, nil +} diff --git a/api/clients/v2/codecs/blob_test.go b/api/clients/v2/codecs/blob_test.go new file mode 100644 index 0000000000..7fae932e47 --- /dev/null +++ b/api/clients/v2/codecs/blob_test.go @@ -0,0 +1,40 @@ +package codecs + +import ( + "bytes" + "testing" + + "github.com/Layr-Labs/eigenda/api/clients/codecs" + "github.com/stretchr/testify/require" +) + +// TestBlobConversion checks that internal blob conversion methods produce consistent results +func FuzzBlobConversion(f *testing.F) { + for _, seed := range [][]byte{{}, {0x00}, {0xFF}, {0x00, 0x00}, {0xFF, 0xFF}, bytes.Repeat([]byte{0x55}, 1000)} { + f.Add(seed) + } + + f.Fuzz( + func(t *testing.T, originalData []byte) { + testBlobConversionForForm(t, originalData, codecs.PolynomialFormEval) + testBlobConversionForForm(t, originalData, codecs.PolynomialFormCoeff) + }) + +} + +func testBlobConversionForForm(t *testing.T, payloadBytes []byte, payloadForm codecs.PolynomialForm) { + blob, err := NewPayload(payloadBytes).ToBlob(payloadForm) + require.NoError(t, err) + + blobDeserialized, err := DeserializeBlob(blob.Serialize(), blob.blobLengthSymbols) + require.NoError(t, err) + + payloadFromBlob, err := blob.ToPayload(payloadForm) + require.NoError(t, err) + + payloadFromDeserializedBlob, err := blobDeserialized.ToPayload(payloadForm) + require.NoError(t, err) + + require.Equal(t, payloadFromBlob.Serialize(), payloadFromDeserializedBlob.Serialize()) + require.Equal(t, payloadBytes, payloadFromBlob.Serialize()) +} diff --git a/api/clients/v2/codecs/encoded_payload.go b/api/clients/v2/codecs/encoded_payload.go new file mode 100644 index 0000000000..a8eb972563 --- /dev/null +++ b/api/clients/v2/codecs/encoded_payload.go @@ -0,0 +1,151 @@ +package codecs + +import ( + "encoding/binary" + "fmt" + + "github.com/Layr-Labs/eigenda/api/clients/codecs" + "github.com/Layr-Labs/eigenda/encoding/rs" + "github.com/Layr-Labs/eigenda/encoding/utils/codec" + "github.com/consensys/gnark-crypto/ecc/bn254/fr" +) + +// encodedPayload represents a payload that has had an encoding applied to it +// +// Example encoding: +// +// Encoded Payload header (32 bytes total) Encoded Payload Data (len is multiple of 32) +// [0x00, version byte, big-endian uint32 len of payload, 0x00, ...] + [0x00, 31 bytes of data, 0x00, 31 bytes of data,...] +type encodedPayload struct { + // the size of these bytes is guaranteed to be a multiple of 32 + bytes []byte +} + +// newEncodedPayload accepts a payload, and performs the PayloadEncodingVersion0 encoding to create an encoded payload +func newEncodedPayload(payload *Payload) (*encodedPayload, error) { + encodedPayloadHeader := make([]byte, 32) + // first byte is always 0 to ensure the payloadHeader is a valid bn254 element + encodedPayloadHeader[1] = byte(codecs.PayloadEncodingVersion0) // encode version byte + + payloadBytes := payload.Serialize() + + // encode payload length as uint32 + binary.BigEndian.PutUint32( + encodedPayloadHeader[2:6], + uint32(len(payloadBytes))) // uint32 should be more than enough to store the length (approx 4gb) + + // encode payload modulo bn254, and align to 32 bytes + encodedData := codec.PadPayload(payloadBytes) + encodedPayloadBytes := append(encodedPayloadHeader, encodedData...) + + return &encodedPayload{encodedPayloadBytes}, nil +} + +// decode applies the inverse of PayloadEncodingVersion0 to an encodedPayload, and returns the decoded Payload +func (ep *encodedPayload) decode() (*Payload, error) { + claimedLength := binary.BigEndian.Uint32(ep.bytes[2:6]) + + // decode raw data modulo bn254 + unpaddedData, err := codec.RemoveInternalPadding(ep.bytes[32:]) + if err != nil { + return nil, fmt.Errorf("remove internal padding: %w", err) + } + + unpaddedDataLength := uint32(len(unpaddedData)) + + // data length is checked when constructing an encoded payload. If this error is encountered, that means there + // must be a flaw in the logic at construction time (or someone was bad and didn't use the proper construction methods) + if unpaddedDataLength < claimedLength { + return nil, fmt.Errorf( + "length of unpadded data %d is less than length claimed in encoded payload header %d. this should never happen", + unpaddedDataLength, claimedLength) + } + + // unpadded data length can be slightly bigger than the claimed length, since RemoveInternalPadding doesn't + // do anything to remove trailing zeros that may have been added when the data was initially padded. + // however, this extra padding shouldn't exceed 31 bytes, because that's the most that would be added + // when padding the data length to 32 bytes. If this error occurs, that means there must be a flaw in the logic at + // construction time (or someone was bad and didn't use the proper construction methods) + if unpaddedDataLength > claimedLength+31 { + return nil, fmt.Errorf( + "length of unpadded data %d is more than 31 bytes longer than claimed length %d. this should never happen", + unpaddedDataLength, claimedLength) + } + + return NewPayload(unpaddedData[0:claimedLength]), nil +} + +// toFieldElements converts the encoded payload to an array of field elements +func (ep *encodedPayload) toFieldElements() ([]fr.Element, error) { + fieldElements, err := rs.ToFrArray(ep.bytes) + if err != nil { + return nil, fmt.Errorf("deserialize field elements: %w", err) + } + + return fieldElements, nil +} + +// encodedPayloadFromElements accepts an array of field elements, and converts them into an encoded payload +// +// maxPayloadLength is the maximum length in bytes that the contained Payload is permitted to be +func encodedPayloadFromElements(fieldElements []fr.Element, maxPayloadLength uint32) (*encodedPayload, error) { + polynomialBytes := rs.SerializeFieldElements(fieldElements) + // this is the payload length in bytes, as claimed by the encoded payload header + payloadLength := binary.BigEndian.Uint32(polynomialBytes[2:6]) + + if payloadLength > maxPayloadLength { + return nil, fmt.Errorf( + "payload length claimed in encoded payload header (%d bytes) is larger than the permitted maximum (%d bytes)", + payloadLength, maxPayloadLength) + } + + // this is the length you would get if you padded a payload of the length claimed in the encoded payload header + paddedLength := codec.GetPaddedDataLength(payloadLength) + // add 32 to the padded data length, since the encoded payload includes an encoded payload header + encodedPayloadLength := paddedLength + 32 + + polynomialByteCount := uint32(len(polynomialBytes)) + + // no matter what, this will be a multiple of 32, since the two possible values, encodedPayloadLength and + // polynomialBytes, are multiples of 32. This is important, since the encoded payload being created is + // expected to have a byte count that's a multiple of 32. + lengthToCopy := encodedPayloadLength + + // if encodedPayloadLength is greater than the polynomial bytes, that indicates that the polynomial bytes we have + // are missing trailing 0 bytes which were originally part of the dispersed blob. For this to happen, it means + // that whichever source provided us with these bytes truncated the trailing 0s. This probably won't happen in + // practice, but if it were to happen, it wouldn't be caught when verifying commitments, since trailing 0s don't + // affect the commitment. This isn't a problem, though: we can handle this edge case here. + if encodedPayloadLength > polynomialByteCount { + // we are copying from the polynomialBytes, so make sure that we don't try to copy more data than actually exists + lengthToCopy = polynomialByteCount + } else if encodedPayloadLength < polynomialByteCount { + // we assume that the polynomialBytes might have additional trailing 0s beyond the expected size of the encoded + // payload. Here, we check the assumption that all trailing values are 0. If there are any non-zero trailing + // values, something has gone wrong in the data pipeline, and this should produce a loud failure. Either a + // dispersing client is playing sneaky games, or there's a bug somewhere. + err := checkTrailingZeros(polynomialBytes, encodedPayloadLength) + if err != nil { + return nil, fmt.Errorf("check that trailing values in polynomial are zeros: %w", err) + } + } + + encodedPayloadBytes := make([]byte, encodedPayloadLength) + copy(encodedPayloadBytes, polynomialBytes[:lengthToCopy]) + + return &encodedPayload{encodedPayloadBytes}, nil +} + +// checkTrailingZeros accepts an array of bytes, and the number of bytes at the front of the array which are permitted +// to be non-zero +// +// This function returns an error if any byte in the array after these permitted non-zero values is found to be non-zero +func checkTrailingZeros(inputBytes []byte, nonZeroLength uint32) error { + for i := uint32(len(inputBytes)) - 1; i >= nonZeroLength; i-- { + if inputBytes[i] != 0x0 { + return fmt.Errorf("byte at index %d was expected to be 0x0, but instead was %x", i, inputBytes[i]) + } + } + + return nil +} diff --git a/api/clients/v2/codecs/encoded_payload_test.go b/api/clients/v2/codecs/encoded_payload_test.go new file mode 100644 index 0000000000..0053715bc9 --- /dev/null +++ b/api/clients/v2/codecs/encoded_payload_test.go @@ -0,0 +1,124 @@ +package codecs + +import ( + "testing" + + "github.com/Layr-Labs/eigenda/common/testutils/random" + "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/utils/codec" + "github.com/consensys/gnark-crypto/ecc/bn254/fr" + "github.com/stretchr/testify/require" +) + +// TestDecodeShortBytes checks that an encoded payload with a length less than claimed length fails at decode time +func TestDecodeShortBytes(t *testing.T) { + testRandom := random.NewTestRandom() + originalData := testRandom.Bytes(testRandom.Intn(1024) + 33) + + encodedPayload, err := newEncodedPayload(NewPayload(originalData)) + require.NoError(t, err) + + // truncate + encodedPayload.bytes = encodedPayload.bytes[:len(encodedPayload.bytes) -32] + + payload, err := encodedPayload.decode() + require.Error(t, err) + require.Nil(t, payload) +} + +// TestDecodeLongBytes checks that an encoded payload with length too much greater than claimed fails at decode +func TestDecodeLongBytes(t *testing.T) { + testRandom := random.NewTestRandom() + originalData := testRandom.Bytes(testRandom.Intn(1024) + 1) + + encodedPayload, err := newEncodedPayload(NewPayload(originalData)) + require.NoError(t, err) + + encodedPayload.bytes = append(encodedPayload.bytes, make([]byte, 32)...) + payload2, err := encodedPayload.decode() + require.Error(t, err) + require.Nil(t, payload2) +} + +// TestEncodeTooManyElements checks that encodedPayloadFromElements fails at the expect limit, relative to payload +// length and blob length +func TestEncodeTooManyElements(t *testing.T) { + testRandom := random.NewTestRandom() + powersOf2 := encoding.GeneratePowersOfTwo(uint32(12)) + + for i := 0; i < len(powersOf2); i++ { + blobLength := powersOf2[i] + maxPermissiblePayloadLength, err := codec.GetMaxPermissiblePayloadLength(blobLength) + require.NoError(t, err) + + almostTooLongData := testRandom.Bytes(int(maxPermissiblePayloadLength)) + almostTooLongEncodedPayload, err := newEncodedPayload(NewPayload(almostTooLongData)) + require.NoError(t, err) + almostTooLongFieldElements, err := almostTooLongEncodedPayload.toFieldElements() + require.NoError(t, err) + // there are almost too many field elements for the defined blob length, but not quite + _, err = encodedPayloadFromElements(almostTooLongFieldElements, maxPermissiblePayloadLength) + require.NoError(t, err) + + tooLongData := testRandom.Bytes(int(maxPermissiblePayloadLength) + 1) + tooLongEncodedPayload, err := newEncodedPayload(NewPayload(tooLongData)) + require.NoError(t, err) + tooLongFieldElements, err := tooLongEncodedPayload.toFieldElements() + require.NoError(t, err) + // there is one too many field elements for the defined blob length + _, err = encodedPayloadFromElements(tooLongFieldElements, maxPermissiblePayloadLength) + require.Error(t, err) + } +} + +// TestTrailingNonZeros checks that any non-zero values that come after the end of the claimed payload length +// cause an error to be returned. +func TestTrailingNonZeros(t *testing.T) { + testRandom := random.NewTestRandom() + originalData := testRandom.Bytes(testRandom.Intn(1024) + 1) + + encodedPayload, err := newEncodedPayload(NewPayload(originalData)) + require.NoError(t, err) + + originalElements, err := encodedPayload.toFieldElements() + require.NoError(t, err) + + fieldElements1 := make([]fr.Element, len(originalElements)) + copy(fieldElements1, originalElements) + + fieldElements2 := make([]fr.Element, len(originalElements)) + copy(fieldElements2, originalElements) + + // adding a 0 is fine + fieldElements1 = append(fieldElements1, fr.Element{}) + _, err = encodedPayloadFromElements(fieldElements1, uint32(len(fieldElements1)*encoding.BYTES_PER_SYMBOL)) + require.NoError(t, err) + + // adding a non-0 is non-fine + fieldElements2 = append(fieldElements2, fr.Element{0,0,0,1}) + _, err = encodedPayloadFromElements(fieldElements2, uint32(len(fieldElements2)*encoding.BYTES_PER_SYMBOL)) + require.Error(t, err) +} + +// TestEncodeWithFewerElements tests that having fewer bytes than expected doesn't throw an error +func TestEncodeWithFewerElements(t *testing.T) { + testRandom := random.NewTestRandom() + originalData := testRandom.Bytes(testRandom.Intn(1024) + 33) + + encodedPayload, err := newEncodedPayload(NewPayload(originalData)) + require.NoError(t, err) + + originalFieldElements, err := encodedPayload.toFieldElements() + require.NoError(t, err) + + truncatedFieldElements := make([]fr.Element, len(originalFieldElements)-1) + // intentionally don't copy all the elements + copy(truncatedFieldElements, originalFieldElements[:len(originalFieldElements)-1]) + + // even though the actual length will be less than the claimed length, we shouldn't see any error + reconstructedEncodedPayload, err := encodedPayloadFromElements( + originalFieldElements, + uint32(len(originalFieldElements))*encoding.BYTES_PER_SYMBOL) + require.NoError(t, err) + require.NotNil(t, reconstructedEncodedPayload) +} diff --git a/api/clients/v2/codecs/payload.go b/api/clients/v2/codecs/payload.go new file mode 100644 index 0000000000..df72ed6fed --- /dev/null +++ b/api/clients/v2/codecs/payload.go @@ -0,0 +1,81 @@ +package codecs + +import ( + "fmt" + + "github.com/Layr-Labs/eigenda/api/clients/codecs" + "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/fft" + "github.com/consensys/gnark-crypto/ecc/bn254/fr" +) + +// Payload represents arbitrary user data, without any processing. +type Payload struct { + bytes []byte +} + +// NewPayload wraps an arbitrary array of bytes into a Payload type. +func NewPayload(payloadBytes []byte) *Payload { + return &Payload{ + bytes: payloadBytes, + } +} + +// ToBlob converts the Payload bytes into a Blob +// +// The payloadForm indicates how payloads are interpreted. The form of a payload dictates what conversion, if any, must +// be performed when creating a blob from the payload. +func (p *Payload) ToBlob(payloadForm codecs.PolynomialForm) (*Blob, error) { + encodedPayload, err := newEncodedPayload(p) + if err != nil { + return nil, fmt.Errorf("encoding payload: %w", err) + } + + fieldElements, err := encodedPayload.toFieldElements() + if err != nil { + return nil, fmt.Errorf("encoded payload to field elements: %w", err) + } + + blobLengthSymbols := uint32(encoding.NextPowerOf2(len(fieldElements))) + + var coeffPolynomial []fr.Element + switch payloadForm { + case codecs.PolynomialFormCoeff: + // the payload is already in coefficient form. no conversion needs to take place, since blobs are also in + // coefficient form + coeffPolynomial = fieldElements + case codecs.PolynomialFormEval: + // the payload is in evaluation form, so we need to convert it to coeff form, since blobs are in coefficient form + coeffPolynomial, err = evalToCoeffPoly(fieldElements, blobLengthSymbols) + if err != nil { + return nil, fmt.Errorf("eval poly to coeff poly: %w", err) + } + default: + return nil, fmt.Errorf("unknown polynomial form: %v", payloadForm) + } + + return BlobFromPolynomial(coeffPolynomial, blobLengthSymbols) +} + +// Serialize returns the bytes that underlie the payload, i.e. the unprocessed user data +func (p *Payload) Serialize() []byte { + return p.bytes +} + +// evalToCoeffPoly converts an evalPoly to a coeffPoly, using the IFFT operation +// +// blobLengthSymbols is required, to be able to choose the correct parameters when performing FFT +func evalToCoeffPoly(evalPoly []fr.Element, blobLengthSymbols uint32) ([]fr.Element, error) { + // TODO (litt3): this could conceivably be optimized, so that multiple objects share an instance of FFTSettings, + // which has enough roots of unity for general use. If the following construction of FFTSettings ever proves + // to present a computational burden, consider making this change. + fftSettings := fft.FFTSettingsFromBlobLengthSymbols(blobLengthSymbols) + + // the FFT method pads to the next power of 2, so we don't need to do that manually + ifftedElements, err := fftSettings.FFT(evalPoly, true) + if err != nil { + return nil, fmt.Errorf("perform IFFT: %w", err) + } + + return ifftedElements, nil +} diff --git a/api/clients/v2/config.go b/api/clients/v2/config.go index efec4d1932..5922d6407d 100644 --- a/api/clients/v2/config.go +++ b/api/clients/v2/config.go @@ -11,8 +11,10 @@ import ( // PayloadClientConfig contains configuration values that are needed by both PayloadRetriever and PayloadDisperser type PayloadClientConfig struct { - // The blob encoding version to use when writing and reading blobs - BlobEncodingVersion codecs.BlobEncodingVersion + // The payload encoding version to use when encoding payload bytes + // + // This is the version that is put into the header of the EncodedPayload. + PayloadEncodingVersion codecs.PayloadEncodingVersion // The address of the EigenDACertVerifier contract EigenDACertVerifierAddr string @@ -112,7 +114,7 @@ type PayloadDisperserConfig struct { // NOTE: EigenDACertVerifierAddr does not have a defined default. It must always be specifically configured. func GetDefaultPayloadClientConfig() *PayloadClientConfig { return &PayloadClientConfig{ - BlobEncodingVersion: codecs.DefaultBlobEncoding, + PayloadEncodingVersion: codecs.PayloadEncodingVersion0, PayloadPolynomialForm: codecs.PolynomialFormEval, ContractCallTimeout: 5 * time.Second, BlockNumberPollInterval: 1 * time.Second, diff --git a/api/clients/v2/payload_disperser.go b/api/clients/v2/payload_disperser.go index 9140baf85f..f799d527c8 100644 --- a/api/clients/v2/payload_disperser.go +++ b/api/clients/v2/payload_disperser.go @@ -88,7 +88,7 @@ func BuildPayloadDisperser(log logging.Logger, payloadDispCfg PayloadDisperserCo } // 5 - create codec - codec, err := codecs.CreateCodec(payloadDispCfg.PayloadPolynomialForm, payloadDispCfg.BlobEncodingVersion) + codec, err := codecs.CreateCodec(payloadDispCfg.PayloadPolynomialForm, payloadDispCfg.PayloadEncodingVersion) if err != nil { return nil, err } diff --git a/api/clients/v2/relay_payload_retriever.go b/api/clients/v2/relay_payload_retriever.go index 4a147fac65..61cc0fd77c 100644 --- a/api/clients/v2/relay_payload_retriever.go +++ b/api/clients/v2/relay_payload_retriever.go @@ -49,7 +49,7 @@ func BuildRelayPayloadRetriever( codec, err := codecs.CreateCodec( relayPayloadRetrieverConfig.PayloadPolynomialForm, - relayPayloadRetrieverConfig.BlobEncodingVersion) + relayPayloadRetrieverConfig.PayloadEncodingVersion) if err != nil { return nil, err } diff --git a/api/clients/v2/validator_payload_retriever.go b/api/clients/v2/validator_payload_retriever.go index a0e552ba27..c5366231ff 100644 --- a/api/clients/v2/validator_payload_retriever.go +++ b/api/clients/v2/validator_payload_retriever.go @@ -76,7 +76,7 @@ func BuildValidatorPayloadRetriever( codec, err := codecs.CreateCodec( validatorPayloadRetrieverConfig.PayloadPolynomialForm, - validatorPayloadRetrieverConfig.BlobEncodingVersion) + validatorPayloadRetrieverConfig.PayloadEncodingVersion) if err != nil { return nil, fmt.Errorf("create codec: %w", err) } diff --git a/encoding/fft/fft.go b/encoding/fft/fft.go index aec175fcca..e9f7a1c5d0 100644 --- a/encoding/fft/fft.go +++ b/encoding/fft/fft.go @@ -27,6 +27,8 @@ package fft import ( + "math" + "github.com/Layr-Labs/eigenda/encoding" "github.com/consensys/gnark-crypto/ecc/bn254/fr" @@ -88,3 +90,9 @@ func NewFFTSettings(maxScale uint8) *FFTSettings { ReverseRootsOfUnity: rootzReverse, } } + +// FFTSettingsFromBlobLengthSymbols accepts a blob length, and returns a new instance of FFT settings +func FFTSettingsFromBlobLengthSymbols(blobLengthSymbols uint32) *FFTSettings { + maxScale := uint8(math.Log2(float64(blobLengthSymbols))) + return NewFFTSettings(maxScale) +} diff --git a/encoding/rs/utils.go b/encoding/rs/utils.go index d1959380b3..dd2a8a1029 100644 --- a/encoding/rs/utils.go +++ b/encoding/rs/utils.go @@ -2,6 +2,7 @@ package rs import ( "errors" + "fmt" "math" "github.com/Layr-Labs/eigenda/encoding" @@ -10,29 +11,55 @@ import ( "github.com/consensys/gnark-crypto/ecc/bn254/fr" ) -func ToFrArray(data []byte) ([]fr.Element, error) { - numEle := GetNumElement(uint64(len(data)), encoding.BYTES_PER_SYMBOL) - eles := make([]fr.Element, numEle) - - for i := uint64(0); i < numEle; i++ { - start := i * uint64(encoding.BYTES_PER_SYMBOL) - end := (i + 1) * uint64(encoding.BYTES_PER_SYMBOL) - if end >= uint64(len(data)) { - padded := make([]byte, encoding.BYTES_PER_SYMBOL) - copy(padded, data[start:]) - err := eles[i].SetBytesCanonical(padded) - if err != nil { - return nil, err - } - } else { - err := eles[i].SetBytesCanonical(data[start:end]) - if err != nil { - return nil, err - } +// ToFrArray accept a byte array as an input, and converts it to an array of field elements +// +// TODO (litt3): it would be nice to rename this to "DeserializeFieldElements", as the counterpart to "SerializeFieldElements", +// but doing so would be a very large diff. I'm leaving this comment as a potential future cleanup. +func ToFrArray(inputData []byte) ([]fr.Element, error) { + bytes := padToBytesPerSymbol(inputData) + + elementCount := len(bytes) / encoding.BYTES_PER_SYMBOL + outputElements := make([]fr.Element, elementCount) + for i := 0; i < elementCount; i++ { + destinationStartIndex := i * encoding.BYTES_PER_SYMBOL + destinationEndIndex := destinationStartIndex + encoding.BYTES_PER_SYMBOL + + err := outputElements[i].SetBytesCanonical(bytes[destinationStartIndex:destinationEndIndex]) + if err != nil { + return nil, fmt.Errorf("fr set bytes canonical: %w", err) } } - return eles, nil + return outputElements, nil +} + +// SerializeFieldElements accepts an array of field elements, and serializes it to an array of bytes +func SerializeFieldElements(fieldElements []fr.Element) []byte { + outputBytes := make([]byte, len(fieldElements)*encoding.BYTES_PER_SYMBOL) + + for i := 0; i < len(fieldElements); i++ { + destinationStartIndex := i * encoding.BYTES_PER_SYMBOL + destinationEndIndex := destinationStartIndex + encoding.BYTES_PER_SYMBOL + + fieldElementBytes := fieldElements[i].Bytes() + + copy(outputBytes[destinationStartIndex:destinationEndIndex], fieldElementBytes[:]) + } + + return outputBytes +} + +// padToBytesPerSymbol accepts input bytes, and returns the bytes padded to a multiple of encoding.BYTES_PER_SYMBOL +func padToBytesPerSymbol(inputBytes []byte) []byte { + remainder := len(inputBytes) % encoding.BYTES_PER_SYMBOL + + if remainder == 0 { + // no padding necessary, since bytes are already a multiple of BYTES_PER_SYMBOL + return inputBytes + } else { + necessaryPadding := encoding.BYTES_PER_SYMBOL - remainder + return append(inputBytes, make([]byte, necessaryPadding)...) + } } // ToByteArray converts a list of Fr to a byte array diff --git a/encoding/test_utils.go b/encoding/test_utils.go new file mode 100644 index 0000000000..3dc78dee0d --- /dev/null +++ b/encoding/test_utils.go @@ -0,0 +1,16 @@ +package encoding + +import ( + "golang.org/x/exp/constraints" +) + +// GeneratePowersOfTwo creates a slice of integers, containing powers of 2 (starting with element == 1), with +// powersToGenerate number of elements +func GeneratePowersOfTwo[T constraints.Integer](powersToGenerate T) []T { + powers := make([]T, powersToGenerate) + for i := T(0); i < powersToGenerate; i++ { + powers[i] = 1 << i + } + + return powers +} diff --git a/encoding/utils/codec/codec.go b/encoding/utils/codec/codec.go index 09659d4332..874453cb3d 100644 --- a/encoding/utils/codec/codec.go +++ b/encoding/utils/codec/codec.go @@ -1,7 +1,10 @@ package codec import ( + "fmt" + "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/fft" ) // ConvertByPaddingEmptyByte takes bytes and insert an empty byte at the front of every 31 byte. @@ -9,6 +12,9 @@ import ( // This ensures every 32 bytes is within the valid range of a field element for bn254 curve. // If the input data is not a multiple of 31, the remainder is added to the output by // inserting a 0 and the remainder. The output is thus not necessarily a multiple of 32. +// +// TODO (litt3): usage of this function should be migrated to use PadPayload instead. I've left it unchanged for now, +// since v1 logic and tests rely on the specific assumptions of this implementation. func ConvertByPaddingEmptyByte(data []byte) []byte { dataSize := len(data) parseSize := encoding.BYTES_PER_SYMBOL - 1 @@ -41,6 +47,9 @@ func ConvertByPaddingEmptyByte(data []byte) []byte { // The function does not assume the input is a multiple of BYTES_PER_SYMBOL(32 bytes). // For the reminder of the input, the first byte is taken out, and the rest is appended to // the output. +// +// TODO (litt3): usage of this function should be migrated to use RemoveInternalPadding instead. I've left it unchanged +// for now, since v1 logic and tests rely on the specific assumptions of this implementation. func RemoveEmptyByteFromPaddedBytes(data []byte) []byte { dataSize := len(data) parseSize := encoding.BYTES_PER_SYMBOL @@ -65,3 +74,128 @@ func RemoveEmptyByteFromPaddedBytes(data []byte) []byte { } return validData[:validLen] } + +// PadPayload internally pads the input data by prepending a 0x00 to each chunk of 31 bytes. This guarantees that +// the data will be a valid field element for the bn254 curve +// +// Additionally, this function will add necessary padding to align the output to 32 bytes +// +// NOTE: this method is a reimplementation of ConvertByPaddingEmptyByte, with one meaningful difference: the alignment +// of the output to encoding.BYTES_PER_SYMBOL. This alignment actually makes the padding logic simpler, and the +// code that uses this function needs an aligned output anyway. +func PadPayload(inputData []byte) []byte { + // 31 bytes, for the bn254 curve + bytesPerChunk := uint32(encoding.BYTES_PER_SYMBOL - 1) + + // this is the length of the output, which is aligned to 32 bytes + outputLength := GetPaddedDataLength(uint32(len(inputData))) + paddedOutput := make([]byte, outputLength) + + // pre-pad the input, so that it aligns to 31 bytes. This means that the internally padded result will automatically + // align to 32 bytes. Doing this padding in advance simplifies the for loop. + requiredPad := (bytesPerChunk - uint32(len(inputData))%bytesPerChunk) % bytesPerChunk + prePaddedPayload := append(inputData, make([]byte, requiredPad)...) + + for element := uint32(0); element < outputLength/encoding.BYTES_PER_SYMBOL; element++ { + // add the 0x00 internal padding to guarantee that the data is in the valid range + zeroByteIndex := element * encoding.BYTES_PER_SYMBOL + paddedOutput[zeroByteIndex] = 0x00 + + destIndex := zeroByteIndex + 1 + srcIndex := element * bytesPerChunk + + // copy 31 bytes of data from the payload to the padded output + copy(paddedOutput[destIndex:destIndex+bytesPerChunk], prePaddedPayload[srcIndex:srcIndex+bytesPerChunk]) + } + + return paddedOutput +} + +// RemoveInternalPadding accepts an array of padded data, and removes the internal padding that was added in PadPayload +// +// This function assumes that the input aligns to 32 bytes. Since it is removing 1 byte for every 31 bytes kept, the +// output from this function is not guaranteed to align to 32 bytes. +// +// NOTE: this method is a reimplementation of RemoveEmptyByteFromPaddedBytes, with one meaningful difference: this +// function relies on the assumption that the input is aligned to encoding.BYTES_PER_SYMBOL, which makes the padding +// removal logic simpler. +func RemoveInternalPadding(paddedData []byte) ([]byte, error) { + if len(paddedData)%encoding.BYTES_PER_SYMBOL != 0 { + return nil, fmt.Errorf( + "padded data (length %d) must be multiple of encoding.BYTES_PER_SYMBOL %d", + len(paddedData), + encoding.BYTES_PER_SYMBOL) + } + + bytesPerChunk := encoding.BYTES_PER_SYMBOL - 1 + + symbolCount := len(paddedData) / encoding.BYTES_PER_SYMBOL + outputLength := symbolCount * bytesPerChunk + + outputData := make([]byte, outputLength) + + for i := 0; i < symbolCount; i++ { + dstIndex := i * bytesPerChunk + srcIndex := i*encoding.BYTES_PER_SYMBOL + 1 + + copy(outputData[dstIndex:dstIndex+bytesPerChunk], paddedData[srcIndex:srcIndex+bytesPerChunk]) + } + + return outputData, nil +} + +// GetPaddedDataLength accepts the length of a byte array, and returns the length that the array would be after +// adding internal byte padding +// +// The value returned from this function will always be a multiple of encoding.BYTES_PER_SYMBOL +func GetPaddedDataLength(inputLen uint32) uint32 { + bytesPerChunk := uint32(encoding.BYTES_PER_SYMBOL - 1) + chunkCount := inputLen / bytesPerChunk + + if inputLen%bytesPerChunk != 0 { + chunkCount++ + } + + return chunkCount * encoding.BYTES_PER_SYMBOL +} + +// GetUnpaddedDataLength accepts the length of an array that has been padded with PadPayload +// +// It returns what the length of the output array would be, if you called RemoveInternalPadding on it. +func GetUnpaddedDataLength(inputLen uint32) (uint32, error) { + if inputLen%encoding.BYTES_PER_SYMBOL != 0 { + return 0, fmt.Errorf( + "%d isn't a multiple of encoding.BYTES_PER_SYMBOL (%d)", + inputLen, encoding.BYTES_PER_SYMBOL) + } + + chunkCount := inputLen / encoding.BYTES_PER_SYMBOL + bytesPerChunk := uint32(encoding.BYTES_PER_SYMBOL - 1) + + unpaddedLength := chunkCount * bytesPerChunk + + return unpaddedLength, nil +} + +// GetMaxPermissiblePayloadLength accepts a blob length, and returns the size IN BYTES of the largest payload +// that could fit inside the blob. +func GetMaxPermissiblePayloadLength(blobLengthSymbols uint32) (uint32, error) { + if blobLengthSymbols == 0 { + return 0, fmt.Errorf("input blobLengthSymbols is zero") + } + + // TODO (litt3): it's awkward to use a method defined in fft for this, but it's not trivial to move to a better + // location, due to the resulting cyclic imports, and I'd prefer not to reimplement. Ideally, a proper location + // would be found for this important utility function + if !fft.IsPowerOfTwo(uint64(blobLengthSymbols)) { + return 0, fmt.Errorf("blobLengthSymbols %d is not a power of two", blobLengthSymbols) + } + + // subtract 32 from the blob length before doing the unpad operation, to account for the encoded payload header + maxPayloadLength, err := GetUnpaddedDataLength(blobLengthSymbols*encoding.BYTES_PER_SYMBOL - 32) + if err != nil { + return 0, fmt.Errorf("get unpadded data length: %w", err) + } + + return maxPayloadLength, nil +} diff --git a/encoding/utils/codec/codec_test.go b/encoding/utils/codec/codec_test.go index 3137d7fe7b..9af9c7ba84 100644 --- a/encoding/utils/codec/codec_test.go +++ b/encoding/utils/codec/codec_test.go @@ -4,6 +4,7 @@ import ( "crypto/rand" "testing" + "github.com/Layr-Labs/eigenda/common/testutils/random" "github.com/Layr-Labs/eigenda/encoding/rs" "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/stretchr/testify/require" @@ -49,3 +50,56 @@ func TestSimplePaddingCodec_Fuzz(t *testing.T) { } } } + +// TestGetPaddedDataLength tests that GetPaddedDataLength behaves relative to hardcoded expected results +func TestGetPaddedDataLengthAgainstKnowns(t *testing.T) { + startLengths := []uint32{0, 30, 31, 32, 33, 68} + expectedResults := []uint32{0, 32, 32, 64, 64, 96} + + for i := range startLengths { + require.Equal(t, codec.GetPaddedDataLength(startLengths[i]), expectedResults[i]) + } +} + +// TestGetUnpaddedDataLengthAgainstKnowns tests that GetPaddedDataLength behaves relative to hardcoded expected results +func TestGetUnpaddedDataLengthAgainstKnowns(t *testing.T) { + startLengths := []uint32{0, 32, 64, 128} + expectedResults := []uint32{0, 31, 62, 124} + + for i := range startLengths { + unpaddedDataLength, err := codec.GetUnpaddedDataLength(startLengths[i]) + require.Nil(t, err) + + require.Equal(t, expectedResults[i], unpaddedDataLength) + } + + unpaddedDataLength, err := codec.GetUnpaddedDataLength(129) + require.Error(t, err) + require.Equal(t, uint32(0), unpaddedDataLength) +} + +// TestPadUnpad makes sure that padding and unpadding doesn't corrupt underlying data +func TestPadUnpad(t *testing.T) { + testRandom := random.NewTestRandom() + testIterations := 1000 + + for i := 0; i < testIterations; i++ { + originalBytes := testRandom.Bytes(testRandom.Intn(1024)) + + paddedBytes := codec.PadPayload(originalBytes) + require.Equal(t, len(paddedBytes)%32, 0) + + unpaddedBytes, err := codec.RemoveInternalPadding(paddedBytes) + require.Nil(t, err) + + expectedUnpaddedLength, err := codec.GetUnpaddedDataLength(uint32(len(paddedBytes))) + require.Nil(t, err) + require.Equal(t, expectedUnpaddedLength, uint32(len(unpaddedBytes))) + + // unpadded payload may have up to 31 extra trailing zeros, since RemoveInternalPadding doesn't consider these + require.Greater(t, len(originalBytes), len(unpaddedBytes)-32) + require.LessOrEqual(t, len(originalBytes), len(unpaddedBytes)) + + require.Equal(t, originalBytes, unpaddedBytes[:len(originalBytes)]) + } +} diff --git a/encoding/utils_test.go b/encoding/utils_test.go new file mode 100644 index 0000000000..5c193509c9 --- /dev/null +++ b/encoding/utils_test.go @@ -0,0 +1,32 @@ +package encoding + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNextPowerOf2(t *testing.T) { + testHeight := 65536 + + // 2 ^ 16 = 65536 + // i.e., the last element generated here == testHeight + powers := GeneratePowersOfTwo(17) + + powerIndex := 0 + for i := 1; i <= testHeight; i++ { + nextPowerOf2 := NextPowerOf2(i) + require.Equal(t, nextPowerOf2, powers[powerIndex]) + + if i == powers[powerIndex] { + powerIndex++ + } + } + + // sanity check the test logic + require.Equal(t, powerIndex, len(powers)) + + // extra sanity check, since we *really* rely on NextPowerOf2 returning + // the same value, if it's already a power of 2 + require.Equal(t, 16, NextPowerOf2(16)) +} diff --git a/test/v2/client/test_client.go b/test/v2/client/test_client.go index ac230e21b1..70d774cbe3 100644 --- a/test/v2/client/test_client.go +++ b/test/v2/client/test_client.go @@ -201,7 +201,7 @@ func NewTestClient( payloadClientConfig := clients.GetDefaultPayloadClientConfig() payloadClientConfig.EigenDACertVerifierAddr = config.EigenDACertVerifierAddress - blobCodec, err := codecs.CreateCodec(codecs.PolynomialFormEval, codecs.DefaultBlobEncoding) + blobCodec, err := codecs.CreateCodec(codecs.PolynomialFormEval, codecs.PayloadEncodingVersion0) if err != nil { return nil, fmt.Errorf("failed to create blob codec: %w", err) } @@ -347,7 +347,7 @@ func (c *TestClient) GetPayloadDisperser(quorums []core.QuorumID) (*clients.Payl DisperseBlobTimeout: 1337 * time.Hour, // this suite enforces its own timeouts } - blobCodec, err := codecs.CreateCodec(codecs.PolynomialFormEval, payloadDisperserConfig.BlobEncodingVersion) + blobCodec, err := codecs.CreateCodec(codecs.PolynomialFormEval, payloadDisperserConfig.PayloadEncodingVersion) if err != nil { return nil, fmt.Errorf("failed to create blob codec: %w", err) }