Skip to content

Commit d915de4

Browse files
kostasdizasiliax
kostasdizas
and
iliax
authored
Add protobuf raw message deserializer (#4041)
Implemented a Protobuf Raw deserialiser that works like protoc --decode_raw. This is a no config alternative to the existing ProtobufFileSerde. Co-authored-by: Ilya Kuramshin <[email protected]>
1 parent 150fc21 commit d915de4

File tree

3 files changed

+169
-0
lines changed

3 files changed

+169
-0
lines changed

Diff for: ‎kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/SerdesInitializer.java

+2
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import com.provectus.kafka.ui.serdes.builtin.Int32Serde;
1717
import com.provectus.kafka.ui.serdes.builtin.Int64Serde;
1818
import com.provectus.kafka.ui.serdes.builtin.ProtobufFileSerde;
19+
import com.provectus.kafka.ui.serdes.builtin.ProtobufRawSerde;
1920
import com.provectus.kafka.ui.serdes.builtin.StringSerde;
2021
import com.provectus.kafka.ui.serdes.builtin.UInt32Serde;
2122
import com.provectus.kafka.ui.serdes.builtin.UInt64Serde;
@@ -50,6 +51,7 @@ public SerdesInitializer() {
5051
.put(Base64Serde.name(), Base64Serde.class)
5152
.put(HexSerde.name(), HexSerde.class)
5253
.put(UuidBinarySerde.name(), UuidBinarySerde.class)
54+
.put(ProtobufRawSerde.name(), ProtobufRawSerde.class)
5355
.build(),
5456
new CustomSerdeLoader()
5557
);

Diff for: ‎kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/builtin/ProtobufRawSerde.java

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package com.provectus.kafka.ui.serdes.builtin;
2+
3+
import com.google.protobuf.UnknownFieldSet;
4+
import com.provectus.kafka.ui.exception.ValidationException;
5+
import com.provectus.kafka.ui.serde.api.DeserializeResult;
6+
import com.provectus.kafka.ui.serde.api.RecordHeaders;
7+
import com.provectus.kafka.ui.serde.api.SchemaDescription;
8+
import com.provectus.kafka.ui.serdes.BuiltInSerde;
9+
import java.util.Map;
10+
import java.util.Optional;
11+
import lombok.SneakyThrows;
12+
13+
public class ProtobufRawSerde implements BuiltInSerde {
14+
15+
public static String name() {
16+
return "ProtobufDecodeRaw";
17+
}
18+
19+
@Override
20+
public Optional<String> getDescription() {
21+
return Optional.empty();
22+
}
23+
24+
@Override
25+
public Optional<SchemaDescription> getSchema(String topic, Target type) {
26+
return Optional.empty();
27+
}
28+
29+
@Override
30+
public boolean canSerialize(String topic, Target type) {
31+
return false;
32+
}
33+
34+
@Override
35+
public boolean canDeserialize(String topic, Target type) {
36+
return true;
37+
}
38+
39+
@Override
40+
public Serializer serializer(String topic, Target type) {
41+
throw new UnsupportedOperationException();
42+
}
43+
44+
@Override
45+
public Deserializer deserializer(String topic, Target type) {
46+
return new Deserializer() {
47+
@SneakyThrows
48+
@Override
49+
public DeserializeResult deserialize(RecordHeaders headers, byte[] data) {
50+
try {
51+
UnknownFieldSet unknownFields = UnknownFieldSet.parseFrom(data);
52+
return new DeserializeResult(unknownFields.toString(), DeserializeResult.Type.STRING, Map.of());
53+
} catch (Exception e) {
54+
throw new ValidationException(e.getMessage());
55+
}
56+
}
57+
};
58+
}
59+
}

Diff for: ‎kafka-ui-api/src/test/java/com/provectus/kafka/ui/serdes/builtin/ProtobufRawSerdeTest.java

+108
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
package com.provectus.kafka.ui.serdes.builtin;
2+
3+
import static org.assertj.core.api.Assertions.assertThat;
4+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
5+
6+
import com.google.protobuf.DescriptorProtos;
7+
import com.google.protobuf.Descriptors;
8+
import com.google.protobuf.DynamicMessage;
9+
import com.provectus.kafka.ui.exception.ValidationException;
10+
import com.provectus.kafka.ui.serde.api.Serde;
11+
import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema;
12+
import lombok.SneakyThrows;
13+
import org.junit.jupiter.api.BeforeEach;
14+
import org.junit.jupiter.api.Test;
15+
16+
class ProtobufRawSerdeTest {
17+
18+
private static final String DUMMY_TOPIC = "dummy-topic";
19+
20+
private ProtobufRawSerde serde;
21+
22+
@BeforeEach
23+
void init() {
24+
serde = new ProtobufRawSerde();
25+
}
26+
27+
@SneakyThrows
28+
ProtobufSchema getSampleSchema() {
29+
return new ProtobufSchema(
30+
"""
31+
syntax = "proto3";
32+
message Message1 {
33+
int32 my_field = 1;
34+
}
35+
"""
36+
);
37+
}
38+
39+
@SneakyThrows
40+
private byte[] getProtobufMessage() {
41+
DynamicMessage.Builder builder = DynamicMessage.newBuilder(getSampleSchema().toDescriptor("Message1"));
42+
builder.setField(builder.getDescriptorForType().findFieldByName("my_field"), 5);
43+
return builder.build().toByteArray();
44+
}
45+
46+
@Test
47+
void deserializeSimpleMessage() {
48+
var deserialized = serde.deserializer(DUMMY_TOPIC, Serde.Target.VALUE)
49+
.deserialize(null, getProtobufMessage());
50+
assertThat(deserialized.getResult()).isEqualTo("1: 5\n");
51+
}
52+
53+
@Test
54+
void deserializeEmptyMessage() {
55+
var deserialized = serde.deserializer(DUMMY_TOPIC, Serde.Target.VALUE)
56+
.deserialize(null, new byte[0]);
57+
assertThat(deserialized.getResult()).isEqualTo("");
58+
}
59+
60+
@Test
61+
void deserializeInvalidMessage() {
62+
var deserializer = serde.deserializer(DUMMY_TOPIC, Serde.Target.VALUE);
63+
assertThatThrownBy(() -> deserializer.deserialize(null, new byte[] { 1, 2, 3 }))
64+
.isInstanceOf(ValidationException.class)
65+
.hasMessageContaining("Protocol message contained an invalid tag");
66+
}
67+
68+
@Test
69+
void deserializeNullMessage() {
70+
var deserializer = serde.deserializer(DUMMY_TOPIC, Serde.Target.VALUE);
71+
assertThatThrownBy(() -> deserializer.deserialize(null, null))
72+
.isInstanceOf(ValidationException.class)
73+
.hasMessageContaining("Cannot read the array length");
74+
}
75+
76+
ProtobufSchema getSampleNestedSchema() {
77+
return new ProtobufSchema(
78+
"""
79+
syntax = "proto3";
80+
message Message2 {
81+
int32 my_nested_field = 1;
82+
}
83+
message Message1 {
84+
int32 my_field = 1;
85+
Message2 my_nested_message = 2;
86+
}
87+
"""
88+
);
89+
}
90+
91+
@SneakyThrows
92+
private byte[] getComplexProtobufMessage() {
93+
DynamicMessage.Builder builder = DynamicMessage.newBuilder(getSampleNestedSchema().toDescriptor("Message1"));
94+
builder.setField(builder.getDescriptorForType().findFieldByName("my_field"), 5);
95+
DynamicMessage.Builder nestedBuilder = DynamicMessage.newBuilder(getSampleNestedSchema().toDescriptor("Message2"));
96+
nestedBuilder.setField(nestedBuilder.getDescriptorForType().findFieldByName("my_nested_field"), 10);
97+
builder.setField(builder.getDescriptorForType().findFieldByName("my_nested_message"), nestedBuilder.build());
98+
99+
return builder.build().toByteArray();
100+
}
101+
102+
@Test
103+
void deserializeNestedMessage() {
104+
var deserialized = serde.deserializer(DUMMY_TOPIC, Serde.Target.VALUE)
105+
.deserialize(null, getComplexProtobufMessage());
106+
assertThat(deserialized.getResult()).isEqualTo("1: 5\n2: {\n 1: 10\n}\n");
107+
}
108+
}

0 commit comments

Comments
 (0)