From 1785c5134a8303e03c9725f7fb518bd155e2f31d Mon Sep 17 00:00:00 2001
From: xiaozhikang <xiaozhikang0916@gmail.com>
Date: Sun, 17 Nov 2024 20:12:26 +0800
Subject: [PATCH 1/6] introduce ProtoUnknownFields

---
 .../protobuf/ProtoUnknownFields.kt            | 131 ++++++++++++++++++
 .../protobuf/internal/Helpers.kt              |  24 +++-
 .../internal/ProtoMessageSerializer.kt        |  90 ++++++++++++
 .../protobuf/internal/ProtobufDecoding.kt     |  60 ++++++--
 .../protobuf/internal/ProtobufEncoding.kt     |  33 +++++
 .../protobuf/internal/ProtobufReader.kt       |  10 ++
 .../protobuf/internal/ProtobufTaggedBase.kt   |   4 +-
 .../protobuf/ProtobufUnknownFieldsTest.kt     | 112 +++++++++++++++
 8 files changed, 446 insertions(+), 18 deletions(-)
 create mode 100644 formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoUnknownFields.kt
 create mode 100644 formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtoMessageSerializer.kt
 create mode 100644 formats/protobuf/commonTest/src/kotlinx/serialization/protobuf/ProtobufUnknownFieldsTest.kt

diff --git a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoUnknownFields.kt b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoUnknownFields.kt
new file mode 100644
index 0000000000..cba50d0383
--- /dev/null
+++ b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoUnknownFields.kt
@@ -0,0 +1,131 @@
+/*
+ * Copyright 2017-2024 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package kotlinx.serialization.protobuf
+
+import kotlinx.serialization.*
+import kotlinx.serialization.protobuf.internal.*
+import kotlinx.serialization.protobuf.internal.ProtoWireType
+
+/**
+ * Mark a property as a holder for unknown fields in protobuf message.
+ */
+@SerialInfo
+@Target(AnnotationTarget.PROPERTY)
+@ExperimentalSerializationApi
+public annotation class ProtoUnknownFields
+
+@Serializable(with = ProtoMessageSerializer::class)
+public class ProtoMessage internal constructor(
+    public val fields: List<ProtoField>
+) {
+    public companion object {
+        public val Empty: ProtoMessage = ProtoMessage(emptyList())
+    }
+
+    public val size: Int get() = fields.size
+    public fun asByteArray(): ByteArray = fields.fold(ByteArray(0)) { acc, protoField -> acc + protoField.asWireContent() }
+
+    public constructor(vararg fields: ProtoField) : this(fields.toList())
+    public fun merge(other: ProtoMessage): ProtoMessage {
+        return ProtoMessage(fields + other.fields)
+    }
+
+    public fun merge(vararg field: ProtoField): ProtoMessage {
+        return ProtoMessage(fields + field)
+    }
+
+    override fun hashCode(): Int {
+        return fields.hashCode()
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as ProtoMessage
+
+        return fields == other.fields
+    }
+}
+
+@OptIn(ExperimentalSerializationApi::class)
+@Serializable(with = ProtoFieldSerializer::class)
+@KeepGeneratedSerializer
+@ConsistentCopyVisibility
+public data class ProtoField internal constructor(
+    internal val id: Int,
+    internal val wireType: ProtoWireType,
+    internal val data: ProtoContentHolder
+) {
+    public companion object {
+        public val Empty: ProtoField = ProtoField(0, ProtoWireType.INVALID, ProtoContentHolder.ByteArrayContent(ByteArray(0)))
+    }
+
+    public fun asWireContent(): ByteArray = byteArrayOf(((id shl 3) or wireType.typeId).toByte()) + data.byteArray
+
+    public val contentLength: Int
+        get() = asWireContent().size
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as ProtoField
+
+        if (id != other.id) return false
+        if (wireType != other.wireType) return false
+        if (!data.contentEquals(other.data)) return false
+
+        return true
+    }
+
+    override fun hashCode(): Int {
+        var result = id
+        result = 31 * result + wireType.hashCode()
+        result = 31 * result + data.contentHashCode()
+        return result
+    }
+}
+
+internal sealed interface ProtoContentHolder {
+    val byteArray: ByteArray
+
+    data class ByteArrayContent(override val byteArray: ByteArray) : ProtoContentHolder {
+        override fun equals(other: Any?): Boolean {
+            return other is ProtoContentHolder && this.contentEquals(other)
+        }
+
+        override fun hashCode(): Int {
+            return this.contentHashCode()
+        }
+    }
+
+    data class MessageContent(val content: ProtoMessage) : ProtoContentHolder {
+        override val byteArray: ByteArray
+            get() = content.asByteArray()
+
+        override fun equals(other: Any?): Boolean {
+            return other is ProtoContentHolder && this.contentEquals(other)
+        }
+
+        override fun hashCode(): Int {
+            return this.contentHashCode()
+        }
+    }
+}
+
+internal fun ProtoContentHolder(content: ByteArray): ProtoContentHolder = ProtoContentHolder.ByteArrayContent(content)
+
+internal val ProtoContentHolder.contentLength: Int
+    get() = byteArray.size
+
+internal fun ProtoContentHolder.contentEquals(other: ProtoContentHolder): Boolean {
+    return byteArray.contentEquals(other.byteArray)
+}
+
+internal fun ProtoContentHolder.contentHashCode(): Int {
+    return byteArray.contentHashCode()
+}
+
diff --git a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/Helpers.kt b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/Helpers.kt
index ea6d4b6823..fed5536755 100644
--- a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/Helpers.kt
+++ b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/Helpers.kt
@@ -37,16 +37,25 @@ internal enum class ProtoWireType(val typeId: Int) {
 }
 
 internal const val ID_HOLDER_ONE_OF = -2
+internal const val ID_HOLDER_UNKNOWN_FIELDS = -3
 
+private const val UNKNOWN_FIELD_MASK = 1L shl 37
 private const val ONEOFMASK = 1L shl 36
 private const val INTTYPEMASK = 3L shl 33
 private const val PACKEDMASK = 1L shl 32
 
 @Suppress("NOTHING_TO_INLINE")
-internal inline fun ProtoDesc(protoId: Int, type: ProtoIntegerType, packed: Boolean = false, oneOf: Boolean = false): ProtoDesc {
+internal inline fun ProtoDesc(
+    protoId: Int,
+    type: ProtoIntegerType,
+    packed: Boolean = false,
+    oneOf: Boolean = false,
+    unknown: Boolean = false,
+): ProtoDesc {
     val packedBits = if (packed) PACKEDMASK else 0L
     val oneOfBits = if (oneOf) ONEOFMASK else 0L
-    return packedBits or oneOfBits or type.signature or protoId.toLong()
+    val unknownBits = if (unknown) UNKNOWN_FIELD_MASK else 0L
+    return packedBits or oneOfBits or type.signature or protoId.toLong() or unknownBits
 }
 
 internal inline val ProtoDesc.protoId: Int get() = (this and Int.MAX_VALUE.toLong()).toInt()
@@ -72,6 +81,9 @@ internal val ProtoDesc.isPacked: Boolean
 internal val ProtoDesc.isOneOf: Boolean
     get() = (this and ONEOFMASK) != 0L
 
+internal val ProtoDesc.isUnknown: Boolean
+    get() = (this and UNKNOWN_FIELD_MASK) != 0L
+
 internal fun ProtoDesc.overrideId(protoId: Int): ProtoDesc {
     return this and (0xFFFFFFF00000000L) or protoId.toLong()
 }
@@ -82,6 +94,7 @@ internal fun SerialDescriptor.extractParameters(index: Int): ProtoDesc {
     var format: ProtoIntegerType = ProtoIntegerType.DEFAULT
     var protoPacked = false
     var isOneOf = false
+    var isUnknown = false
 
     for (i in annotations.indices) { // Allocation-friendly loop
         val annotation = annotations[i]
@@ -94,6 +107,8 @@ internal fun SerialDescriptor.extractParameters(index: Int): ProtoDesc {
             protoPacked = true
         } else if (annotation is ProtoOneOf) {
             isOneOf = true
+        } else if (annotation is ProtoUnknownFields) {
+            isUnknown = true
         }
     }
     if (isOneOf) {
@@ -102,7 +117,7 @@ internal fun SerialDescriptor.extractParameters(index: Int): ProtoDesc {
         // See [kotlinx.serialization.protobuf.internal.ProtobufDecoder.decodeElementIndex] for detail
         protoId = index + 1
     }
-    return ProtoDesc(protoId, format, protoPacked, isOneOf)
+    return ProtoDesc(protoId, format, protoPacked, isOneOf, isUnknown)
 }
 
 /**
@@ -117,6 +132,9 @@ internal fun extractProtoId(descriptor: SerialDescriptor, index: Int, zeroBasedD
         if (annotation is ProtoOneOf) {
             // Fast return for one of field
             return ID_HOLDER_ONE_OF
+        } else if (annotation is ProtoUnknownFields) {
+            // Fast return for unknown fields holder
+            return ID_HOLDER_UNKNOWN_FIELDS
         } else if (annotation is ProtoNumber) {
             result = annotation.number
             // 0 or negative numbers are acceptable for enums
diff --git a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtoMessageSerializer.kt b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtoMessageSerializer.kt
new file mode 100644
index 0000000000..5fa4d8faec
--- /dev/null
+++ b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtoMessageSerializer.kt
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2017-2024 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package kotlinx.serialization.protobuf.internal
+
+import kotlinx.serialization.*
+import kotlinx.serialization.builtins.*
+import kotlinx.serialization.descriptors.*
+import kotlinx.serialization.encoding.*
+import kotlinx.serialization.protobuf.*
+
+internal object ProtoMessageSerializer : KSerializer<ProtoMessage> {
+    internal val fieldsSerializer = ProtoFieldSerializer
+
+    override val descriptor: SerialDescriptor
+        get() = UnknownFieldsDescriptor(fieldsSerializer.descriptor)
+
+    override fun deserialize(decoder: Decoder): ProtoMessage {
+        if (decoder is ProtobufDecoder) {
+            return decoder.decodeStructure(descriptor) {
+                ProtoMessage(fieldsSerializer.deserializeComposite(this))
+            }
+        }
+        return ProtoMessage.Empty
+    }
+
+    override fun serialize(encoder: Encoder, value: ProtoMessage) {
+        if (encoder is ProtobufEncoder) {
+            value.fields.forEach {
+                fieldsSerializer.serialize(encoder, it)
+            }
+        }
+    }
+}
+
+internal object ProtoFieldSerializer : KSerializer<ProtoField> {
+    private val delegate = ByteArraySerializer()
+
+    override val descriptor: SerialDescriptor
+        get() = UnknownFieldsDescriptor(delegate.descriptor)
+
+    fun deserializeComposite(compositeDecoder: CompositeDecoder): ProtoField {
+        if (compositeDecoder is ProtobufDecoder) {
+            return deserialize(compositeDecoder)
+        }
+        return ProtoField.Empty
+    }
+
+    override fun deserialize(decoder: Decoder): ProtoField {
+        if (decoder is ProtobufDecoder) {
+            return deserialize(decoder, decoder.currentTag)
+        }
+        return ProtoField.Empty
+    }
+
+    internal fun deserialize(protobufDecoder: ProtobufDecoder, currentTag: ProtoDesc): ProtoField {
+        if (currentTag != MISSING_TAG) {
+            val id = currentTag.protoId
+            val type = protobufDecoder.currentType
+            val data = protobufDecoder.decodeRawElement()
+            val field = ProtoField(
+                id = id,
+                wireType = type,
+                data = ProtoContentHolder(data),
+            )
+            return field
+        }
+        return ProtoField.Empty
+    }
+
+    override fun serialize(encoder: Encoder, value: ProtoField) {
+        if (encoder is ProtobufEncoder) {
+            encoder.encodeRawElement(value.id, value.wireType, value.data.byteArray)
+        }
+    }
+}
+
+internal class UnknownFieldsDescriptor(private val original: SerialDescriptor) : SerialDescriptor by original {
+    override val serialName: String
+        get() = "UnknownProtoFieldsHolder"
+
+    override fun equals(other: Any?): Boolean {
+        return other is UnknownFieldsDescriptor && other.original == original
+    }
+
+    override fun hashCode(): Int {
+        return original.hashCode()
+    }
+}
diff --git a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt
index 56884b12a7..d9740a5d03 100644
--- a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt
+++ b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt
@@ -31,9 +31,14 @@ internal open class ProtobufDecoder(
     // Index -> proto id for oneof element. An oneof element of certain index may refer to different proto id in runtime.
     private var index2IdMap: MutableMap<Int, Int>? = null
 
+    private var unknownHolderIndex: Int = -1
+
     private var nullValue: Boolean = false
     private val elementMarker = ElementMarker(descriptor, ::readIfAbsent)
 
+    internal val currentType: ProtoWireType
+        get() = reader.currentType
+
     init {
         populateCache(descriptor)
     }
@@ -52,10 +57,10 @@ internal open class ProtobufDecoder(
             val cache = IntArray(elements + 1) { -1 }
             for (i in 0 until elements) {
                 val protoId = extractProtoId(descriptor, i, false)
-                // If any element is marked as ProtoOneOf,
+                // If any element is marked as ProtoOneOf on Unknown field holder,
                 // the fast path is not applicable
-                // because it will contain more id than elements
-                if (protoId <= elements && protoId != ID_HOLDER_ONE_OF) {
+                // because num of id does not match the elements
+                if (protoId in 0..elements) {
                     cache[protoId] = i
                 } else {
                     return populateCacheMap(descriptor, elements)
@@ -72,14 +77,24 @@ internal open class ProtobufDecoder(
         var oneOfCount = 0
         for (i in 0 until elements) {
             val id = extractProtoId(descriptor, i, false)
-            if (id == ID_HOLDER_ONE_OF) {
-                descriptor.getElementDescriptor(i)
-                    .getAllOneOfSerializerOfField(serializersModule)
-                    .map { it.extractParameters(0).protoId }
-                    .forEach { map.putProtoId(it, i) }
-                oneOfCount ++
-            } else {
-                map.putProtoId(extractProtoId(descriptor, i, false),  i)
+            when (id) {
+                ID_HOLDER_ONE_OF -> {
+                    descriptor.getElementDescriptor(i)
+                        .getAllOneOfSerializerOfField(serializersModule)
+                        .map { it.extractParameters(0).protoId }
+                        .forEach { map.putProtoId(it, i) }
+                    oneOfCount ++
+                }
+                ID_HOLDER_UNKNOWN_FIELDS -> {
+                    require(unknownHolderIndex == -1) {
+                        "Only one unknown fields holder is allowed in a message"
+                    }
+                    oneOfCount ++
+                    unknownHolderIndex = i
+                }
+                else -> {
+                    map.putProtoId(id, i)
+                }
             }
         }
         if (oneOfCount > 0) {
@@ -102,7 +117,7 @@ internal open class ProtobufDecoder(
 
     private fun getIndexByNumSlowPath(
         protoTag: Int
-    ): Int = sparseIndexCache!!.getOrElse(protoTag) { -1 }
+    ): Int = sparseIndexCache!!.getOrElse(protoTag) { unknownHolderIndex }
 
     private fun findIndexByTag(descriptor: SerialDescriptor, protoTag: Int): Int {
         // Fast-path: tags are incremental, 1-based
@@ -252,6 +267,9 @@ internal open class ProtobufDecoder(
             deserializer.descriptor == ByteArraySerializer().descriptor -> deserializeByteArray(previousValue as ByteArray?) as T
             deserializer is AbstractCollectionSerializer<*, *, *> ->
                 (deserializer as AbstractCollectionSerializer<*, T, *>).merge(this, previousValue)
+            deserializer == ProtoMessageSerializer -> {
+                decodeUnknownFields(previousValue as? ProtoMessage) as T
+            }
 
             else -> deserializer.deserialize(this)
         }
@@ -317,7 +335,8 @@ internal open class ProtobufDecoder(
                 if (index == -1) { // not found
                     reader.skipElement()
                 } else {
-                    if (descriptor.extractParameters(index).isOneOf) {
+                    val tag = descriptor.extractParameters(index)
+                    if (tag.isOneOf || tag.isUnknown) {
                         /**
                          * While decoding message with one-of field,
                          * the proto id read from wire data cannot be easily found
@@ -356,6 +375,21 @@ internal open class ProtobufDecoder(
         return false
     }
 
+    private fun decodeUnknownFields(previous: ProtoMessage?): ProtoMessage {
+        require(currentTagOrDefault != MISSING_TAG) {
+            "Cannot deserialize directly to kotlinx.serialization.protobuf.ProtoMessage."
+        }
+        val serializer = ProtoFieldSerializer
+        val restoredTag = index2IdMap?.get(unknownHolderIndex)?.let { currentTag.overrideId(it) } ?: currentTag
+        return serializer.deserialize(this, restoredTag).let {
+            previous?.merge(it) ?: ProtoMessage(it)
+        }
+    }
+
+    internal fun decodeRawElement(): ByteArray {
+        return reader.readRawElement()
+    }
+
     private inline fun <T> decodeOrThrow(tag: ProtoDesc, action: (tag: ProtoDesc) -> T): T {
         try {
             return action(tag)
diff --git a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufEncoding.kt b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufEncoding.kt
index b7d5dd28e5..edacc6f295 100644
--- a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufEncoding.kt
+++ b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufEncoding.kt
@@ -142,10 +142,36 @@ internal open class ProtobufEncoder(
         serializer is MapLikeSerializer<*, *, *, *> -> {
             serializeMap(serializer as SerializationStrategy<T>, value)
         }
+        serializer == ProtoMessageSerializer -> {
+            serializeUnknownFields(serializer as ProtoMessageSerializer, value as ProtoMessage)
+        }
         serializer.descriptor == ByteArraySerializer().descriptor -> serializeByteArray(value as ByteArray)
         else -> serializer.serialize(this, value)
     }
 
+    internal fun encodeRawElement(id: Int, wireType: ProtoWireType, data: ByteArray) {
+        when(wireType) {
+            ProtoWireType.INVALID -> {}
+            ProtoWireType.VARINT -> writer.writeInt(
+                value = data.first().toInt(),
+                tag = id,
+                format = ProtoIntegerType.DEFAULT
+            )
+
+            ProtoWireType.i64 -> writer.writeLong(
+                value = data.first().toLong(),
+                tag = id,
+                format = ProtoIntegerType.FIXED
+            )
+            ProtoWireType.SIZE_DELIMITED -> writer.writeBytes(data, id)
+            ProtoWireType.i32 -> writer.writeInt(
+                value = data.first().toInt(),
+                tag = id,
+                format = ProtoIntegerType.FIXED
+            )
+        }
+    }
+
     private fun serializeByteArray(value: ByteArray) {
         val tag = popTagOrDefault()
         if (tag == MISSING_TAG) {
@@ -155,6 +181,13 @@ internal open class ProtobufEncoder(
         }
     }
 
+    private fun serializeUnknownFields(serializer: SerializationStrategy<ProtoMessage>, protoMessage: ProtoMessage) {
+        require(currentTagOrDefault != MISSING_TAG) {
+            "Cannot serialize directly from kotlinx.serialization.protobuf.ProtoMessage."
+        }
+        serializer.serialize(this, protoMessage)
+    }
+
     @Suppress("UNCHECKED_CAST")
     private fun <T> serializeMap(serializer: SerializationStrategy<T>, value: T) {
         // encode maps as collection of map entries, not merged collection of key-values
diff --git a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufReader.kt b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufReader.kt
index 5b8ce1c292..f8b6a5ea45 100644
--- a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufReader.kt
+++ b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufReader.kt
@@ -65,6 +65,16 @@ internal class ProtobufReader(private val input: ByteArrayInput) {
         }
     }
 
+    fun readRawElement(): ByteArray {
+        return when (currentType) {
+            ProtoWireType.VARINT -> byteArrayOf(readInt(ProtoIntegerType.DEFAULT).toByte())
+            ProtoWireType.i64 -> byteArrayOf(readLong(ProtoIntegerType.FIXED).toByte())
+            ProtoWireType.SIZE_DELIMITED -> readByteArray()
+            ProtoWireType.i32 -> byteArrayOf(readInt(ProtoIntegerType.FIXED).toByte())
+            else -> throw ProtobufDecodingException("Unsupported start group or end group wire type: $currentType")
+        }
+    }
+
     @Suppress("NOTHING_TO_INLINE")
     private inline fun assertWireType(expected: ProtoWireType) {
         if (currentType != expected) throw ProtobufDecodingException("Expected wire type $expected, but found $currentType")
diff --git a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufTaggedBase.kt b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufTaggedBase.kt
index ffa9f47940..f9282dc3cf 100644
--- a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufTaggedBase.kt
+++ b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufTaggedBase.kt
@@ -28,10 +28,10 @@ internal abstract class ProtobufTaggedBase {
     @JvmField
     protected var stackSize = -1
 
-    protected val currentTag: ProtoDesc
+    internal val currentTag: ProtoDesc
         get() = tagsStack[stackSize]
 
-    protected val currentTagOrDefault: ProtoDesc
+    internal val currentTagOrDefault: ProtoDesc
         get() = if (stackSize == -1) MISSING_TAG else tagsStack[stackSize]
 
     protected fun popTagOrDefault(): ProtoDesc = if (stackSize == -1) MISSING_TAG else tagsStack[stackSize--]
diff --git a/formats/protobuf/commonTest/src/kotlinx/serialization/protobuf/ProtobufUnknownFieldsTest.kt b/formats/protobuf/commonTest/src/kotlinx/serialization/protobuf/ProtobufUnknownFieldsTest.kt
new file mode 100644
index 0000000000..b8b9add349
--- /dev/null
+++ b/formats/protobuf/commonTest/src/kotlinx/serialization/protobuf/ProtobufUnknownFieldsTest.kt
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2017-2024 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package kotlinx.serialization.protobuf
+
+import kotlinx.serialization.*
+import kotlin.test.*
+
+class ProtobufUnknownFieldsTest {
+    @Serializable
+    data class InnerData(val name: String, val b: Int, val c: List<String>)
+    @Serializable
+    data class BuildData(val a: Int, val b: String, val c: ByteArray, val d: List<Int>, val e: InnerData) {
+        override fun equals(other: Any?): Boolean {
+            if (this === other) return true
+            if (other == null || this::class != other::class) return false
+
+            other as BuildData
+
+            if (a != other.a) return false
+            if (b != other.b) return false
+            if (!c.contentEquals(other.c)) return false
+            if (d != other.d) return false
+            if (e != other.e) return false
+
+            return true
+        }
+
+        override fun hashCode(): Int {
+            var result = a
+            result = 31 * result + b.hashCode()
+            result = 31 * result + c.contentHashCode()
+            result = 31 * result + d.hashCode()
+            result = 31 * result + e.hashCode()
+            return result
+        }
+
+    }
+
+    @Serializable
+    data class DataWithUnknownFields(
+        val a: Int,
+        @ProtoUnknownFields val unknownFields: ProtoMessage
+    )
+
+    @Test
+    fun testDecodeWithUnknownField() {
+        val data = BuildData(42, "42", byteArrayOf(42, 42, 42), listOf(42, 42, 42), InnerData("42", 42, listOf("42", "42", "42")))
+        val encoded = "082a120234321a032a2a2a202a202a202a2a120a023432102a1a0234321a0234321a023432"
+        val decoded = ProtoBuf.decodeFromHexString(DataWithUnknownFields.serializer(), encoded)
+        assertEquals(data.a, decoded.a)
+        assertEquals(6, decoded.unknownFields.size)
+
+        val encoded2 = ProtoBuf.encodeToHexString(DataWithUnknownFields.serializer(), decoded)
+        assertEquals(encoded, encoded2)
+        val data2 = ProtoBuf.decodeFromHexString(BuildData.serializer(), encoded2)
+        assertEquals(data, data2)
+    }
+
+    @Test
+    fun testCannotDecodeArbitraryMessage() {
+        assertFailsWith<IllegalArgumentException> {
+            ProtoBuf.decodeFromHexString(ProtoMessage.serializer(), "")
+        }
+    }
+
+    @Test
+    fun testCannotEncodeArbitraryMessage() {
+        assertFailsWith<IllegalArgumentException> {
+            ProtoBuf.encodeToHexString(ProtoMessage.serializer(), ProtoMessage.Empty)
+        }
+    }
+
+    @Serializable
+    data class DataWithMultipleUnknownFields(
+        val a: Int,
+        @ProtoUnknownFields val unknownFields: ProtoMessage,
+        @ProtoUnknownFields val unknownFields2: ProtoMessage
+    )
+
+    @Test
+    fun testOnlyOneUnknownFieldAllowed() {
+        val encoded = "082a120234321a032a2a2a202a202a202a2a120a023432102a1a0234321a0234321a023432"
+        assertFailsWith<IllegalArgumentException> {
+            ProtoBuf.decodeFromHexString(DataWithMultipleUnknownFields.serializer(), encoded)
+        }
+    }
+
+    @Serializable
+    data class DataWithStaggeredFields(
+        @ProtoNumber(2)
+        val b: String,
+        @ProtoUnknownFields val unknownFields: ProtoMessage,
+        @ProtoNumber(4)
+        val d: List<Int>
+    )
+
+    @Test
+    fun testUnknownFieldBeforeKnownField() {
+        val data = BuildData(42, "42", byteArrayOf(42, 42, 42), listOf(42, 42, 42), InnerData("42", 42, listOf("42", "42", "42")))
+        val hex = "082a120234321a032a2a2a202a202a202a2a120a023432102a1a0234321a0234321a023432"
+        val decoded = ProtoBuf.decodeFromHexString(DataWithStaggeredFields.serializer(), hex)
+        assertEquals(3, decoded.unknownFields.size)
+        assertEquals("42", decoded.b)
+        assertEquals(listOf(42, 42, 42), decoded.d)
+
+        val encoded = ProtoBuf.encodeToHexString(DataWithStaggeredFields.serializer(), decoded)
+        val decodeOrigin = ProtoBuf.decodeFromHexString(BuildData.serializer(), encoded)
+        assertEquals(data, decodeOrigin)
+    }
+}
\ No newline at end of file

From 7bdceee07a4cfdb36026ba35c022817cc951f940 Mon Sep 17 00:00:00 2001
From: xiaozhikang <xiaozhikang0916@gmail.com>
Date: Sun, 17 Nov 2024 22:11:37 +0800
Subject: [PATCH 2/6] add some test

---
 .../protobuf/ProtoUnknownFields.kt            |  18 +++
 .../protobuf/internal/ProtobufDecoding.kt     |   4 +-
 .../protobuf/ProtobufUnknownFieldsTest.kt     | 143 ++++++++++++++++++
 3 files changed, 162 insertions(+), 3 deletions(-)

diff --git a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoUnknownFields.kt b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoUnknownFields.kt
index cba50d0383..c8d2ae5ce6 100644
--- a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoUnknownFields.kt
+++ b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoUnknownFields.kt
@@ -28,6 +28,9 @@ public class ProtoMessage internal constructor(
     public fun asByteArray(): ByteArray = fields.fold(ByteArray(0)) { acc, protoField -> acc + protoField.asWireContent() }
 
     public constructor(vararg fields: ProtoField) : this(fields.toList())
+
+    public operator fun plus(other: ProtoMessage): ProtoMessage = merge(other)
+
     public fun merge(other: ProtoMessage): ProtoMessage {
         return ProtoMessage(fields + other.fields)
     }
@@ -50,6 +53,21 @@ public class ProtoMessage internal constructor(
     }
 }
 
+public fun ProtoMessage?.merge(other: ProtoMessage?): ProtoMessage {
+    return when {
+        this == null -> other ?: ProtoMessage.Empty
+        other == null -> this
+        else -> this + other
+    }
+}
+
+public fun ProtoMessage?.merge(vararg fields: ProtoField): ProtoMessage {
+    return when {
+        this == null -> ProtoMessage(fields.toList())
+        else -> this.merge(ProtoMessage(fields.toList()))
+    }
+}
+
 @OptIn(ExperimentalSerializationApi::class)
 @Serializable(with = ProtoFieldSerializer::class)
 @KeepGeneratedSerializer
diff --git a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt
index d9740a5d03..f4924e3e29 100644
--- a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt
+++ b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt
@@ -381,9 +381,7 @@ internal open class ProtobufDecoder(
         }
         val serializer = ProtoFieldSerializer
         val restoredTag = index2IdMap?.get(unknownHolderIndex)?.let { currentTag.overrideId(it) } ?: currentTag
-        return serializer.deserialize(this, restoredTag).let {
-            previous?.merge(it) ?: ProtoMessage(it)
-        }
+        return previous.merge(serializer.deserialize(this, restoredTag))
     }
 
     internal fun decodeRawElement(): ByteArray {
diff --git a/formats/protobuf/commonTest/src/kotlinx/serialization/protobuf/ProtobufUnknownFieldsTest.kt b/formats/protobuf/commonTest/src/kotlinx/serialization/protobuf/ProtobufUnknownFieldsTest.kt
index b8b9add349..2d171e13ba 100644
--- a/formats/protobuf/commonTest/src/kotlinx/serialization/protobuf/ProtobufUnknownFieldsTest.kt
+++ b/formats/protobuf/commonTest/src/kotlinx/serialization/protobuf/ProtobufUnknownFieldsTest.kt
@@ -5,6 +5,8 @@
 package kotlinx.serialization.protobuf
 
 import kotlinx.serialization.*
+import kotlinx.serialization.descriptors.*
+import kotlinx.serialization.encoding.*
 import kotlin.test.*
 
 class ProtobufUnknownFieldsTest {
@@ -47,6 +49,22 @@ class ProtobufUnknownFieldsTest {
     @Test
     fun testDecodeWithUnknownField() {
         val data = BuildData(42, "42", byteArrayOf(42, 42, 42), listOf(42, 42, 42), InnerData("42", 42, listOf("42", "42", "42")))
+
+        /**
+         * 1: 42
+         * 2: {"42"}
+         * 3: {"***"}
+         * 4: 42
+         * 4: 42
+         * 4: 42
+         * 5: {
+         *   1: {"42"}
+         *   2: 42
+         *   3: {"42"}
+         *   3: {"42"}
+         *   3: {"42"}
+         * }
+         */
         val encoded = "082a120234321a032a2a2a202a202a202a2a120a023432102a1a0234321a0234321a023432"
         val decoded = ProtoBuf.decodeFromHexString(DataWithUnknownFields.serializer(), encoded)
         assertEquals(data.a, decoded.a)
@@ -99,6 +117,23 @@ class ProtobufUnknownFieldsTest {
     @Test
     fun testUnknownFieldBeforeKnownField() {
         val data = BuildData(42, "42", byteArrayOf(42, 42, 42), listOf(42, 42, 42), InnerData("42", 42, listOf("42", "42", "42")))
+
+        /**
+         * 1: 42
+         * 2: {"42"}
+         * 3: {"***"}
+         * 4: 42
+         * 4: 42
+         * 4: 42
+         * 5: {
+         *   1: {"42"}
+         *   2: 42
+         *   3: {"42"}
+         *   3: {"42"}
+         *   3: {"42"}
+         * }
+         * }
+         */
         val hex = "082a120234321a032a2a2a202a202a202a2a120a023432102a1a0234321a0234321a023432"
         val decoded = ProtoBuf.decodeFromHexString(DataWithStaggeredFields.serializer(), hex)
         assertEquals(3, decoded.unknownFields.size)
@@ -106,7 +141,115 @@ class ProtobufUnknownFieldsTest {
         assertEquals(listOf(42, 42, 42), decoded.d)
 
         val encoded = ProtoBuf.encodeToHexString(DataWithStaggeredFields.serializer(), decoded)
+        /**
+         * fields are re-ordered but acceptable in protobuf wire data
+         *
+         * 2: {"42"}
+         * 1: 42
+         * 3: {"***"}
+         * 5: {
+         *   1: {"42"}
+         *   2: 42
+         *   3: {"42"}
+         *   3: {"42"}
+         *   3: {"42"}
+         * }
+         * 4: 42
+         * 4: 42
+         * 4: 42
+         */
+        assertEquals("12023432082a1a032a2a2a2a120a023432102a1a0234321a0234321a023432202a202a202a", encoded)
         val decodeOrigin = ProtoBuf.decodeFromHexString(BuildData.serializer(), encoded)
         assertEquals(data, decodeOrigin)
     }
+
+    @Serializable
+    data class TotalKnownData(@ProtoUnknownFields val fields: ProtoMessage = ProtoMessage.Empty)
+
+    @Serializable
+    data class NestedUnknownData(val a: Int, @ProtoNumber(5) val inner: TotalKnownData, @ProtoUnknownFields val unknown: ProtoMessage)
+
+    @Test
+    fun testDecodeNestedUnknownData() {
+        /**
+         * 1: 42
+         * 2: {"42"}
+         * 3: {"***"}
+         * 4: 42
+         * 4: 42
+         * 4: 42
+         * 5: {
+         *   1: {"42"}
+         *   2: 42
+         *   3: {"42"}
+         *   3: {"42"}
+         *   3: {"42"}
+         * }
+         */
+        val hex = "082a120234321a032a2a2a202a202a202a2a120a023432102a1a0234321a0234321a023432"
+        val decoded = ProtoBuf.decodeFromHexString(NestedUnknownData.serializer(), hex)
+        assertEquals(5, decoded.unknown.size)
+    }
+
+    object CustomSerializer: KSerializer<DataWithUnknownFields> {
+        override val descriptor: SerialDescriptor
+            get() = buildClassSerialDescriptor("CustomData") {
+                element<Int>("a", annotations = listOf(ProtoNumber(1)))
+                element<ProtoMessage>("unknownFields", annotations = listOf(ProtoUnknownFields()))
+            }
+
+        override fun deserialize(decoder: Decoder): DataWithUnknownFields {
+            var a = 0
+            var unknownFields = ProtoMessage.Empty
+            decoder.decodeStructure(descriptor) {
+                loop@ while (true) {
+                    when (val index = decodeElementIndex(descriptor)) {
+                        CompositeDecoder.DECODE_DONE -> break@loop
+                        0 -> a = decodeIntElement(descriptor, index)
+                        1 -> unknownFields += decodeSerializableElement(descriptor, index, ProtoMessage.serializer())
+                        else -> error("Unexpected index: $index")
+                    }
+                }
+            }
+            return DataWithUnknownFields(a, unknownFields)
+        }
+
+        override fun serialize(encoder: Encoder, value: DataWithUnknownFields) {
+            encoder.encodeStructure(descriptor) {
+                encodeIntElement(descriptor, 0, value.a)
+                encodeSerializableElement(descriptor, 1, ProtoMessage.serializer(), value.unknownFields)
+            }
+        }
+    }
+
+    @Test
+    fun testCustomSerializer() {
+        val data = BuildData(42, "42", byteArrayOf(42, 42, 42), listOf(42, 42, 42), InnerData("42", 42, listOf("42", "42", "42")))
+
+        /**
+         * 1: 42
+         * 2: {"42"}
+         * 3: {"***"}
+         * 4: 42
+         * 4: 42
+         * 4: 42
+         * 5: {
+         *   1: {"42"}
+         *   2: 42
+         *   3: {"42"}
+         *   3: {"42"}
+         *   3: {"42"}
+         * }
+         */
+        val encoded = "082a120234321a032a2a2a202a202a202a2a120a023432102a1a0234321a0234321a023432"
+        val decoded = ProtoBuf.decodeFromHexString(CustomSerializer, encoded)
+
+        assertEquals(data.a, decoded.a)
+        assertEquals(6, decoded.unknownFields.size)
+
+        val encoded2 = ProtoBuf.encodeToHexString(CustomSerializer, decoded)
+        assertEquals(encoded, encoded2)
+        val data2 = ProtoBuf.decodeFromHexString(BuildData.serializer(), encoded2)
+        assertEquals(data, data2)
+    }
 }
\ No newline at end of file

From cfcdddb4ccf5d79f8a00e20374e356773563b681 Mon Sep 17 00:00:00 2001
From: xiaozhikang <xiaozhikang0916@gmail.com>
Date: Sun, 17 Nov 2024 22:55:32 +0800
Subject: [PATCH 3/6] comments

---
 .../protobuf/ProtoUnknownFields.kt            | 85 ++++++++++++++++---
 .../protobuf/internal/ProtobufDecoding.kt     | 24 +++---
 2 files changed, 87 insertions(+), 22 deletions(-)

diff --git a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoUnknownFields.kt b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoUnknownFields.kt
index c8d2ae5ce6..64a86e2e8e 100644
--- a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoUnknownFields.kt
+++ b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoUnknownFields.kt
@@ -9,33 +9,62 @@ import kotlinx.serialization.protobuf.internal.*
 import kotlinx.serialization.protobuf.internal.ProtoWireType
 
 /**
- * Mark a property as a holder for unknown fields in protobuf message.
+ * Mark a property with type [ProtoMessage] as a holder for unknown fields in protobuf message.
+ *
+ * All the contents with unregistered proto number will be stored in this field.
  */
 @SerialInfo
 @Target(AnnotationTarget.PROPERTY)
 @ExperimentalSerializationApi
 public annotation class ProtoUnknownFields
 
+/**
+ * Represents a protobuf message.
+ *
+ * Especially used as a holder of unknown proto fields in an arbitrary protobuf message.
+ */
 @Serializable(with = ProtoMessageSerializer::class)
 public class ProtoMessage internal constructor(
-    public val fields: List<ProtoField>
+    internal val fields: List<ProtoField>
 ) {
     public companion object {
+        /**
+         * An empty [ProtoMessage] instance.
+         *
+         * Useful as a default value for [ProtoUnknownFields] properties.
+         */
         public val Empty: ProtoMessage = ProtoMessage(emptyList())
     }
 
+    /**
+     * Number of fields holding in the message.
+     */
     public val size: Int get() = fields.size
-    public fun asByteArray(): ByteArray = fields.fold(ByteArray(0)) { acc, protoField -> acc + protoField.asWireContent() }
 
-    public constructor(vararg fields: ProtoField) : this(fields.toList())
+    /**
+     * Returns a byte array representing of the message.
+     */
+    public fun asByteArray(): ByteArray =
+        fields.fold(ByteArray(0)) { acc, protoField -> acc + protoField.asWireContent() }
+
+    internal constructor(vararg fields: ProtoField) : this(fields.toList())
 
+    /**
+     * Merges two [ProtoMessage] instances.
+     */
     public operator fun plus(other: ProtoMessage): ProtoMessage = merge(other)
 
+    /**
+     * Merges two [ProtoMessage] instances.
+     */
     public fun merge(other: ProtoMessage): ProtoMessage {
         return ProtoMessage(fields + other.fields)
     }
 
-    public fun merge(vararg field: ProtoField): ProtoMessage {
+    /**
+     * Convenience method to merge multiple [ProtoField] with this message.
+     */
+    internal fun merge(vararg field: ProtoField): ProtoMessage {
         return ProtoMessage(fields + field)
     }
 
@@ -53,6 +82,9 @@ public class ProtoMessage internal constructor(
     }
 }
 
+/**
+ * Convenience method to merge two nullable [ProtoMessage] instances.
+ */
 public fun ProtoMessage?.merge(other: ProtoMessage?): ProtoMessage {
     return when {
         this == null -> other ?: ProtoMessage.Empty
@@ -61,29 +93,35 @@ public fun ProtoMessage?.merge(other: ProtoMessage?): ProtoMessage {
     }
 }
 
-public fun ProtoMessage?.merge(vararg fields: ProtoField): ProtoMessage {
+/**
+ * Convenience method to merge multiple [ProtoField] with a nullable [ProtoMessage].
+ */
+internal fun ProtoMessage?.merge(vararg fields: ProtoField): ProtoMessage {
     return when {
         this == null -> ProtoMessage(fields.toList())
         else -> this.merge(ProtoMessage(fields.toList()))
     }
 }
 
+/**
+ * Represents a single field in a protobuf message.
+ */
 @OptIn(ExperimentalSerializationApi::class)
 @Serializable(with = ProtoFieldSerializer::class)
 @KeepGeneratedSerializer
 @ConsistentCopyVisibility
-public data class ProtoField internal constructor(
+internal data class ProtoField internal constructor(
     internal val id: Int,
     internal val wireType: ProtoWireType,
     internal val data: ProtoContentHolder
 ) {
-    public companion object {
-        public val Empty: ProtoField = ProtoField(0, ProtoWireType.INVALID, ProtoContentHolder.ByteArrayContent(ByteArray(0)))
+    companion object {
+        val Empty: ProtoField = ProtoField(0, ProtoWireType.INVALID, ProtoContentHolder.ByteArrayContent(ByteArray(0)))
     }
 
-    public fun asWireContent(): ByteArray = byteArrayOf(((id shl 3) or wireType.typeId).toByte()) + data.byteArray
+    fun asWireContent(): ByteArray = byteArrayOf(((id shl 3) or wireType.typeId).toByte()) + data.byteArray
 
-    public val contentLength: Int
+    val contentLength: Int
         get() = asWireContent().size
 
     override fun equals(other: Any?): Boolean {
@@ -107,9 +145,19 @@ public data class ProtoField internal constructor(
     }
 }
 
+/**
+ * A data representation of a protobuf field in [ProtoField.data], without the field number and wire type.
+ */
 internal sealed interface ProtoContentHolder {
+
+    /**
+     * Returns a byte array representation of the content.
+     */
     val byteArray: ByteArray
 
+    /**
+     * Represents the content in raw byte array.
+     */
     data class ByteArrayContent(override val byteArray: ByteArray) : ProtoContentHolder {
         override fun equals(other: Any?): Boolean {
             return other is ProtoContentHolder && this.contentEquals(other)
@@ -120,6 +168,9 @@ internal sealed interface ProtoContentHolder {
         }
     }
 
+    /**
+     * Represents the content with a nested [ProtoMessage].
+     */
     data class MessageContent(val content: ProtoMessage) : ProtoContentHolder {
         override val byteArray: ByteArray
             get() = content.asByteArray()
@@ -134,15 +185,27 @@ internal sealed interface ProtoContentHolder {
     }
 }
 
+/**
+ * Creates a [ProtoContentHolder] instance with a byte array content.
+ */
 internal fun ProtoContentHolder(content: ByteArray): ProtoContentHolder = ProtoContentHolder.ByteArrayContent(content)
 
+/**
+ * Get the length in bytes of the content in the [ProtoContentHolder].
+ */
 internal val ProtoContentHolder.contentLength: Int
     get() = byteArray.size
 
+/**
+ * Checks if the content of two [ProtoContentHolder] instances are equal.
+ */
 internal fun ProtoContentHolder.contentEquals(other: ProtoContentHolder): Boolean {
     return byteArray.contentEquals(other.byteArray)
 }
 
+/**
+ * Calculates the hash code of the content in the [ProtoContentHolder].
+ */
 internal fun ProtoContentHolder.contentHashCode(): Int {
     return byteArray.contentHashCode()
 }
diff --git a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt
index f4924e3e29..61979e8f5f 100644
--- a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt
+++ b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt
@@ -28,7 +28,8 @@ internal open class ProtobufDecoder(
     private var indexCache: IntArray? = null
     private var sparseIndexCache: MutableMap<Int, Int>? = null
 
-    // Index -> proto id for oneof element. An oneof element of certain index may refer to different proto id in runtime.
+    // Index -> proto id for oneof element or unknown fields.
+    // These kind of elements in certain index may refer to different proto id in runtime.
     private var index2IdMap: MutableMap<Int, Int>? = null
 
     private var unknownHolderIndex: Int = -1
@@ -57,7 +58,7 @@ internal open class ProtobufDecoder(
             val cache = IntArray(elements + 1) { -1 }
             for (i in 0 until elements) {
                 val protoId = extractProtoId(descriptor, i, false)
-                // If any element is marked as ProtoOneOf on Unknown field holder,
+                // If any element is marked as ProtoOneOf or Unknown field holder,
                 // the fast path is not applicable
                 // because num of id does not match the elements
                 if (protoId in 0..elements) {
@@ -74,7 +75,7 @@ internal open class ProtobufDecoder(
 
     private fun populateCacheMap(descriptor: SerialDescriptor, elements: Int) {
         val map = HashMap<Int, Int>(elements, 1f)
-        var oneOfCount = 0
+        var mapSize = 0
         for (i in 0 until elements) {
             val id = extractProtoId(descriptor, i, false)
             when (id) {
@@ -83,13 +84,13 @@ internal open class ProtobufDecoder(
                         .getAllOneOfSerializerOfField(serializersModule)
                         .map { it.extractParameters(0).protoId }
                         .forEach { map.putProtoId(it, i) }
-                    oneOfCount ++
+                    mapSize ++
                 }
                 ID_HOLDER_UNKNOWN_FIELDS -> {
                     require(unknownHolderIndex == -1) {
                         "Only one unknown fields holder is allowed in a message"
                     }
-                    oneOfCount ++
+                    mapSize ++
                     unknownHolderIndex = i
                 }
                 else -> {
@@ -97,8 +98,8 @@ internal open class ProtobufDecoder(
                 }
             }
         }
-        if (oneOfCount > 0) {
-            index2IdMap = HashMap(oneOfCount, 1f)
+        if (mapSize > 0) {
+            index2IdMap = HashMap(mapSize, 1f)
         }
         sparseIndexCache = map
     }
@@ -338,12 +339,13 @@ internal open class ProtobufDecoder(
                     val tag = descriptor.extractParameters(index)
                     if (tag.isOneOf || tag.isUnknown) {
                         /**
-                         * While decoding message with one-of field,
+                         * While decoding message with one-of field or unknown fields,
                          * the proto id read from wire data cannot be easily found
                          * in the properties of this type,
-                         * So the index of this one-of property and the id read from the wire
-                         * are saved in this map, then restored in [beginStructure]
-                         * and passed to [OneOfPolymorphicReader] to get the actual deserializer.
+                         * So the index of this property and the id read from the wire
+                         * are saved in this map, then
+                         * 1. restored in [beginStructure] and passed to [OneOfPolymorphicReader] to get the actual deserializer, or
+                         * 2. restored in [decodeUnknownFields] to get the right proto id for the unknown fields.
                          */
                         index2IdMap?.put(index, protoId)
                     }

From 310c9c430bc7f849a729ad9c7313d1dea48ad44d Mon Sep 17 00:00:00 2001
From: xiaozhikang <xiaozhikang0916@gmail.com>
Date: Mon, 18 Nov 2024 11:26:23 +0800
Subject: [PATCH 4/6] docs

---
 docs/formats.md                               |  71 +++++++--
 docs/serialization-guide.md                   |   1 +
 ...{ProtoUnknownFields.kt => ProtoMessage.kt} |  13 --
 .../serialization/protobuf/ProtoTypes.kt      |  12 +-
 .../protobuf/internal/ProtobufDecoding.kt     |   3 +
 .../protobuf/internal/ProtobufEncoding.kt     |   3 +
 .../protobuf/ProtobufUnknownFieldsTest.kt     |  46 ++++++
 guide/example/example-formats-09.kt           |  26 +++-
 guide/example/example-formats-10.kt           |  19 +--
 guide/example/example-formats-11.kt           |  35 +----
 guide/example/example-formats-12.kt           |  31 +---
 guide/example/example-formats-13.kt           |   6 +-
 guide/example/example-formats-14.kt           |  24 +--
 guide/example/example-formats-15.kt           |  12 +-
 guide/example/example-formats-16.kt           |  78 ++++------
 guide/example/example-formats-17.kt           |  47 +-----
 guide/example/example-formats-18.kt           | 142 ++++++++++++++++++
 guide/test/FormatsTest.kt                     |  42 +++---
 18 files changed, 374 insertions(+), 237 deletions(-)
 rename formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/{ProtoUnknownFields.kt => ProtoMessage.kt} (92%)
 create mode 100644 guide/example/example-formats-18.kt

diff --git a/docs/formats.md b/docs/formats.md
index 307601234d..081af3a848 100644
--- a/docs/formats.md
+++ b/docs/formats.md
@@ -25,6 +25,7 @@ stable, these are currently experimental features of Kotlin Serialization.
   * [Oneof field (experimental)](#oneof-field-experimental)
     * [Usage](#usage)
     * [Alternative](#alternative)
+  * [Preserving unknown fields (experimental)](#preserving-unknown-fields-experimental)
   * [ProtoBuf schema generator (experimental)](#protobuf-schema-generator-experimental)
 * [Properties (experimental)](#properties-experimental)
 * [Custom formats (experimental)](#custom-formats-experimental)
@@ -483,7 +484,7 @@ Field #3: 1D Fixed32 Value = 3, Hex = 03-00-00-00
 
 ### Lists as repeated fields
 
-By default, kotlin lists and other collections are representend as repeated fields. 
+By default, kotlin lists and other collections are represented as repeated fields. 
 In the protocol buffers when the list is empty there are no elements in the 
 stream with the corresponding number. For Kotlin Serialization you must explicitly specify a default of `emptyList()`
 for any property of a collection or map type. Otherwise you will not be able deserialize an empty
@@ -642,6 +643,54 @@ is also compatible with the `message Data` given above, which means the same inp
 
 But please note that there are no exclusivity checks. This means that if an instance of `Data2` has both (or none) `homeNumber` and `workNumber` as non-null values and is serialized to protobuf, it no longer complies with the original schema. If you send such data to another parser, one of the fields may be omitted, leading to an unknown issue.
 
+### Preserving unknown fields (experimental)
+
+You may keep updating your schema by adding new fields, but you may not want to break compatibility with the old data.
+
+Kotlin Serialization `ProtoBuf` format supports preserving unknown fields, as described in the [Protocol Buffer-Unknown Fields](https://protobuf.dev/programming-guides/proto3/#unknowns).
+
+To keep the unknown fields, add a property in type `ProtoMessage` with default value `null` or `ProtoMessage.Empty`, and annotation `@ProtoUnknownFields` to your data class.
+
+<!--- INCLUDE
+import kotlinx.serialization.*
+import kotlinx.serialization.protobuf.*
+-->
+
+```kotlin
+@OptIn(ExperimentalSerializationApi::class)
+@Serializable
+data class Data(
+    @ProtoNumber(1) val name: String,
+    @ProtoUnknownFields val unknownFields: ProtoMessage = ProtoMessage.Empty
+)
+
+@OptIn(ExperimentalSerializationApi::class)
+@Serializable
+data class NewData(
+    @ProtoNumber(1) val name: String,
+    @ProtoNumber(2) val age: Int,
+)
+
+@OptIn(ExperimentalSerializationApi::class)
+fun main() {
+  val dataFromNewBinary = NewData("Tom", 25)
+  val hexString = ProtoBuf.encodeToHexString(dataFromNewBinary)
+  val dataInOldBinary = ProtoBuf.decodeFromHexString<Data>(hexString)
+  val hexOfOldData = ProtoBuf.encodeToHexString(dataInOldBinary)
+  println(hexOfOldData)
+  println(hexString)
+  assert(hexOfOldData == hexString)
+}
+```
+
+> You can get the full code [here](../guide/example/example-formats-09.kt).
+
+```text
+0a03546f6d1019
+0a03546f6d1019
+```
+
+<!--- TEST -->
 ### ProtoBuf schema generator (experimental)
 
 As mentioned above, when working with protocol buffers you usually use a ".proto" file and a code generator for your
@@ -676,7 +725,7 @@ fun main() {
   println(schemas)
 }
 ```
-> You can get the full code [here](../guide/example/example-formats-09.kt).
+> You can get the full code [here](../guide/example/example-formats-10.kt).
 
 Which would output as follows.
 
@@ -684,7 +733,7 @@ Which would output as follows.
 syntax = "proto2";
 
 
-// serial name 'example.exampleFormats09.SampleData'
+// serial name 'example.exampleFormats10.SampleData'
 message SampleData {
   required int64 amount = 1;
   optional string description = 2;
@@ -729,7 +778,7 @@ fun main() {
 }
 ```      
 
-> You can get the full code [here](../guide/example/example-formats-10.kt).
+> You can get the full code [here](../guide/example/example-formats-11.kt).
 
 The resulting map has dot-separated keys representing keys of the nested objects.
 
@@ -814,7 +863,7 @@ fun main() {
 }
 ```                                    
 
-> You can get the full code [here](../guide/example/example-formats-11.kt).
+> You can get the full code [here](../guide/example/example-formats-12.kt).
 
 As a result, we got all the primitive values in our object graph visited and put into a list
 in _serial_ order.
@@ -923,7 +972,7 @@ fun main() {
 }
 ```
 
-> You can get the full code [here](../guide/example/example-formats-12.kt).
+> You can get the full code [here](../guide/example/example-formats-13.kt).
 
 Now we can convert a list of primitives back to an object tree.
 
@@ -1021,7 +1070,7 @@ fun main() {
 }
 -->
 
-> You can get the full code [here](../guide/example/example-formats-13.kt).
+> You can get the full code [here](../guide/example/example-formats-14.kt).
 
 <!--- TEST 
 [kotlinx.serialization, kotlin, 9000]
@@ -1135,7 +1184,7 @@ fun main() {
 }
 ```
 
-> You can get the full code [here](../guide/example/example-formats-14.kt).
+> You can get the full code [here](../guide/example/example-formats-15.kt).
 
 We see the size of the list added to the result, letting the decoder know where to stop. 
 
@@ -1254,7 +1303,7 @@ fun main() {
 
 ```
 
-> You can get the full code [here](../guide/example/example-formats-15.kt).
+> You can get the full code [here](../guide/example/example-formats-16.kt).
 
 In the output we see how not-null`!!` and `NULL` marks are used.
 
@@ -1389,7 +1438,7 @@ fun main() {
 }
 ```
               
-> You can get the full code [here](../guide/example/example-formats-16.kt).
+> You can get the full code [here](../guide/example/example-formats-17.kt).
 
 As we can see, the result is a dense binary format that only contains the data that is being serialized. 
 It can be easily tweaked for any kind of domain-specific compact encoding.
@@ -1590,7 +1639,7 @@ fun main() {
 }
 ```
               
-> You can get the full code [here](../guide/example/example-formats-17.kt).
+> You can get the full code [here](../guide/example/example-formats-18.kt).
 
 As we can see, our custom byte array format is being used, with the compact encoding of its size in one byte. 
 
diff --git a/docs/serialization-guide.md b/docs/serialization-guide.md
index 01ada5fa9b..c9987a522b 100644
--- a/docs/serialization-guide.md
+++ b/docs/serialization-guide.md
@@ -161,6 +161,7 @@ Once the project is set up, we can start serializing some classes.
   * <a name='oneof-field-experimental'></a>[Oneof field (experimental)](formats.md#oneof-field-experimental)
     * <a name='usage'></a>[Usage](formats.md#usage)
     * <a name='alternative'></a>[Alternative](formats.md#alternative)
+  * <a name='preserving-unknown-fields-experimental'></a>[Preserving unknown fields (experimental)](formats.md#preserving-unknown-fields-experimental)
   * <a name='protobuf-schema-generator-experimental'></a>[ProtoBuf schema generator (experimental)](formats.md#protobuf-schema-generator-experimental)
 * <a name='properties-experimental'></a>[Properties (experimental)](formats.md#properties-experimental)
 * <a name='custom-formats-experimental'></a>[Custom formats (experimental)](formats.md#custom-formats-experimental)
diff --git a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoUnknownFields.kt b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoMessage.kt
similarity index 92%
rename from formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoUnknownFields.kt
rename to formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoMessage.kt
index 64a86e2e8e..b8ac13cf20 100644
--- a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoUnknownFields.kt
+++ b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoMessage.kt
@@ -6,17 +6,6 @@ package kotlinx.serialization.protobuf
 
 import kotlinx.serialization.*
 import kotlinx.serialization.protobuf.internal.*
-import kotlinx.serialization.protobuf.internal.ProtoWireType
-
-/**
- * Mark a property with type [ProtoMessage] as a holder for unknown fields in protobuf message.
- *
- * All the contents with unregistered proto number will be stored in this field.
- */
-@SerialInfo
-@Target(AnnotationTarget.PROPERTY)
-@ExperimentalSerializationApi
-public annotation class ProtoUnknownFields
 
 /**
  * Represents a protobuf message.
@@ -106,9 +95,7 @@ internal fun ProtoMessage?.merge(vararg fields: ProtoField): ProtoMessage {
 /**
  * Represents a single field in a protobuf message.
  */
-@OptIn(ExperimentalSerializationApi::class)
 @Serializable(with = ProtoFieldSerializer::class)
-@KeepGeneratedSerializer
 @ConsistentCopyVisibility
 internal data class ProtoField internal constructor(
     internal val id: Int,
diff --git a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoTypes.kt b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoTypes.kt
index e673f16f28..99b2b0f783 100644
--- a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoTypes.kt
+++ b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/ProtoTypes.kt
@@ -63,4 +63,14 @@ public annotation class ProtoPacked
 @SerialInfo
 @Target(AnnotationTarget.PROPERTY)
 @ExperimentalSerializationApi
-public annotation class ProtoOneOf
\ No newline at end of file
+public annotation class ProtoOneOf
+
+/**
+ * Mark a property with type [ProtoMessage] as a holder for unknown fields in protobuf message.
+ *
+ * All the contents with unregistered proto number will be stored in this field.
+ */
+@SerialInfo
+@Target(AnnotationTarget.PROPERTY)
+@ExperimentalSerializationApi
+public annotation class ProtoUnknownFields
\ No newline at end of file
diff --git a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt
index 61979e8f5f..ac5ad8222c 100644
--- a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt
+++ b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt
@@ -90,6 +90,9 @@ internal open class ProtobufDecoder(
                     require(unknownHolderIndex == -1) {
                         "Only one unknown fields holder is allowed in a message"
                     }
+                    require(descriptor.getElementDescriptor(i).nullable == ProtoMessageSerializer.descriptor.nullable) {
+                        "ProtoUnknownFields is only allowed on property with type kotlinx.serialization.protobuf.ProtoMessage, with its original serializer."
+                    }
                     mapSize ++
                     unknownHolderIndex = i
                 }
diff --git a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufEncoding.kt b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufEncoding.kt
index edacc6f295..a16c610610 100644
--- a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufEncoding.kt
+++ b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufEncoding.kt
@@ -185,6 +185,9 @@ internal open class ProtobufEncoder(
         require(currentTagOrDefault != MISSING_TAG) {
             "Cannot serialize directly from kotlinx.serialization.protobuf.ProtoMessage."
         }
+        require(currentTagOrDefault.isUnknown) {
+            "kotlinx.serialization.protobuf.ProtoMessage should be annotated with @ProtoUnknownFields."
+        }
         serializer.serialize(this, protoMessage)
     }
 
diff --git a/formats/protobuf/commonTest/src/kotlinx/serialization/protobuf/ProtobufUnknownFieldsTest.kt b/formats/protobuf/commonTest/src/kotlinx/serialization/protobuf/ProtobufUnknownFieldsTest.kt
index 2d171e13ba..e274a5c728 100644
--- a/formats/protobuf/commonTest/src/kotlinx/serialization/protobuf/ProtobufUnknownFieldsTest.kt
+++ b/formats/protobuf/commonTest/src/kotlinx/serialization/protobuf/ProtobufUnknownFieldsTest.kt
@@ -252,4 +252,50 @@ class ProtobufUnknownFieldsTest {
         val data2 = ProtoBuf.decodeFromHexString(BuildData.serializer(), encoded2)
         assertEquals(data, data2)
     }
+
+    @Serializable
+    data class DataWithWrongTypeUnknownFields(
+        val a: Int,
+        @ProtoUnknownFields val unknownFields: Map<Int, ByteArray>,
+    )
+
+    @Test
+    fun testCannotDecodeWrongTypeUnknownFields() {
+        assertFailsWith<IllegalArgumentException> {
+            ProtoBuf.decodeFromHexString(DataWithWrongTypeUnknownFields.serializer(), "")
+        }
+    }
+
+    @Serializable
+    data class DataWithMissingUnknownFields(
+        val a: Int,
+        val unknownFields: ProtoMessage = ProtoMessage.Empty
+    )
+
+    @Test
+    fun testCannotEncodeMissingAnnotationUnknownFields() {
+        val encoded = "082a120234321a032a2a2a202a202a202a2a120a023432102a1a0234321a0234321a023432"
+        val decoded = ProtoBuf.decodeFromHexString(DataWithMissingUnknownFields.serializer(), encoded)
+        assertFailsWith<IllegalArgumentException> {
+            ProtoBuf.encodeToHexString(DataWithMissingUnknownFields.serializer(), decoded)
+        }
+    }
+
+    @Serializable
+    data class DataWithNullableUnknownFields(
+        @ProtoNumber(1) val a: Int,
+        @ProtoUnknownFields val unknownFields: ProtoMessage? = null
+    )
+    @Test
+    fun testDataWithNullableUnknownFields() {
+        val encoded = "082a120234321a032a2a2a202a202a202a2a120a023432102a1a0234321a0234321a023432"
+        val decoded = ProtoBuf.decodeFromHexString(DataWithNullableUnknownFields.serializer(), encoded)
+        assertEquals(42, decoded.a)
+        assertEquals(6, decoded.unknownFields?.size)
+
+        val encoded2 = "082a"
+        val decoded2 = ProtoBuf.decodeFromHexString(DataWithNullableUnknownFields.serializer(), encoded2)
+        assertEquals(42, decoded2.a)
+        assertNull(decoded2.unknownFields)
+    }
 }
\ No newline at end of file
diff --git a/guide/example/example-formats-09.kt b/guide/example/example-formats-09.kt
index 8c1ee22386..cef3fe3f22 100644
--- a/guide/example/example-formats-09.kt
+++ b/guide/example/example-formats-09.kt
@@ -3,18 +3,28 @@ package example.exampleFormats09
 
 import kotlinx.serialization.*
 import kotlinx.serialization.protobuf.*
-import kotlinx.serialization.protobuf.schema.ProtoBufSchemaGenerator
 
+@OptIn(ExperimentalSerializationApi::class)
+@Serializable
+data class Data(
+    @ProtoNumber(1) val name: String,
+    @ProtoUnknownFields val unknownFields: ProtoMessage = ProtoMessage.Empty
+)
+
+@OptIn(ExperimentalSerializationApi::class)
 @Serializable
-data class SampleData(
-    val amount: Long,
-    val description: String?,
-    val department: String = "QA"
+data class NewData(
+    @ProtoNumber(1) val name: String,
+    @ProtoNumber(2) val age: Int,
 )
 
 @OptIn(ExperimentalSerializationApi::class)
 fun main() {
-  val descriptors = listOf(SampleData.serializer().descriptor)
-  val schemas = ProtoBufSchemaGenerator.generateSchemaText(descriptors)
-  println(schemas)
+  val dataFromNewBinary = NewData("Tom", 25)
+  val hexString = ProtoBuf.encodeToHexString(dataFromNewBinary)
+  val dataInOldBinary = ProtoBuf.decodeFromHexString<Data>(hexString)
+  val hexOfOldData = ProtoBuf.encodeToHexString(dataInOldBinary)
+  println(hexOfOldData)
+  println(hexString)
+  assert(hexOfOldData == hexString)
 }
diff --git a/guide/example/example-formats-10.kt b/guide/example/example-formats-10.kt
index 6941e45971..e6e1505a36 100644
--- a/guide/example/example-formats-10.kt
+++ b/guide/example/example-formats-10.kt
@@ -2,18 +2,19 @@
 package example.exampleFormats10
 
 import kotlinx.serialization.*
-import kotlinx.serialization.properties.Properties // todo: remove when no longer needed
-import kotlinx.serialization.properties.*
+import kotlinx.serialization.protobuf.*
+import kotlinx.serialization.protobuf.schema.ProtoBufSchemaGenerator
 
 @Serializable
-class Project(val name: String, val owner: User)
-
-@Serializable
-class User(val name: String)
+data class SampleData(
+    val amount: Long,
+    val description: String?,
+    val department: String = "QA"
+)
 
 @OptIn(ExperimentalSerializationApi::class)
 fun main() {
-    val data = Project("kotlinx.serialization",  User("kotlin"))
-    val map = Properties.encodeToMap(data)
-    map.forEach { (k, v) -> println("$k = $v") }
+  val descriptors = listOf(SampleData.serializer().descriptor)
+  val schemas = ProtoBufSchemaGenerator.generateSchemaText(descriptors)
+  println(schemas)
 }
diff --git a/guide/example/example-formats-11.kt b/guide/example/example-formats-11.kt
index 6b8ccd1174..36e21a7700 100644
--- a/guide/example/example-formats-11.kt
+++ b/guide/example/example-formats-11.kt
@@ -2,39 +2,18 @@
 package example.exampleFormats11
 
 import kotlinx.serialization.*
-import kotlinx.serialization.descriptors.*
-import kotlinx.serialization.encoding.*
-import kotlinx.serialization.modules.*
-
-@ExperimentalSerializationApi
-class ListEncoder : AbstractEncoder() {
-    val list = mutableListOf<Any>()
-
-    override val serializersModule: SerializersModule = EmptySerializersModule()
-
-    override fun encodeValue(value: Any) {
-        list.add(value)
-    }
-}
-
-@ExperimentalSerializationApi
-fun <T> encodeToList(serializer: SerializationStrategy<T>, value: T): List<Any> {
-    val encoder = ListEncoder()
-    encoder.encodeSerializableValue(serializer, value)
-    return encoder.list
-}
-
-@ExperimentalSerializationApi
-inline fun <reified T> encodeToList(value: T) = encodeToList(serializer(), value)
+import kotlinx.serialization.properties.Properties // todo: remove when no longer needed
+import kotlinx.serialization.properties.*
 
 @Serializable
-data class Project(val name: String, val owner: User, val votes: Int)
+class Project(val name: String, val owner: User)
 
 @Serializable
-data class User(val name: String)
+class User(val name: String)
 
 @OptIn(ExperimentalSerializationApi::class)
 fun main() {
-    val data = Project("kotlinx.serialization",  User("kotlin"), 9000)
-    println(encodeToList(data))
+    val data = Project("kotlinx.serialization",  User("kotlin"))
+    val map = Properties.encodeToMap(data)
+    map.forEach { (k, v) -> println("$k = $v") }
 }
diff --git a/guide/example/example-formats-12.kt b/guide/example/example-formats-12.kt
index 5f087d94d3..1cf9e5ffef 100644
--- a/guide/example/example-formats-12.kt
+++ b/guide/example/example-formats-12.kt
@@ -27,32 +27,6 @@ fun <T> encodeToList(serializer: SerializationStrategy<T>, value: T): List<Any>
 @ExperimentalSerializationApi
 inline fun <reified T> encodeToList(value: T) = encodeToList(serializer(), value)
 
-@ExperimentalSerializationApi
-class ListDecoder(val list: ArrayDeque<Any>) : AbstractDecoder() {
-    private var elementIndex = 0
-
-    override val serializersModule: SerializersModule = EmptySerializersModule()
-
-    override fun decodeValue(): Any = list.removeFirst()
-    
-    override fun decodeElementIndex(descriptor: SerialDescriptor): Int {
-        if (elementIndex == descriptor.elementsCount) return CompositeDecoder.DECODE_DONE
-        return elementIndex++
-    }
-
-    override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder =
-        ListDecoder(list)
-}
-
-@ExperimentalSerializationApi
-fun <T> decodeFromList(list: List<Any>, deserializer: DeserializationStrategy<T>): T {
-    val decoder = ListDecoder(ArrayDeque(list))
-    return decoder.decodeSerializableValue(deserializer)
-}
-
-@ExperimentalSerializationApi
-inline fun <reified T> decodeFromList(list: List<Any>): T = decodeFromList(list, serializer())
-
 @Serializable
 data class Project(val name: String, val owner: User, val votes: Int)
 
@@ -62,8 +36,5 @@ data class User(val name: String)
 @OptIn(ExperimentalSerializationApi::class)
 fun main() {
     val data = Project("kotlinx.serialization",  User("kotlin"), 9000)
-    val list = encodeToList(data)
-    println(list)
-    val obj = decodeFromList<Project>(list)
-    println(obj)
+    println(encodeToList(data))
 }
diff --git a/guide/example/example-formats-13.kt b/guide/example/example-formats-13.kt
index f2915f6454..8cc99d8078 100644
--- a/guide/example/example-formats-13.kt
+++ b/guide/example/example-formats-13.kt
@@ -41,10 +41,8 @@ class ListDecoder(val list: ArrayDeque<Any>) : AbstractDecoder() {
     }
 
     override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder =
-        ListDecoder(list) 
-
-    override fun decodeSequentially(): Boolean = true
-}        
+        ListDecoder(list)
+}
 
 @ExperimentalSerializationApi
 fun <T> decodeFromList(list: List<Any>, deserializer: DeserializationStrategy<T>): T {
diff --git a/guide/example/example-formats-14.kt b/guide/example/example-formats-14.kt
index 782022a2e6..79d7c5e62a 100644
--- a/guide/example/example-formats-14.kt
+++ b/guide/example/example-formats-14.kt
@@ -14,12 +14,7 @@ class ListEncoder : AbstractEncoder() {
 
     override fun encodeValue(value: Any) {
         list.add(value)
-    }                               
-
-    override fun beginCollection(descriptor: SerialDescriptor, collectionSize: Int): CompositeEncoder {
-        encodeInt(collectionSize)
-        return this
-    }                                                
+    }
 }
 
 @ExperimentalSerializationApi
@@ -33,26 +28,23 @@ fun <T> encodeToList(serializer: SerializationStrategy<T>, value: T): List<Any>
 inline fun <reified T> encodeToList(value: T) = encodeToList(serializer(), value)
 
 @ExperimentalSerializationApi
-class ListDecoder(val list: ArrayDeque<Any>, var elementsCount: Int = 0) : AbstractDecoder() {
+class ListDecoder(val list: ArrayDeque<Any>) : AbstractDecoder() {
     private var elementIndex = 0
 
     override val serializersModule: SerializersModule = EmptySerializersModule()
 
     override fun decodeValue(): Any = list.removeFirst()
-
+    
     override fun decodeElementIndex(descriptor: SerialDescriptor): Int {
-        if (elementIndex == elementsCount) return CompositeDecoder.DECODE_DONE
+        if (elementIndex == descriptor.elementsCount) return CompositeDecoder.DECODE_DONE
         return elementIndex++
     }
 
     override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder =
-        ListDecoder(list, descriptor.elementsCount)
+        ListDecoder(list) 
 
     override fun decodeSequentially(): Boolean = true
-
-    override fun decodeCollectionSize(descriptor: SerialDescriptor): Int =
-        decodeInt().also { elementsCount = it }
-}
+}        
 
 @ExperimentalSerializationApi
 fun <T> decodeFromList(list: List<Any>, deserializer: DeserializationStrategy<T>): T {
@@ -64,14 +56,14 @@ fun <T> decodeFromList(list: List<Any>, deserializer: DeserializationStrategy<T>
 inline fun <reified T> decodeFromList(list: List<Any>): T = decodeFromList(list, serializer())
 
 @Serializable
-data class Project(val name: String, val owners: List<User>, val votes: Int)
+data class Project(val name: String, val owner: User, val votes: Int)
 
 @Serializable
 data class User(val name: String)
 
 @OptIn(ExperimentalSerializationApi::class)
 fun main() {
-    val data = Project("kotlinx.serialization",  listOf(User("kotlin"), User("jetbrains")), 9000)
+    val data = Project("kotlinx.serialization",  User("kotlin"), 9000)
     val list = encodeToList(data)
     println(list)
     val obj = decodeFromList<Project>(list)
diff --git a/guide/example/example-formats-15.kt b/guide/example/example-formats-15.kt
index 354ea1cba3..192003679e 100644
--- a/guide/example/example-formats-15.kt
+++ b/guide/example/example-formats-15.kt
@@ -20,9 +20,6 @@ class ListEncoder : AbstractEncoder() {
         encodeInt(collectionSize)
         return this
     }                                                
-
-    override fun encodeNull() = encodeValue("NULL")
-    override fun encodeNotNullMark() = encodeValue("!!")
 }
 
 @ExperimentalSerializationApi
@@ -38,7 +35,7 @@ inline fun <reified T> encodeToList(value: T) = encodeToList(serializer(), value
 @ExperimentalSerializationApi
 class ListDecoder(val list: ArrayDeque<Any>, var elementsCount: Int = 0) : AbstractDecoder() {
     private var elementIndex = 0
-    
+
     override val serializersModule: SerializersModule = EmptySerializersModule()
 
     override fun decodeValue(): Any = list.removeFirst()
@@ -55,8 +52,6 @@ class ListDecoder(val list: ArrayDeque<Any>, var elementsCount: Int = 0) : Abstr
 
     override fun decodeCollectionSize(descriptor: SerialDescriptor): Int =
         decodeInt().also { elementsCount = it }
-
-    override fun decodeNotNullMark(): Boolean = decodeString() != "NULL"
 }
 
 @ExperimentalSerializationApi
@@ -69,17 +64,16 @@ fun <T> decodeFromList(list: List<Any>, deserializer: DeserializationStrategy<T>
 inline fun <reified T> decodeFromList(list: List<Any>): T = decodeFromList(list, serializer())
 
 @Serializable
-data class Project(val name: String, val owner: User?, val votes: Int?)
+data class Project(val name: String, val owners: List<User>, val votes: Int)
 
 @Serializable
 data class User(val name: String)
 
 @OptIn(ExperimentalSerializationApi::class)
 fun main() {
-    val data = Project("kotlinx.serialization",  User("kotlin") , null)
+    val data = Project("kotlinx.serialization",  listOf(User("kotlin"), User("jetbrains")), 9000)
     val list = encodeToList(data)
     println(list)
     val obj = decodeFromList<Project>(list)
     println(obj)
 }
-
diff --git a/guide/example/example-formats-16.kt b/guide/example/example-formats-16.kt
index 7a93ce2717..c29fb71961 100644
--- a/guide/example/example-formats-16.kt
+++ b/guide/example/example-formats-16.kt
@@ -2,58 +2,46 @@
 package example.exampleFormats16
 
 import kotlinx.serialization.*
-import kotlinx.serialization.Serializable
 import kotlinx.serialization.descriptors.*
 import kotlinx.serialization.encoding.*
 import kotlinx.serialization.modules.*
-import java.io.*
 
 @ExperimentalSerializationApi
-class DataOutputEncoder(val output: DataOutput) : AbstractEncoder() {
+class ListEncoder : AbstractEncoder() {
+    val list = mutableListOf<Any>()
+
     override val serializersModule: SerializersModule = EmptySerializersModule()
-    override fun encodeBoolean(value: Boolean) = output.writeByte(if (value) 1 else 0)
-    override fun encodeByte(value: Byte) = output.writeByte(value.toInt())
-    override fun encodeShort(value: Short) = output.writeShort(value.toInt())
-    override fun encodeInt(value: Int) = output.writeInt(value)
-    override fun encodeLong(value: Long) = output.writeLong(value)
-    override fun encodeFloat(value: Float) = output.writeFloat(value)
-    override fun encodeDouble(value: Double) = output.writeDouble(value)
-    override fun encodeChar(value: Char) = output.writeChar(value.code)
-    override fun encodeString(value: String) = output.writeUTF(value)
-    override fun encodeEnum(enumDescriptor: SerialDescriptor, index: Int) = output.writeInt(index)
+
+    override fun encodeValue(value: Any) {
+        list.add(value)
+    }                               
 
     override fun beginCollection(descriptor: SerialDescriptor, collectionSize: Int): CompositeEncoder {
         encodeInt(collectionSize)
         return this
-    }
+    }                                                
 
-    override fun encodeNull() = encodeBoolean(false)
-    override fun encodeNotNullMark() = encodeBoolean(true)
+    override fun encodeNull() = encodeValue("NULL")
+    override fun encodeNotNullMark() = encodeValue("!!")
 }
 
 @ExperimentalSerializationApi
-fun <T> encodeTo(output: DataOutput, serializer: SerializationStrategy<T>, value: T) {
-    val encoder = DataOutputEncoder(output)
+fun <T> encodeToList(serializer: SerializationStrategy<T>, value: T): List<Any> {
+    val encoder = ListEncoder()
     encoder.encodeSerializableValue(serializer, value)
+    return encoder.list
 }
 
 @ExperimentalSerializationApi
-inline fun <reified T> encodeTo(output: DataOutput, value: T) = encodeTo(output, serializer(), value)
+inline fun <reified T> encodeToList(value: T) = encodeToList(serializer(), value)
 
 @ExperimentalSerializationApi
-class DataInputDecoder(val input: DataInput, var elementsCount: Int = 0) : AbstractDecoder() {
+class ListDecoder(val list: ArrayDeque<Any>, var elementsCount: Int = 0) : AbstractDecoder() {
     private var elementIndex = 0
+    
     override val serializersModule: SerializersModule = EmptySerializersModule()
-    override fun decodeBoolean(): Boolean = input.readByte().toInt() != 0
-    override fun decodeByte(): Byte = input.readByte()
-    override fun decodeShort(): Short = input.readShort()
-    override fun decodeInt(): Int = input.readInt()
-    override fun decodeLong(): Long = input.readLong()
-    override fun decodeFloat(): Float = input.readFloat()
-    override fun decodeDouble(): Double = input.readDouble()
-    override fun decodeChar(): Char = input.readChar()
-    override fun decodeString(): String = input.readUTF()
-    override fun decodeEnum(enumDescriptor: SerialDescriptor): Int = input.readInt()
+
+    override fun decodeValue(): Any = list.removeFirst()
 
     override fun decodeElementIndex(descriptor: SerialDescriptor): Int {
         if (elementIndex == elementsCount) return CompositeDecoder.DECODE_DONE
@@ -61,41 +49,37 @@ class DataInputDecoder(val input: DataInput, var elementsCount: Int = 0) : Abstr
     }
 
     override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder =
-        DataInputDecoder(input, descriptor.elementsCount)
+        ListDecoder(list, descriptor.elementsCount)
 
     override fun decodeSequentially(): Boolean = true
 
     override fun decodeCollectionSize(descriptor: SerialDescriptor): Int =
         decodeInt().also { elementsCount = it }
 
-    override fun decodeNotNullMark(): Boolean = decodeBoolean()
+    override fun decodeNotNullMark(): Boolean = decodeString() != "NULL"
 }
 
 @ExperimentalSerializationApi
-fun <T> decodeFrom(input: DataInput, deserializer: DeserializationStrategy<T>): T {
-    val decoder = DataInputDecoder(input)
+fun <T> decodeFromList(list: List<Any>, deserializer: DeserializationStrategy<T>): T {
+    val decoder = ListDecoder(ArrayDeque(list))
     return decoder.decodeSerializableValue(deserializer)
 }
 
 @ExperimentalSerializationApi
-inline fun <reified T> decodeFrom(input: DataInput): T = decodeFrom(input, serializer())
+inline fun <reified T> decodeFromList(list: List<Any>): T = decodeFromList(list, serializer())
 
-fun ByteArray.toAsciiHexString() = joinToString("") {
-    if (it in 32..127) it.toInt().toChar().toString() else
-        "{${it.toUByte().toString(16).padStart(2, '0').uppercase()}}"
-}
+@Serializable
+data class Project(val name: String, val owner: User?, val votes: Int?)
 
 @Serializable
-data class Project(val name: String, val language: String)
+data class User(val name: String)
 
 @OptIn(ExperimentalSerializationApi::class)
 fun main() {
-    val data = Project("kotlinx.serialization", "Kotlin")
-    val output = ByteArrayOutputStream()
-    encodeTo(DataOutputStream(output), data)
-    val bytes = output.toByteArray()
-    println(bytes.toAsciiHexString())
-    val input = ByteArrayInputStream(bytes)
-    val obj = decodeFrom<Project>(DataInputStream(input))
+    val data = Project("kotlinx.serialization",  User("kotlin") , null)
+    val list = encodeToList(data)
+    println(list)
+    val obj = decodeFromList<Project>(list)
     println(obj)
 }
+
diff --git a/guide/example/example-formats-17.kt b/guide/example/example-formats-17.kt
index ab16bf7e3b..162f0b330c 100644
--- a/guide/example/example-formats-17.kt
+++ b/guide/example/example-formats-17.kt
@@ -4,11 +4,10 @@ package example.exampleFormats17
 import kotlinx.serialization.*
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.descriptors.*
-import kotlinx.serialization.modules.*
 import kotlinx.serialization.encoding.*
+import kotlinx.serialization.modules.*
 import java.io.*
 
-private val byteArraySerializer = serializer<ByteArray>()
 @ExperimentalSerializationApi
 class DataOutputEncoder(val output: DataOutput) : AbstractEncoder() {
     override val serializersModule: SerializersModule = EmptySerializersModule()
@@ -30,27 +29,6 @@ class DataOutputEncoder(val output: DataOutput) : AbstractEncoder() {
 
     override fun encodeNull() = encodeBoolean(false)
     override fun encodeNotNullMark() = encodeBoolean(true)
-
-    override fun <T> encodeSerializableValue(serializer: SerializationStrategy<T>, value: T) {
-        if (serializer.descriptor == byteArraySerializer.descriptor)
-            encodeByteArray(value as ByteArray)
-        else
-            super.encodeSerializableValue(serializer, value)
-    }
-
-    private fun encodeByteArray(bytes: ByteArray) {
-        encodeCompactSize(bytes.size)
-        output.write(bytes)
-    }
-    
-    private fun encodeCompactSize(value: Int) {
-        if (value < 0xff) {
-            output.writeByte(value)
-        } else {
-            output.writeByte(0xff)
-            output.writeInt(value)
-        }
-    }            
 }
 
 @ExperimentalSerializationApi
@@ -91,25 +69,6 @@ class DataInputDecoder(val input: DataInput, var elementsCount: Int = 0) : Abstr
         decodeInt().also { elementsCount = it }
 
     override fun decodeNotNullMark(): Boolean = decodeBoolean()
-
-    @Suppress("UNCHECKED_CAST")
-    override fun <T> decodeSerializableValue(deserializer: DeserializationStrategy<T>, previousValue: T?): T =
-        if (deserializer.descriptor == byteArraySerializer.descriptor)
-            decodeByteArray() as T
-        else
-            super.decodeSerializableValue(deserializer, previousValue)
-
-    private fun decodeByteArray(): ByteArray {
-        val bytes = ByteArray(decodeCompactSize())
-        input.readFully(bytes)
-        return bytes
-    }
-
-    private fun decodeCompactSize(): Int {
-        val byte = input.readByte().toInt() and 0xff
-        if (byte < 0xff) return byte
-        return input.readInt()
-    }
 }
 
 @ExperimentalSerializationApi
@@ -127,11 +86,11 @@ fun ByteArray.toAsciiHexString() = joinToString("") {
 }
 
 @Serializable
-data class Project(val name: String, val attachment: ByteArray)
+data class Project(val name: String, val language: String)
 
 @OptIn(ExperimentalSerializationApi::class)
 fun main() {
-    val data = Project("kotlinx.serialization", byteArrayOf(0x0A, 0x0B, 0x0C, 0x0D))
+    val data = Project("kotlinx.serialization", "Kotlin")
     val output = ByteArrayOutputStream()
     encodeTo(DataOutputStream(output), data)
     val bytes = output.toByteArray()
diff --git a/guide/example/example-formats-18.kt b/guide/example/example-formats-18.kt
new file mode 100644
index 0000000000..f1aa4569f8
--- /dev/null
+++ b/guide/example/example-formats-18.kt
@@ -0,0 +1,142 @@
+// This file was automatically generated from formats.md by Knit tool. Do not edit.
+package example.exampleFormats18
+
+import kotlinx.serialization.*
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.descriptors.*
+import kotlinx.serialization.modules.*
+import kotlinx.serialization.encoding.*
+import java.io.*
+
+private val byteArraySerializer = serializer<ByteArray>()
+@ExperimentalSerializationApi
+class DataOutputEncoder(val output: DataOutput) : AbstractEncoder() {
+    override val serializersModule: SerializersModule = EmptySerializersModule()
+    override fun encodeBoolean(value: Boolean) = output.writeByte(if (value) 1 else 0)
+    override fun encodeByte(value: Byte) = output.writeByte(value.toInt())
+    override fun encodeShort(value: Short) = output.writeShort(value.toInt())
+    override fun encodeInt(value: Int) = output.writeInt(value)
+    override fun encodeLong(value: Long) = output.writeLong(value)
+    override fun encodeFloat(value: Float) = output.writeFloat(value)
+    override fun encodeDouble(value: Double) = output.writeDouble(value)
+    override fun encodeChar(value: Char) = output.writeChar(value.code)
+    override fun encodeString(value: String) = output.writeUTF(value)
+    override fun encodeEnum(enumDescriptor: SerialDescriptor, index: Int) = output.writeInt(index)
+
+    override fun beginCollection(descriptor: SerialDescriptor, collectionSize: Int): CompositeEncoder {
+        encodeInt(collectionSize)
+        return this
+    }
+
+    override fun encodeNull() = encodeBoolean(false)
+    override fun encodeNotNullMark() = encodeBoolean(true)
+
+    override fun <T> encodeSerializableValue(serializer: SerializationStrategy<T>, value: T) {
+        if (serializer.descriptor == byteArraySerializer.descriptor)
+            encodeByteArray(value as ByteArray)
+        else
+            super.encodeSerializableValue(serializer, value)
+    }
+
+    private fun encodeByteArray(bytes: ByteArray) {
+        encodeCompactSize(bytes.size)
+        output.write(bytes)
+    }
+    
+    private fun encodeCompactSize(value: Int) {
+        if (value < 0xff) {
+            output.writeByte(value)
+        } else {
+            output.writeByte(0xff)
+            output.writeInt(value)
+        }
+    }            
+}
+
+@ExperimentalSerializationApi
+fun <T> encodeTo(output: DataOutput, serializer: SerializationStrategy<T>, value: T) {
+    val encoder = DataOutputEncoder(output)
+    encoder.encodeSerializableValue(serializer, value)
+}
+
+@ExperimentalSerializationApi
+inline fun <reified T> encodeTo(output: DataOutput, value: T) = encodeTo(output, serializer(), value)
+
+@ExperimentalSerializationApi
+class DataInputDecoder(val input: DataInput, var elementsCount: Int = 0) : AbstractDecoder() {
+    private var elementIndex = 0
+    override val serializersModule: SerializersModule = EmptySerializersModule()
+    override fun decodeBoolean(): Boolean = input.readByte().toInt() != 0
+    override fun decodeByte(): Byte = input.readByte()
+    override fun decodeShort(): Short = input.readShort()
+    override fun decodeInt(): Int = input.readInt()
+    override fun decodeLong(): Long = input.readLong()
+    override fun decodeFloat(): Float = input.readFloat()
+    override fun decodeDouble(): Double = input.readDouble()
+    override fun decodeChar(): Char = input.readChar()
+    override fun decodeString(): String = input.readUTF()
+    override fun decodeEnum(enumDescriptor: SerialDescriptor): Int = input.readInt()
+
+    override fun decodeElementIndex(descriptor: SerialDescriptor): Int {
+        if (elementIndex == elementsCount) return CompositeDecoder.DECODE_DONE
+        return elementIndex++
+    }
+
+    override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder =
+        DataInputDecoder(input, descriptor.elementsCount)
+
+    override fun decodeSequentially(): Boolean = true
+
+    override fun decodeCollectionSize(descriptor: SerialDescriptor): Int =
+        decodeInt().also { elementsCount = it }
+
+    override fun decodeNotNullMark(): Boolean = decodeBoolean()
+
+    @Suppress("UNCHECKED_CAST")
+    override fun <T> decodeSerializableValue(deserializer: DeserializationStrategy<T>, previousValue: T?): T =
+        if (deserializer.descriptor == byteArraySerializer.descriptor)
+            decodeByteArray() as T
+        else
+            super.decodeSerializableValue(deserializer, previousValue)
+
+    private fun decodeByteArray(): ByteArray {
+        val bytes = ByteArray(decodeCompactSize())
+        input.readFully(bytes)
+        return bytes
+    }
+
+    private fun decodeCompactSize(): Int {
+        val byte = input.readByte().toInt() and 0xff
+        if (byte < 0xff) return byte
+        return input.readInt()
+    }
+}
+
+@ExperimentalSerializationApi
+fun <T> decodeFrom(input: DataInput, deserializer: DeserializationStrategy<T>): T {
+    val decoder = DataInputDecoder(input)
+    return decoder.decodeSerializableValue(deserializer)
+}
+
+@ExperimentalSerializationApi
+inline fun <reified T> decodeFrom(input: DataInput): T = decodeFrom(input, serializer())
+
+fun ByteArray.toAsciiHexString() = joinToString("") {
+    if (it in 32..127) it.toInt().toChar().toString() else
+        "{${it.toUByte().toString(16).padStart(2, '0').uppercase()}}"
+}
+
+@Serializable
+data class Project(val name: String, val attachment: ByteArray)
+
+@OptIn(ExperimentalSerializationApi::class)
+fun main() {
+    val data = Project("kotlinx.serialization", byteArrayOf(0x0A, 0x0B, 0x0C, 0x0D))
+    val output = ByteArrayOutputStream()
+    encodeTo(DataOutputStream(output), data)
+    val bytes = output.toByteArray()
+    println(bytes.toAsciiHexString())
+    val input = ByteArrayInputStream(bytes)
+    val obj = decodeFrom<Project>(DataInputStream(input))
+    println(obj)
+}
diff --git a/guide/test/FormatsTest.kt b/guide/test/FormatsTest.kt
index f305a59bf2..34762d8c13 100644
--- a/guide/test/FormatsTest.kt
+++ b/guide/test/FormatsTest.kt
@@ -72,10 +72,18 @@ class FormatsTest {
     @Test
     fun testExampleFormats09() {
         captureOutput("ExampleFormats09") { example.exampleFormats09.main() }.verifyOutputLines(
+            "0a03546f6d1019",
+            "0a03546f6d1019"
+        )
+    }
+
+    @Test
+    fun testExampleFormats10() {
+        captureOutput("ExampleFormats10") { example.exampleFormats10.main() }.verifyOutputLines(
             "syntax = \"proto2\";",
             "",
             "",
-            "// serial name 'example.exampleFormats09.SampleData'",
+            "// serial name 'example.exampleFormats10.SampleData'",
             "message SampleData {",
             "  required int64 amount = 1;",
             "  optional string description = 2;",
@@ -87,63 +95,63 @@ class FormatsTest {
     }
 
     @Test
-    fun testExampleFormats10() {
-        captureOutput("ExampleFormats10") { example.exampleFormats10.main() }.verifyOutputLines(
+    fun testExampleFormats11() {
+        captureOutput("ExampleFormats11") { example.exampleFormats11.main() }.verifyOutputLines(
             "name = kotlinx.serialization",
             "owner.name = kotlin"
         )
     }
 
     @Test
-    fun testExampleFormats11() {
-        captureOutput("ExampleFormats11") { example.exampleFormats11.main() }.verifyOutputLines(
+    fun testExampleFormats12() {
+        captureOutput("ExampleFormats12") { example.exampleFormats12.main() }.verifyOutputLines(
             "[kotlinx.serialization, kotlin, 9000]"
         )
     }
 
     @Test
-    fun testExampleFormats12() {
-        captureOutput("ExampleFormats12") { example.exampleFormats12.main() }.verifyOutputLines(
+    fun testExampleFormats13() {
+        captureOutput("ExampleFormats13") { example.exampleFormats13.main() }.verifyOutputLines(
             "[kotlinx.serialization, kotlin, 9000]",
             "Project(name=kotlinx.serialization, owner=User(name=kotlin), votes=9000)"
         )
     }
 
     @Test
-    fun testExampleFormats13() {
-        captureOutput("ExampleFormats13") { example.exampleFormats13.main() }.verifyOutputLines(
+    fun testExampleFormats14() {
+        captureOutput("ExampleFormats14") { example.exampleFormats14.main() }.verifyOutputLines(
             "[kotlinx.serialization, kotlin, 9000]",
             "Project(name=kotlinx.serialization, owner=User(name=kotlin), votes=9000)"
         )
     }
 
     @Test
-    fun testExampleFormats14() {
-        captureOutput("ExampleFormats14") { example.exampleFormats14.main() }.verifyOutputLines(
+    fun testExampleFormats15() {
+        captureOutput("ExampleFormats15") { example.exampleFormats15.main() }.verifyOutputLines(
             "[kotlinx.serialization, 2, kotlin, jetbrains, 9000]",
             "Project(name=kotlinx.serialization, owners=[User(name=kotlin), User(name=jetbrains)], votes=9000)"
         )
     }
 
     @Test
-    fun testExampleFormats15() {
-        captureOutput("ExampleFormats15") { example.exampleFormats15.main() }.verifyOutputLines(
+    fun testExampleFormats16() {
+        captureOutput("ExampleFormats16") { example.exampleFormats16.main() }.verifyOutputLines(
             "[kotlinx.serialization, !!, kotlin, NULL]",
             "Project(name=kotlinx.serialization, owner=User(name=kotlin), votes=null)"
         )
     }
 
     @Test
-    fun testExampleFormats16() {
-        captureOutput("ExampleFormats16") { example.exampleFormats16.main() }.verifyOutputLines(
+    fun testExampleFormats17() {
+        captureOutput("ExampleFormats17") { example.exampleFormats17.main() }.verifyOutputLines(
             "{00}{15}kotlinx.serialization{00}{06}Kotlin",
             "Project(name=kotlinx.serialization, language=Kotlin)"
         )
     }
 
     @Test
-    fun testExampleFormats17() {
-        captureOutput("ExampleFormats17") { example.exampleFormats17.main() }.verifyOutputLines(
+    fun testExampleFormats18() {
+        captureOutput("ExampleFormats18") { example.exampleFormats18.main() }.verifyOutputLines(
             "{00}{15}kotlinx.serialization{04}{0A}{0B}{0C}{0D}",
             "Project(name=kotlinx.serialization, attachment=[10, 11, 12, 13])"
         )

From e95931bf0b52f56c46b4d90dafacbd3c6b600372 Mon Sep 17 00:00:00 2001
From: xiaozhikang <xiaozhikang0916@gmail.com>
Date: Mon, 18 Nov 2024 11:38:48 +0800
Subject: [PATCH 5/6] api dump

---
 .../api/kotlinx-serialization-protobuf.api    | 26 +++++++++++++++++++
 .../kotlinx-serialization-protobuf.klib.api   | 23 ++++++++++++++++
 2 files changed, 49 insertions(+)

diff --git a/formats/protobuf/api/kotlinx-serialization-protobuf.api b/formats/protobuf/api/kotlinx-serialization-protobuf.api
index 682e4bf16b..30918eec70 100644
--- a/formats/protobuf/api/kotlinx-serialization-protobuf.api
+++ b/formats/protobuf/api/kotlinx-serialization-protobuf.api
@@ -30,6 +30,25 @@ public final class kotlinx/serialization/protobuf/ProtoIntegerType : java/lang/E
 	public static fun values ()[Lkotlinx/serialization/protobuf/ProtoIntegerType;
 }
 
+public final class kotlinx/serialization/protobuf/ProtoMessage {
+	public static final field Companion Lkotlinx/serialization/protobuf/ProtoMessage$Companion;
+	public final fun asByteArray ()[B
+	public fun equals (Ljava/lang/Object;)Z
+	public final fun getSize ()I
+	public fun hashCode ()I
+	public final fun merge (Lkotlinx/serialization/protobuf/ProtoMessage;)Lkotlinx/serialization/protobuf/ProtoMessage;
+	public final fun plus (Lkotlinx/serialization/protobuf/ProtoMessage;)Lkotlinx/serialization/protobuf/ProtoMessage;
+}
+
+public final class kotlinx/serialization/protobuf/ProtoMessage$Companion {
+	public final fun getEmpty ()Lkotlinx/serialization/protobuf/ProtoMessage;
+	public final fun serializer ()Lkotlinx/serialization/KSerializer;
+}
+
+public final class kotlinx/serialization/protobuf/ProtoMessageKt {
+	public static final fun merge (Lkotlinx/serialization/protobuf/ProtoMessage;Lkotlinx/serialization/protobuf/ProtoMessage;)Lkotlinx/serialization/protobuf/ProtoMessage;
+}
+
 public abstract interface annotation class kotlinx/serialization/protobuf/ProtoNumber : java/lang/annotation/Annotation {
 	public abstract fun number ()I
 }
@@ -62,6 +81,13 @@ public synthetic class kotlinx/serialization/protobuf/ProtoType$Impl : kotlinx/s
 	public final synthetic fun type ()Lkotlinx/serialization/protobuf/ProtoIntegerType;
 }
 
+public abstract interface annotation class kotlinx/serialization/protobuf/ProtoUnknownFields : java/lang/annotation/Annotation {
+}
+
+public synthetic class kotlinx/serialization/protobuf/ProtoUnknownFields$Impl : kotlinx/serialization/protobuf/ProtoUnknownFields {
+	public fun <init> ()V
+}
+
 public final class kotlinx/serialization/protobuf/schema/ProtoBufSchemaGenerator {
 	public static final field INSTANCE Lkotlinx/serialization/protobuf/schema/ProtoBufSchemaGenerator;
 	public final fun generateSchemaText (Ljava/util/List;Ljava/lang/String;Ljava/util/Map;)Ljava/lang/String;
diff --git a/formats/protobuf/api/kotlinx-serialization-protobuf.klib.api b/formats/protobuf/api/kotlinx-serialization-protobuf.klib.api
index 4e61b61738..68b968c71f 100644
--- a/formats/protobuf/api/kotlinx-serialization-protobuf.klib.api
+++ b/formats/protobuf/api/kotlinx-serialization-protobuf.klib.api
@@ -28,6 +28,10 @@ open annotation class kotlinx.serialization.protobuf/ProtoType : kotlin/Annotati
         final fun <get-type>(): kotlinx.serialization.protobuf/ProtoIntegerType // kotlinx.serialization.protobuf/ProtoType.type.<get-type>|<get-type>(){}[0]
 }
 
+open annotation class kotlinx.serialization.protobuf/ProtoUnknownFields : kotlin/Annotation { // kotlinx.serialization.protobuf/ProtoUnknownFields|null[0]
+    constructor <init>() // kotlinx.serialization.protobuf/ProtoUnknownFields.<init>|<init>(){}[0]
+}
+
 final enum class kotlinx.serialization.protobuf/ProtoIntegerType : kotlin/Enum<kotlinx.serialization.protobuf/ProtoIntegerType> { // kotlinx.serialization.protobuf/ProtoIntegerType|null[0]
     enum entry DEFAULT // kotlinx.serialization.protobuf/ProtoIntegerType.DEFAULT|null[0]
     enum entry FIXED // kotlinx.serialization.protobuf/ProtoIntegerType.FIXED|null[0]
@@ -49,6 +53,24 @@ final class kotlinx.serialization.protobuf/ProtoBufBuilder { // kotlinx.serializ
         final fun <set-serializersModule>(kotlinx.serialization.modules/SerializersModule) // kotlinx.serialization.protobuf/ProtoBufBuilder.serializersModule.<set-serializersModule>|<set-serializersModule>(kotlinx.serialization.modules.SerializersModule){}[0]
 }
 
+final class kotlinx.serialization.protobuf/ProtoMessage { // kotlinx.serialization.protobuf/ProtoMessage|null[0]
+    final val size // kotlinx.serialization.protobuf/ProtoMessage.size|{}size[0]
+        final fun <get-size>(): kotlin/Int // kotlinx.serialization.protobuf/ProtoMessage.size.<get-size>|<get-size>(){}[0]
+
+    final fun asByteArray(): kotlin/ByteArray // kotlinx.serialization.protobuf/ProtoMessage.asByteArray|asByteArray(){}[0]
+    final fun equals(kotlin/Any?): kotlin/Boolean // kotlinx.serialization.protobuf/ProtoMessage.equals|equals(kotlin.Any?){}[0]
+    final fun hashCode(): kotlin/Int // kotlinx.serialization.protobuf/ProtoMessage.hashCode|hashCode(){}[0]
+    final fun merge(kotlinx.serialization.protobuf/ProtoMessage): kotlinx.serialization.protobuf/ProtoMessage // kotlinx.serialization.protobuf/ProtoMessage.merge|merge(kotlinx.serialization.protobuf.ProtoMessage){}[0]
+    final fun plus(kotlinx.serialization.protobuf/ProtoMessage): kotlinx.serialization.protobuf/ProtoMessage // kotlinx.serialization.protobuf/ProtoMessage.plus|plus(kotlinx.serialization.protobuf.ProtoMessage){}[0]
+
+    final object Companion { // kotlinx.serialization.protobuf/ProtoMessage.Companion|null[0]
+        final val Empty // kotlinx.serialization.protobuf/ProtoMessage.Companion.Empty|{}Empty[0]
+            final fun <get-Empty>(): kotlinx.serialization.protobuf/ProtoMessage // kotlinx.serialization.protobuf/ProtoMessage.Companion.Empty.<get-Empty>|<get-Empty>(){}[0]
+
+        final fun serializer(): kotlinx.serialization/KSerializer<kotlinx.serialization.protobuf/ProtoMessage> // kotlinx.serialization.protobuf/ProtoMessage.Companion.serializer|serializer(){}[0]
+    }
+}
+
 sealed class kotlinx.serialization.protobuf/ProtoBuf : kotlinx.serialization/BinaryFormat { // kotlinx.serialization.protobuf/ProtoBuf|null[0]
     open val serializersModule // kotlinx.serialization.protobuf/ProtoBuf.serializersModule|{}serializersModule[0]
         open fun <get-serializersModule>(): kotlinx.serialization.modules/SerializersModule // kotlinx.serialization.protobuf/ProtoBuf.serializersModule.<get-serializersModule>|<get-serializersModule>(){}[0]
@@ -64,4 +86,5 @@ final object kotlinx.serialization.protobuf.schema/ProtoBufSchemaGenerator { //
     final fun generateSchemaText(kotlinx.serialization.descriptors/SerialDescriptor, kotlin/String? = ..., kotlin.collections/Map<kotlin/String, kotlin/String> = ...): kotlin/String // kotlinx.serialization.protobuf.schema/ProtoBufSchemaGenerator.generateSchemaText|generateSchemaText(kotlinx.serialization.descriptors.SerialDescriptor;kotlin.String?;kotlin.collections.Map<kotlin.String,kotlin.String>){}[0]
 }
 
+final fun (kotlinx.serialization.protobuf/ProtoMessage?).kotlinx.serialization.protobuf/merge(kotlinx.serialization.protobuf/ProtoMessage?): kotlinx.serialization.protobuf/ProtoMessage // kotlinx.serialization.protobuf/merge|merge@kotlinx.serialization.protobuf.ProtoMessage?(kotlinx.serialization.protobuf.ProtoMessage?){}[0]
 final fun kotlinx.serialization.protobuf/ProtoBuf(kotlinx.serialization.protobuf/ProtoBuf = ..., kotlin/Function1<kotlinx.serialization.protobuf/ProtoBufBuilder, kotlin/Unit>): kotlinx.serialization.protobuf/ProtoBuf // kotlinx.serialization.protobuf/ProtoBuf|ProtoBuf(kotlinx.serialization.protobuf.ProtoBuf;kotlin.Function1<kotlinx.serialization.protobuf.ProtoBufBuilder,kotlin.Unit>){}[0]

From 06df0a5e20a60914048107604dab4d7704d55e0d Mon Sep 17 00:00:00 2001
From: xiaozhikang <xiaozhikang0916@gmail.com>
Date: Mon, 18 Nov 2024 18:59:06 +0800
Subject: [PATCH 6/6] add unit test for unknown one-of field

---
 .../protobuf/ProtobufUnknownFieldsTest.kt     | 71 +++++++++++++++++--
 1 file changed, 66 insertions(+), 5 deletions(-)

diff --git a/formats/protobuf/commonTest/src/kotlinx/serialization/protobuf/ProtobufUnknownFieldsTest.kt b/formats/protobuf/commonTest/src/kotlinx/serialization/protobuf/ProtobufUnknownFieldsTest.kt
index e274a5c728..4935d4262d 100644
--- a/formats/protobuf/commonTest/src/kotlinx/serialization/protobuf/ProtobufUnknownFieldsTest.kt
+++ b/formats/protobuf/commonTest/src/kotlinx/serialization/protobuf/ProtobufUnknownFieldsTest.kt
@@ -12,6 +12,7 @@ import kotlin.test.*
 class ProtobufUnknownFieldsTest {
     @Serializable
     data class InnerData(val name: String, val b: Int, val c: List<String>)
+
     @Serializable
     data class BuildData(val a: Int, val b: String, val c: ByteArray, val d: List<Int>, val e: InnerData) {
         override fun equals(other: Any?): Boolean {
@@ -48,7 +49,13 @@ class ProtobufUnknownFieldsTest {
 
     @Test
     fun testDecodeWithUnknownField() {
-        val data = BuildData(42, "42", byteArrayOf(42, 42, 42), listOf(42, 42, 42), InnerData("42", 42, listOf("42", "42", "42")))
+        val data = BuildData(
+            42,
+            "42",
+            byteArrayOf(42, 42, 42),
+            listOf(42, 42, 42),
+            InnerData("42", 42, listOf("42", "42", "42"))
+        )
 
         /**
          * 1: 42
@@ -116,7 +123,13 @@ class ProtobufUnknownFieldsTest {
 
     @Test
     fun testUnknownFieldBeforeKnownField() {
-        val data = BuildData(42, "42", byteArrayOf(42, 42, 42), listOf(42, 42, 42), InnerData("42", 42, listOf("42", "42", "42")))
+        val data = BuildData(
+            42,
+            "42",
+            byteArrayOf(42, 42, 42),
+            listOf(42, 42, 42),
+            InnerData("42", 42, listOf("42", "42", "42"))
+        )
 
         /**
          * 1: 42
@@ -167,7 +180,11 @@ class ProtobufUnknownFieldsTest {
     data class TotalKnownData(@ProtoUnknownFields val fields: ProtoMessage = ProtoMessage.Empty)
 
     @Serializable
-    data class NestedUnknownData(val a: Int, @ProtoNumber(5) val inner: TotalKnownData, @ProtoUnknownFields val unknown: ProtoMessage)
+    data class NestedUnknownData(
+        val a: Int,
+        @ProtoNumber(5) val inner: TotalKnownData,
+        @ProtoUnknownFields val unknown: ProtoMessage
+    )
 
     @Test
     fun testDecodeNestedUnknownData() {
@@ -191,7 +208,7 @@ class ProtobufUnknownFieldsTest {
         assertEquals(5, decoded.unknown.size)
     }
 
-    object CustomSerializer: KSerializer<DataWithUnknownFields> {
+    object CustomSerializer : KSerializer<DataWithUnknownFields> {
         override val descriptor: SerialDescriptor
             get() = buildClassSerialDescriptor("CustomData") {
                 element<Int>("a", annotations = listOf(ProtoNumber(1)))
@@ -224,7 +241,13 @@ class ProtobufUnknownFieldsTest {
 
     @Test
     fun testCustomSerializer() {
-        val data = BuildData(42, "42", byteArrayOf(42, 42, 42), listOf(42, 42, 42), InnerData("42", 42, listOf("42", "42", "42")))
+        val data = BuildData(
+            42,
+            "42",
+            byteArrayOf(42, 42, 42),
+            listOf(42, 42, 42),
+            InnerData("42", 42, listOf("42", "42", "42"))
+        )
 
         /**
          * 1: 42
@@ -286,6 +309,7 @@ class ProtobufUnknownFieldsTest {
         @ProtoNumber(1) val a: Int,
         @ProtoUnknownFields val unknownFields: ProtoMessage? = null
     )
+
     @Test
     fun testDataWithNullableUnknownFields() {
         val encoded = "082a120234321a032a2a2a202a202a202a2a120a023432102a1a0234321a0234321a023432"
@@ -298,4 +322,41 @@ class ProtobufUnknownFieldsTest {
         assertEquals(42, decoded2.a)
         assertNull(decoded2.unknownFields)
     }
+
+    @Serializable
+    data class ToBuildOneOf(val a: String? = null, val b: Long? = null, val c: InnerData? = null)
+
+    @Serializable
+    data class TestFewerOneOf(
+        @ProtoOneOf val oneOf: OneOf? = null,
+        @ProtoUnknownFields val unknownFields: ProtoMessage = ProtoMessage.Empty
+    )
+
+    @Serializable
+    sealed interface OneOf {
+        @Serializable
+        data class A(
+            @ProtoNumber(1) val a: String
+        ) : OneOf
+
+        @Serializable
+        data class B(
+            @ProtoNumber(2) val b: Long
+        ) : OneOf
+    }
+
+    @Test
+    fun testUnknownOneOfField() {
+        val present = ToBuildOneOf(a = "test")
+        val encoded = ProtoBuf.encodeToHexString(present)
+        val decoded = ProtoBuf.decodeFromHexString(TestFewerOneOf.serializer(), encoded)
+        assertEquals(OneOf.A("test"), decoded.oneOf)
+        assertEquals(0, decoded.unknownFields.size)
+
+        val absent = ToBuildOneOf(c = InnerData("test", 42, listOf("test")))
+        val encoded2 = ProtoBuf.encodeToHexString(absent)
+        val decoded2 = ProtoBuf.decodeFromHexString(TestFewerOneOf.serializer(), encoded2)
+        assertNull(decoded2.oneOf)
+        assertEquals(1, decoded2.unknownFields.size)
+    }
 }
\ No newline at end of file