Refactor serialization: Replace UsedLanguage with LanguageVersion, rename SerializedChunk to SerializationChunk, and implement support for Protocol Buffer serialization

Author: ftomassetti

README.md

@@ -60,4 +60,10 @@ And then generate a report with:
 ```
 coverage html                                         
 # report generated under htmlcov/index.html
+```
+
+## Update Protobuffer classes
+
+```
+protoc --proto_path=./src --python_out=./src ./src/lionweb/serialization/proto/Chunk.proto
 ```

requirements.txt

@@ -56,6 +56,7 @@ platformdirs==4.3.6
 pluggy==1.5.0
 pre_commit==4.1.0
 prompt_toolkit==3.0.50
+protobuf==6.32.1
 ptyprocess==0.7.0
 pure_eval==0.2.3
 pydantic==2.10.6
@@ -84,6 +85,7 @@ tqdm==4.67.1
 traitlets==5.14.3
 trove-classifiers==2025.3.13.13
 twine==6.1.0
+types-protobuf==6.30.2.20250822
 types-requests==2.32.0.20241016
 typing_extensions==4.12.2
 urllib3==2.3.0

src/lionweb/serialization/__init__.py

@@ -1,5 +1,5 @@
 from .abstract_serialization import AbstractSerialization
-from .data import (MetaPointer, SerializedChunk, SerializedClassifierInstance,
+from .data import (MetaPointer, SerializationChunk, SerializedClassifierInstance,
                    SerializedContainmentValue, SerializedPropertyValue,
                    SerializedReferenceValue)
 from .instantiator import InstantiationError
@@ -8,6 +8,7 @@
 from .serialization_provider import (create_standard_json_serialization,
                                      setup_standard_initialization)
 from .serialized_json_comparison_utils import SerializedJsonComparisonUtils
+#from .protobuf_serialization import ProtobufSerialization
 
 __all__ = [
     "AbstractSerialization",
@@ -17,10 +18,10 @@
     "setup_standard_initialization",
     "SerializedJsonComparisonUtils",
     "MetaPointer",
-    "SerializedChunk",
+    "SerializationChunk",
     "SerializedClassifierInstance",
     "SerializedContainmentValue",
     "SerializedPropertyValue",
     "SerializedReferenceValue",
-    "LowLevelJsonSerialization",
+    "LowLevelJsonSerialization"
 ]

src/lionweb/serialization/abstract_serialization.py

@@ -6,7 +6,7 @@
 from lionweb.model.has_settable_parent import HasSettableParent
 from lionweb.serialization.classifier_resolver import ClassifierResolver
 from lionweb.serialization.data.metapointer import MetaPointer
-from lionweb.serialization.data.serialized_chunk import SerializedChunk
+from lionweb.serialization.data.serialized_chunk import SerializationChunk
 from lionweb.serialization.data.serialized_classifier_instance import \
     SerializedClassifierInstance
 from lionweb.serialization.data.serialized_containment_value import \
@@ -15,7 +15,7 @@
     SerializedPropertyValue
 from lionweb.serialization.data.serialized_reference_value import (
     SerializedReferenceValue, SerializedReferenceValueEntry)
-from lionweb.serialization.data.used_language import UsedLanguage
+from lionweb.serialization.data.language_version import LanguageVersion
 from lionweb.serialization.deserialization_exception import \
     DeserializationException
 from lionweb.serialization.deserialization_status import DeserializationStatus
@@ -77,7 +77,7 @@ def collect_self_and_descendants(
         return collection
 
     def serialize_nodes_to_serialization_chunk(self, classifier_instances):
-        serialized_chunk = SerializedChunk()
+        serialized_chunk = SerializationChunk()
         serialized_chunk.serialization_format_version = self.lion_web_version.value
 
         for classifier_instance in classifier_instances:
@@ -135,7 +135,7 @@ def serialize_nodes_to_serialization_chunk(self, classifier_instances):
 
     def _consider_language_during_serialization(self, serialized_chunk, language):
         self.register_language(language)
-        used_language = UsedLanguage(language.get_key(), language.get_version())
+        used_language = LanguageVersion(language.get_key(), language.get_version())
         if used_language not in serialized_chunk.languages:
             serialized_chunk.languages.append(used_language)
 
@@ -281,7 +281,7 @@ def _serialize_annotations(
             annotation.id for annotation in classifier_instance.get_annotations()
         ]
 
-    def deserialize_serialization_chunk(self, serialized_chunk: SerializedChunk):
+    def deserialize_serialization_chunk(self, serialized_chunk: SerializationChunk):
         serialized_instances = serialized_chunk.classifier_instances
         return self._deserialize_classifier_instances(
             self.lion_web_version, serialized_instances
@@ -388,7 +388,7 @@ def _deserialize_classifier_instances(
         return nodes_with_original_sorting
 
     def _validate_serialization_chunk(
-        self, serialization_chunk: SerializedChunk
+        self, serialization_chunk: SerializationChunk
     ) -> None:
         if serialization_chunk is None:
             raise ValueError("serialization_chunk should not be null")

src/lionweb/serialization/data/__init__.py

@@ -1,13 +1,15 @@
 from .metapointer import MetaPointer
-from .serialized_chunk import SerializedChunk
+from .serialized_chunk import SerializationChunk
 from .serialized_classifier_instance import SerializedClassifierInstance
 from .serialized_containment_value import SerializedContainmentValue
 from .serialized_property_value import SerializedPropertyValue
 from .serialized_reference_value import SerializedReferenceValue
+from .language_version import LanguageVersion
 
 __all__ = [
+    "LanguageVersion",
     "MetaPointer",
-    "SerializedChunk",
+    "SerializationChunk",
     "SerializedClassifierInstance",
     "SerializedContainmentValue",
     "SerializedPropertyValue",

…nweb/serialization/data/used_language.py …b/serialization/data/language_version.pysrc/lionweb/serialization/data/used_language.py renamed to src/lionweb/serialization/data/language_version.py src/lionweb/serialization/data/used_language.py renamed to src/lionweb/serialization/data/language_version.py

@@ -1,7 +1,7 @@
 from typing import Optional
 
 
-class UsedLanguage:
+class LanguageVersion:
     """
     The pair Language Key and Language Version identify a specific version of a language.
     Corresponds to the Java class 'UsedLanguage'.
@@ -20,7 +20,7 @@ def from_language(language):
             language: An object with `key` and `version` attributes.
 
         Returns:
-            UsedLanguage: An instance of UsedLanguage.
+            LanguageVersion: An instance of UsedLanguage.
 
         Raises:
             ValueError: If language or its attributes are None.
@@ -29,7 +29,7 @@ def from_language(language):
             raise ValueError("Language parameter should not be null")
         if language.version is None:
             raise ValueError("Language version should not be null")
-        return UsedLanguage(language.key, language.version)
+        return LanguageVersion(language.key, language.version)
 
     @staticmethod
     def from_meta_pointer(meta_pointer):
@@ -40,7 +40,7 @@ def from_meta_pointer(meta_pointer):
             meta_pointer: An object with `language` and `version` attributes.
 
         Returns:
-            UsedLanguage: An instance of UsedLanguage.
+            LanguageVersion: An instance of UsedLanguage.
 
         Raises:
             ValueError: If meta_pointer or its attributes are None.
@@ -51,7 +51,7 @@ def from_meta_pointer(meta_pointer):
             raise ValueError("meta_pointer language should not be null")
         if meta_pointer.version is None:
             raise ValueError("meta_pointer version should not be null")
-        return UsedLanguage(meta_pointer.language, meta_pointer.version)
+        return LanguageVersion(meta_pointer.language, meta_pointer.version)
 
     def get_key(self) -> Optional[str]:
         return self.key
@@ -66,7 +66,7 @@ def set_version(self, version: str):
         self.version = version
 
     def __eq__(self, other):
-        if not isinstance(other, UsedLanguage):
+        if not isinstance(other, LanguageVersion):
             return False
         return self.key == other.key and self.version == other.version
 

src/lionweb/serialization/data/metapointer.py

@@ -2,6 +2,7 @@
 from typing import TYPE_CHECKING, Optional
 
 
+# TODO adopt interning
 @dataclass(frozen=True, eq=True)
 class MetaPointer:
     if TYPE_CHECKING:

src/lionweb/serialization/data/serialized_chunk.py

@@ -3,13 +3,13 @@
 
 from lionweb.serialization.data.serialized_classifier_instance import \
     SerializedClassifierInstance
-from lionweb.serialization.data.used_language import UsedLanguage
+from lionweb.serialization.data.language_version import LanguageVersion
 
 
 @dataclass
-class SerializedChunk:
+class SerializationChunk:
     serialization_format_version: str = ""
-    languages: List[UsedLanguage] = field(default_factory=list)
+    languages: List[LanguageVersion] = field(default_factory=list)
     classifier_instances: List[SerializedClassifierInstance] = field(
         default_factory=list
     )
@@ -37,7 +37,7 @@ def __str__(self):
         )
 
     def __eq__(self, other):
-        if not isinstance(other, SerializedChunk):
+        if not isinstance(other, SerializationChunk):
             return False
         return (
             self.serialization_format_version == other.serialization_format_version
@@ -81,6 +81,6 @@ def populate_used_languages(self) -> None:
                 self._consider_meta_pointer(property_value.get_meta_pointer())
 
     def _consider_meta_pointer(self, meta_pointer):
-        used_language = UsedLanguage.from_meta_pointer(meta_pointer)
+        used_language = LanguageVersion.from_meta_pointer(meta_pointer)
         if used_language not in self.languages:
             self.languages.append(used_language)

src/lionweb/serialization/data/serialized_property_value.py

@@ -2,7 +2,7 @@
 
 from lionweb.serialization.data.metapointer import MetaPointer
 
-
+# TODO intern values
 class SerializedPropertyValue:
     def __init__(self, meta_pointer: MetaPointer, value: Optional[str]):
         self.meta_pointer = meta_pointer

src/lionweb/serialization/low_level_json_serialization.py

@@ -3,7 +3,7 @@
 
 from lionweb import LionWebVersion
 from lionweb.serialization.data.metapointer import MetaPointer
-from lionweb.serialization.data.serialized_chunk import SerializedChunk
+from lionweb.serialization.data.serialized_chunk import SerializationChunk
 from lionweb.serialization.data.serialized_classifier_instance import \
     SerializedClassifierInstance
 from lionweb.serialization.data.serialized_containment_value import \
@@ -12,7 +12,7 @@
     SerializedPropertyValue
 from lionweb.serialization.data.serialized_reference_value import \
     SerializedReferenceValue
-from lionweb.serialization.data.used_language import UsedLanguage
+from lionweb.serialization.data.language_version import LanguageVersion
 from lionweb.serialization.deserialization_exception import \
     DeserializationException
 from lionweb.serialization.json_utils import JsonArray, JsonElement, JsonObject
@@ -22,8 +22,8 @@
 class LowLevelJsonSerialization:
     def deserialize_serialization_block(
         self, json_element: JsonElement
-    ) -> SerializedChunk:
-        serialized_chunk = SerializedChunk()
+    ) -> SerializationChunk:
+        serialized_chunk = SerializationChunk()
         if isinstance(json_element, dict):
             self._check_no_extra_keys(
                 json_element, ["nodes", "serializationFormatVersion", "languages"]
@@ -38,7 +38,7 @@ def deserialize_serialization_block(
             )
 
     def serialize_to_json_element(
-        self, serialized_chunk: SerializedChunk
+        self, serialized_chunk: SerializationChunk
     ) -> JsonObject:
         serialized_nodes = []
         for node in serialized_chunk.get_classifier_instances():
@@ -97,7 +97,7 @@ def serialize_to_json_element(
         }
 
     def _serialize_language_to_json_element(
-        self, language_key_version: UsedLanguage
+        self, language_key_version: LanguageVersion
     ) -> JsonObject:
         json_object = {
             "key": language_key_version.get_key(),
@@ -114,7 +114,7 @@ def _serialize_metapointer_to_json_element(
             "key": meta_pointer.key,
         }
 
-    def serialize_to_json_string(self, serialized_chunk: SerializedChunk) -> str:
+    def serialize_to_json_string(self, serialized_chunk: SerializationChunk) -> str:
         return json.dumps(
             self.serialize_to_json_element(serialized_chunk),
             indent=2,
@@ -123,7 +123,7 @@ def serialize_to_json_string(self, serialized_chunk: SerializedChunk) -> str:
     def deserialize_serialization_block_from_string(
         self,
         json_string: str,
-    ) -> SerializedChunk:
+    ) -> SerializationChunk:
         try:
             json_element = json.loads(json_string)
             return self.deserialize_serialization_block(json_element)
@@ -132,7 +132,7 @@ def deserialize_serialization_block_from_string(
 
     def deserialize_serialization_block_from_file(
         self, file_path: str
-    ) -> SerializedChunk:
+    ) -> SerializationChunk:
         try:
             with open(file_path, "r") as file:
                 json_element = json.load(file)
@@ -153,7 +153,7 @@ def _check_no_extra_keys(
             )
 
     def _read_serialization_format_version(
-        self, serialized_chunk: SerializedChunk, top_level: JsonObject
+        self, serialized_chunk: SerializationChunk, top_level: JsonObject
     ) -> None:
         if "serializationFormatVersion" not in top_level:
             raise ValueError("serializationFormatVersion not specified")
@@ -173,24 +173,24 @@ def require_is_string(value, desc: str):
     def group_nodes_into_serialization_block(
         serialized_classifier_instances: Iterable[SerializedClassifierInstance],
         lion_web_version: LionWebVersion,
-    ) -> SerializedChunk:
-        serialized_chunk = SerializedChunk()
+    ) -> SerializationChunk:
+        serialized_chunk = SerializationChunk()
         serialized_chunk.serialization_format_version = lion_web_version.value
         for sci in serialized_classifier_instances:
             serialized_chunk.add_classifier_instance(sci)
         serialized_chunk.populate_used_languages()
         return serialized_chunk
 
     def _read_languages(
-        self, serialized_chunk: SerializedChunk, top_level: JsonObject
+        self, serialized_chunk: SerializationChunk, top_level: JsonObject
     ) -> None:
         if "languages" not in top_level:
             raise ValueError("languages not specified")
         languages = top_level.get("languages")
         if isinstance(languages, list):
             for element in languages:
                 try:
-                    language_key_version = UsedLanguage()
+                    language_key_version = LanguageVersion()
                     if isinstance(element, dict):
                         extra_keys = set(element.keys()) - {"key", "version"}
                         if extra_keys:
@@ -220,7 +220,7 @@ def _read_languages(
             raise ValueError(f"We expected a list, we got instead: {languages}")
 
     def _deserialize_classifier_instances(
-        self, serialized_chunk: SerializedChunk, top_level: JsonObject
+        self, serialized_chunk: SerializationChunk, top_level: JsonObject
     ) -> None:
         if "nodes" not in top_level:
             raise ValueError("nodes not specified")