feat: Add InlineArray type for inline JSON serialization; update related documentation and utilities

Author: afshawnlotfi

python/README.md

@@ -26,10 +26,11 @@ pip install meshly
 
 - **`Array`**: Generic array type with meshoptimizer compression
 - **`IndexSequence`**: Optimized encoding for mesh indices (1D array)
+- **`InlineArray`**: Array serialized as inline JSON list (no binary compression)
 
 ### Key Capabilities
 
-- Automatic encoding/decoding of numpy array attributes via `Array`, `IndexSequence` type annotations
+- Automatic encoding/decoding of numpy array attributes via `Array`, `IndexSequence`, `InlineArray` type annotations
 - Custom subclasses with additional array fields are automatically serialized
 - **Extract/Reconstruct API** for content-addressable storage with deduplication
 - **PackableStore** for file-based persistent storage with automatic deduplication
@@ -158,7 +159,7 @@ loaded = TexturedMesh.load_from_zip("textured.zip")
 Use specialized array types for optimized encoding:
 
 ```python
-from meshly import Packable, Array, IndexSequence
+from meshly import Packable, Array, IndexSequence, InlineArray
 from pydantic import Field
 
 class OptimizedMesh(Packable):
@@ -171,6 +172,9 @@ class OptimizedMesh(Packable):
 
     # IndexSequence: optimized for mesh indices
     indices: IndexSequence = Field(..., description="Triangle indices")
+    
+    # InlineArray: small arrays serialized inline (no binary compression)
+    color: InlineArray = Field(..., description="RGB color")
 ```
 
 > **Note:** All dtypes are supported. Arrays with non-4-byte dtypes (e.g., `float16`, `int8`, `uint8`) are automatically padded to 4-byte alignment during encoding and unpadded during decoding (meshoptimizer requirement). For best performance, prefer 4-byte aligned dtypes like `float32`, `int32`, or `float64`.
@@ -469,6 +473,7 @@ Packable (base class)
 ```
 Array          → Generic meshoptimizer compression
 IndexSequence  → Optimized for mesh indices
+InlineArray    → Serialized as inline JSON list (no binary $ref)
 ```
 
 The `Packable` base class provides:
@@ -625,13 +630,14 @@ jax_mesh = mesh.convert_to("jax")
 ### Array Type Annotations
 
 ```python
-from meshly import Array, IndexSequence
+from meshly import Array, IndexSequence, InlineArray
 
 # Use in Pydantic models for automatic encoding
 class MyData(Packable):
     generic_data: Array           # Generic meshoptimizer compression
     vertices: Array               # All arrays use meshoptimizer compression
     indices: IndexSequence        # Optimized for mesh indices
+    color: InlineArray            # Small arrays as inline JSON (no $ref)
 ```
 
 ### ArrayUtils

python/meshly/__init__.py

@@ -21,7 +21,7 @@
     ExtractedArray,
     ArrayEncoding,
     IndexSequence,
-    List,
+    InlineArray,
 )
 from meshly.cell_types import (
     CellType,
@@ -55,7 +55,7 @@
     "TMesh",
     # Array types and utilities
     "Array",
-    "List",
+    "InlineArray",
     "IndexSequence",
     "ExtractedArray",
     "ArrayEncoding",

python/meshly/array.py

@@ -85,7 +85,7 @@ def __eq__(self, other):
         return isinstance(other, _ArrayAnnotation) and self.encoding == other.encoding
 
 
-class _ListAnnotation:
+class _InlineArrayAnnotation:
     """Pydantic annotation for arrays serialized as inline JSON lists."""
 
     def __get_pydantic_core_schema__(
@@ -108,13 +108,13 @@ def _validate(self, v: Any) -> np.ndarray:
     def __get_pydantic_json_schema__(
         self, core_schema: CoreSchema, handler: GetJsonSchemaHandler
     ) -> JsonSchemaValue:
-        return {"type": "list"}
+        return {"type": "inline_array"}
 
     def __hash__(self):
-        return hash("list")
+        return hash("inline_array")
 
     def __eq__(self, other):
-        return isinstance(other, _ListAnnotation)
+        return isinstance(other, _InlineArrayAnnotation)
 
 
 # Public type aliases for Pydantic models
@@ -124,8 +124,8 @@ def __eq__(self, other):
 IndexSequence = Annotated[np.ndarray, _ArrayAnnotation("index_sequence")]
 """Optimized for mesh indices (1D array)."""
 
-List = Annotated[np.ndarray, _ListAnnotation()]
-"""Array serialized as inline JSON list (no binary $ref)."""
+InlineArray = Annotated[Union[list, np.ndarray], _InlineArrayAnnotation()]
+"""Array serialized as inline JSON list (no binary $ref). Accepts list or ndarray input."""
 
 
 # =============================================================================
@@ -225,20 +225,20 @@ def get_array_encoding(annotation: Any) -> ArrayEncoding:
         return "array"
 
     @staticmethod
-    def is_list_annotation(annotation: Any) -> bool:
-        """Check if a type annotation contains _ListAnnotation."""
+    def is_inlined_array_annotation(annotation: Any) -> bool:
+        """Check if a type annotation contains _InlineArrayAnnotation."""
         if annotation is None:
             return False
         origin = get_origin(annotation)
         if origin is Union or isinstance(annotation, types.UnionType):
             for arg in get_args(annotation):
                 if arg is not type(None):
-                    if ArrayUtils.is_list_annotation(arg):
+                    if ArrayUtils.is_inlined_array_annotation(arg):
                         return True
             return False
         if get_origin(annotation) is Annotated:
             for arg in get_args(annotation):
-                if isinstance(arg, _ListAnnotation):
+                if isinstance(arg, _InlineArrayAnnotation):
                     return True
         return False
 

python/meshly/packable.py

@@ -15,6 +15,23 @@
 Serialization options:
 - save_to_zip() / load_from_zip(): Single self-contained zip file
 - save() / load(): File-based asset store with deduplication
+
+Checksum Scheme:
+    Packable checksums are computed from the JSON representation of extracted data.
+    This makes checksum recreation straightforward outside this library.
+    
+    Format: SHA256 of compact JSON: {"data":<data>,"json_schema":<schema>}
+            Keys are sorted, no whitespace (single line).
+    
+    The `data` dict contains $ref entries (e.g. {"$ref":"abc123..."}) pointing
+    to asset checksums, so the packable checksum transitively covers all binary
+    content without embedding the actual bytes.
+    
+    To recreate a checksum externally:
+        import hashlib, json
+        payload = {"data": packable_data, "json_schema": schema}
+        compact_json = json.dumps(payload, sort_keys=True, separators=(',', ':'))
+        checksum = hashlib.sha256(compact_json.encode()).hexdigest()
 """
 
 import time
@@ -68,9 +85,33 @@ class ExtractedPackable(BaseModel):
 
     data: dict[str, Any] = Field(..., description="Serializable dict with primitive fields and checksum refs for arrays")
     json_schema: Optional[dict[str, Any]] = Field(default=None, description="JSON Schema with encoding info")
-    checksum: Optional[str] = Field(default=None, description="Content checksum of the source Packable")
     assets: dict[str, bytes] = Field(default_factory=dict, exclude=True, description="Map of checksum -> encoded bytes for all arrays")
 
+    @cached_property
+    def checksum(self) -> str:
+        """SHA256 checksum computed from data and json_schema.
+        
+        Checksum Format:
+            SHA256 of compact JSON: {"data":<data>,"json_schema":<schema>}
+            Keys are sorted, no whitespace (single line).
+        
+        Why JSON-based:
+            The data dict contains $ref entries pointing to asset checksums,
+            so this checksum transitively covers all array/binary content.
+            This format makes checksum recreation straightforward outside meshly:
+            
+                import hashlib, json
+                payload = {"data": extracted_data, "json_schema": schema}
+                compact_json = json.dumps(payload, sort_keys=True, separators=(',', ':'))
+                checksum = hashlib.sha256(compact_json.encode()).hexdigest()
+            
+        Returns:
+            SHA256 hex digest string
+        """
+        payload = {"data": self.data, "json_schema": self.json_schema}
+        json_bytes = orjson.dumps(payload, option=orjson.OPT_SORT_KEYS)
+        return ChecksumUtils.compute_bytes_checksum(json_bytes)
+
     def extract_checksums(self) -> list[str]:
         """Extract all $ref checksums from a serialized data dict.
 
@@ -209,7 +250,6 @@ def load_extracted(self, key: str) -> "ExtractedPackable":
         return ExtractedPackable(
             data=extracted_data["data"],
             json_schema=extracted_data.get("json_schema"),
-            checksum=extracted_data.get("checksum"),
         )
 
     def extracted_exists(self, key: str) -> bool:
@@ -298,7 +338,7 @@ def extract(self) -> "ExtractedPackable":
         Results are cached for efficiency. Subsequent calls return the cached result.
         
         Returns:
-            ExtractedPackable with metadata (data + schema) and binary assets.
+            ExtractedPackable with metadata (data + schema + checksum) and binary assets.
         """
         if self._cached_extract is not None:
             return self._cached_extract
@@ -310,11 +350,13 @@ def extract(self) -> "ExtractedPackable":
 
         assert isinstance(extracted_result.value, dict), "Extracted value must be a dict for Packable models"
 
-        self._cached_extract = ExtractedPackable(
+        extracted = ExtractedPackable(
             data=extracted_result.value,
             json_schema=type(self).cached_json_schema(),
             assets=extracted_result.assets,
         )
+        
+        self._cached_extract = extracted
         return self._cached_extract
 
     @cached_property
@@ -351,8 +393,22 @@ def encode(self) -> bytes:
 
     @cached_property
     def checksum(self) -> str:
-        """SHA256 checksum of this Packable's encoded bytes (cached)."""
-        return ChecksumUtils.compute_bytes_checksum(self._encoded)
+        """SHA256 checksum of this Packable's extracted JSON representation (cached).
+        
+        Checksum Format:
+            SHA256 of compact JSON: {"data":<extracted_data>,"json_schema":<schema>}
+            Keys are sorted, no whitespace (single line).
+        
+        The data dict contains $ref entries pointing to asset checksums (e.g., 
+        {"$ref":"abc123..."}), so this checksum transitively covers all binary content.
+        
+        To recreate this checksum outside meshly:
+            import hashlib, json
+            payload = {"data": packable_data, "json_schema": schema}
+            compact_json = json.dumps(payload, sort_keys=True, separators=(',', ':'))
+            checksum = hashlib.sha256(compact_json.encode()).hexdigest()
+        """
+        return self.extract().checksum
 
     def set_checksum(self, checksum: str) -> None:
         """Pre-populate the cached checksum to avoid re-encoding."""
@@ -392,9 +448,8 @@ def decode(
         )
         result = cls.reconstruct(extracted, array_type=array_type)
 
-        # Cache to ensure consistent checksums on re-encoding
+        # Cache for efficiency
         result._cached_encode = buf
-        result.set_checksum(ChecksumUtils.compute_bytes_checksum(buf))
         result._cached_extract = extracted
         return result
 
@@ -445,7 +500,7 @@ def reconstruct(
             resolved_data = SchemaUtils.resolve_from_class(cls, extracted.data, asset_provider, array_type)
             result = cls(**resolved_data)
 
-        if extracted.checksum and isinstance(result, Packable):
+        if isinstance(result, Packable):
             result.set_checksum(extracted.checksum)
         return result
 

python/meshly/resource.py

@@ -4,7 +4,7 @@
 """
 
 from pathlib import Path
-from typing import Union
+from typing import Optional, Union
 
 from pydantic import ConfigDict, Field, computed_field
 
@@ -43,8 +43,8 @@ class SimulationCase(Packable):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     data: bytes = Field(exclude=True)
-    ext: str = ""
-    name: str = ""
+    ext: Optional[str] = None
+    name: Optional[str] = None
 
     @staticmethod
     def from_path(path: Union[str, Path]) -> "Resource":

python/meshly/utils/schema_utils.py

@@ -180,7 +180,7 @@ def _resolve_with_type(
             if SchemaUtils._is_resource_ref(expected_type):
                 asset_bytes = SerializationUtils.get_asset(assets, value["$ref"])
                 data = gzip.decompress(asset_bytes)
-                return Resource(data=data, ext=value.get("ext", ""), name=value.get("name", ""))
+                return Resource(data=data, ext=value.get("ext"), name=value.get("name"))
             if isinstance(expected_type, type) and issubclass(expected_type, Packable):
                 return expected_type.decode(
                     SerializationUtils.get_asset(assets, value["$ref"]), array_type
@@ -230,7 +230,7 @@ def _resolve_with_type(
             return {k: SchemaUtils._resolve_with_type(v, object, assets, array_type) for k, v in value.items()}
 
         # List annotation → reconstruct numpy array from inline JSON list
-        if isinstance(value, list) and ArrayUtils.is_list_annotation(expected_type):
+        if isinstance(value, list) and ArrayUtils.is_inlined_array_annotation(expected_type):
             return ArrayUtils.convert_array(np.array(value), array_type)
 
         # List/tuple
@@ -313,7 +313,7 @@ def _resolve_with_prop(
             if prop and prop.is_resource_type():
                 # Resource - assets from _extract_resource are always gzip compressed
                 data = gzip.decompress(asset_bytes)
-                return Resource(data=data, ext=metadata.get("ext", ""), name=metadata.get("name", ""))
+                return Resource(data=data, ext=metadata.get("ext"), name=metadata.get("name"))
 
             if prop and prop.is_array_type():
                 # Array

python/meshly/utils/serialization_utils.py

@@ -253,7 +253,7 @@ def extract_basemodel(value: BaseModel, include_computed: bool = False) -> Extra
                 continue
 
             if ArrayUtils.is_array(field_value):
-                if ArrayUtils.is_list_annotation(hints.get(name)):
+                if ArrayUtils.is_inlined_array_annotation(hints.get(name)):
                     data[name] = np.asarray(field_value).tolist()
                 else:
                     encoding = ArrayUtils.get_array_encoding(hints.get(name))

typescript/src/json-schema.ts

@@ -140,20 +140,29 @@ export interface JsonSchema {
  */
 export class JsonSchemaUtils {
     /**
-     * Check if a property is a meshly array type (not a JSON Schema array like list[str]).
+     * Check if a property is a meshly binary array type (uses $ref for serialization).
+     * Note: inline_array is NOT included here - it serializes inline as JSON, not as $ref.
      */
     static isArrayType(prop: JsonSchemaProperty): boolean {
         // vertex_buffer and index_sequence are always meshly types
         if (prop.type === "vertex_buffer" || prop.type === "index_sequence") {
             return true
         }
-        // type="array" with items is a JSON Schema list, without items is a meshly array
+        // type="array" with items is a JSON Schema list[T], without items is a meshly array
         if (prop.type === "array") {
             return prop.items === undefined
         }
         return false
     }
 
+    /**
+     * Check if a property is an inline array type (serialized as JSON list, no $ref).
+     * This matches Python's InlineArray annotation.
+     */
+    static isInlineArrayType(prop: JsonSchemaProperty): boolean {
+        return prop.type === "inline_array"
+    }
+
     /**
      * Check if a property is a resource type.
      */