Include more precise field's data types. (#3143)

* Update croissant_utils.py Include more precise field's data types. * Fix tests
huggingface · Mar 3, 2025 · 34a2879 · 34a2879
1 parent 60a1954
commit 34a2879
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 10 deletions.
diff --git a/libs/libcommon/src/libcommon/croissant_utils.py b/libs/libcommon/src/libcommon/croissant_utils.py
@@ -44,13 +44,13 @@ def truncate_features_from_croissant_crumbs_response(content: Mapping[str, Any])
     "date32": "sc:Date",
     "date64": "sc:Date",
     "float8": "sc:Float",
-    "float16": "sc:Float",
-    "float32": "sc:Float",
-    "float64": "sc:Float",
-    "int8": "sc:Integer",
-    "int16": "sc:Integer",
-    "int32": "sc:Integer",
-    "int64": "sc:Integer",
+    "float16": "cr:Float16",
+    "float32": "cr:Float32",
+    "float64": "cr:Float64",
+    "int8": "cr:Int8",
+    "int16": "cr:Int16",
+    "int32": "cr:Int32",
+    "int64": "cr:Int64",
     "large_string": "sc:Text",
     "string": "sc:Text",
     "time32": "sc:Date",

diff --git a/libs/libcommon/tests/test_croissant_utils.py b/libs/libcommon/tests/test_croissant_utils.py
@@ -38,7 +38,7 @@ def test_truncate_features_from_croissant_crumbs_response(num_columns: int) -> N
             {
                 "@type": "cr:Field",
                 "@id": "field_name",
-                "dataType": "sc:Integer",
+                "dataType": "cr:Int32",
                 "source": {"fileSet": {"@id": "distribution_name"}, "extract": {"column": "column_name"}},
             },
         ),
@@ -47,7 +47,7 @@ def test_truncate_features_from_croissant_crumbs_response(num_columns: int) -> N
             {
                 "@type": "cr:Field",
                 "@id": "field_name",
-                "dataType": "sc:Integer",
+                "dataType": "cr:Int32",
                 "source": {"fileSet": {"@id": "distribution_name"}, "extract": {"column": "column_name"}},
                 "isArray": True,
                 "arrayShape": "-1",
@@ -58,7 +58,7 @@ def test_truncate_features_from_croissant_crumbs_response(num_columns: int) -> N
             {
                 "@type": "cr:Field",
                 "@id": "field_name",
-                "dataType": "sc:Integer",
+                "dataType": "cr:Int32",
                 "source": {"fileSet": {"@id": "distribution_name"}, "extract": {"column": "column_name"}},
                 "isArray": True,
                 "arrayShape": "-1",