Ragged l2

dropsonde.cfg

@@ -4,6 +4,9 @@ data_directory = ./example_data
 [OPTIONAL]
 path_to_flight_ids = {platform}/Level_0
 path_to_l0_files = {platform}/Level_0/{flight_id}
+write_ragged_l2 = True
+l2_ragged_dir = ./example_data/Level_2/
+l2_ragged_filename = Level_2.zarr
 
 [GLOBAL_ATTRS]
 example_attr = this is a test
@@ -14,6 +17,7 @@ title = this is a specific level 3 title
 
 [L2_ATTRS]
 l2_attr = this is an l2 specific attributes
+title = test l2
 
 [L4_ATTRS]
 example_attr = this is an l4 specific test

pydropsonde/pipeline.py

@@ -1,4 +1,5 @@
 from .helper.paths import Platform, Flight
+from .helper.xarray_helper import write_ds
 from .helper.__init__ import (
     path_to_flight_ids,
     path_to_l0_files,
@@ -402,6 +403,54 @@ def sondes_to_gridded(sondes: list[Sonde], config: configparser.ConfigParser):
     return gridded
 
 
+def write_ragged_l2(sondes: list[Sonde], config: configparser.ConfigParser):
+    if config["OPTIONAL"].getboolean("write_ragged_l2", fallback=False):
+        vars2d = [
+            var
+            for var in sondes[0].l2_ds.data_vars
+            if "time" in sondes[0].l2_ds[var].dims
+        ]
+        vars1d = [
+            var
+            for var in sondes[0].l2_ds.data_vars
+            if "time" not in sondes[0].l2_ds[var].dims
+        ]
+        l2_ds = xr.concat(
+            [sonde.interim_l2_ds[vars2d] for sonde in sondes],
+            dim="time",
+            data_vars="minimal",
+            combine_attrs="drop_conflicts",
+        ).assign_coords(
+            times_per_sonde=(
+                "sonde",
+                [s.l2_ds.sizes["time"] for s in sondes],
+                {
+                    "description": "Number of times per sonde",
+                    "sample_dimension": "time",
+                },
+            ),
+        )
+        l2_ds = xr.merge(
+            [
+                l2_ds,
+                xr.concat(
+                    [sonde.interim_l2_ds[vars1d] for sonde in sondes], dim="sonde"
+                ),
+            ],
+        )
+
+        write_ds(
+            l2_ds,
+            dir=config["OPTIONAL"].get("l2_ragged_dir", fallback="./"),
+            filename=config["OPTIONAL"].get(
+                "l2_ragged_filename", fallback="L2_ragged.zarr"
+            ),
+            object_dims=(sondes[0].sonde_dim,),
+            alt_dim="time",
+        )
+    return sondes
+
+
 def apply_method_to_dataset(
     obj: Gridded,
     functions: list,
@@ -553,18 +602,25 @@ def run_pipeline(pipeline: dict, config: configparser.ConfigParser):
             "add_qc_to_l2",
             "get_l2_filename",
             "update_history_l2",
+            "update_attrs_l2",
             "write_l2",
+            "add_l2_ds",
         ],
         "output": "sondes",
         "comment": "This steps creates and saves the L2 data after the QC.",
     },
+    "write_ragged_l2": {
+        "intake": "sondes",
+        "apply": write_ragged_l2,
+        "output": "sondes",
+        "comment": "This step writes a ragged L2 dataset if specified in the config file.",
+    },
     "process_L2": {
         "intake": "sondes",
         "apply": iterate_Sonde_method_over_list_of_Sondes_objects,
         "functions": [
             "check_interim_l3",
             "get_l2_filename",
-            "add_l2_ds",
             "create_interim_l3",
             "replace_alt_dim",
             "remove_above_aircraft",

pydropsonde/processor.py

@@ -952,6 +952,22 @@ def get_l2_filename(
 
         return self
 
+    def update_attrs_l2(self):
+        ds = self.interim_l2_ds
+        ds.attrs.update(
+            self.global_attrs["global"],
+        )
+        ds.attrs.update(
+            self.global_attrs["l2"],
+        )
+        ds.attrs.update(
+            dict(
+                history=self.history,
+            )
+        )
+        self.interim_l2_ds = ds
+        return self
+
     def write_l2(self, l2_dir: str = None):
         """
         Writes the L2 file to the specified directory.
@@ -970,22 +986,15 @@ def write_l2(self, l2_dir: str = None):
         if l2_dir is None:
             l2_dir = self.l2_dir
 
-        ds = self.interim_l2_ds
+        ds = self.interim_l2_ds.copy()
         if hasattr(self, "broken_sondes"):
             if self.serial_id in self.broken_sondes:
                 ds.attrs.update(
                     {"comment": self.broken_sondes[self.serial_id]["error"]}
                 )
 
-        ds.attrs.update(
-            self.global_attrs["global"],
-        )
-        ds.attrs.update(
-            self.global_attrs["l2"],
-        )
         ds.attrs.update(
             dict(
-                history=self.history,
                 title=self.global_attrs["l2"].get(
                     "title",
                     self.global_attrs.get("title", "Dropsonde Data") + " Level 2",

pyproject.toml

@@ -28,11 +28,9 @@ dependencies = [
     "circle-fit",
     "flox",
     "fsspec!=0.9.0",
-    "llvmlite",
     "moist-thermodynamics",
     "netcdf4",
     "numpy",
-    "numba",
     "numcodecs<0.16.0",
     "pandas",
     "requests",