Process turbulence

Author: nikoleskinen

scripts/cloudnet.py

@@ -357,10 +357,10 @@ def _process_file(
                         process_instrument(processor, instru_params, Path(directory))
                     except utils.SkipTaskError as err:
                         logging.warning("Skipped task: %s", err)
-    elif product.id in ("mwr-single", "mwr-multi"):
+    elif product.id in ("mwr-single", "mwr-multi", "epsilon-lidar"):
         if args.uuids:
             instruments = {processor.get_instrument(uuid) for uuid in args.uuids}
-        else:
+        elif product.id in ("mwr-single", "mwr-multi"):
             payload = {
                 "site": site.id,
                 "date": date.isoformat(),
@@ -369,6 +369,15 @@ def _process_file(
             metadata = processor.md_api.get("api/files", payload)
             instrument_uuids = {meta["instrumentInfoUuid"] for meta in metadata}
             instruments = {processor.get_instrument(uuid) for uuid in instrument_uuids}
+        else:
+            payload = {
+                "site": site.id,
+                "date": date.isoformat(),
+                "product": "doppler-lidar",
+            }
+            metadata = processor.md_api.get("api/files", payload)
+            instrument_uuids = {meta["instrumentInfoUuid"] for meta in metadata}
+            instruments = {processor.get_instrument(uuid) for uuid in instrument_uuids}
         for instrument in instruments:
             _print_header(
                 {

src/processing/harmonizer/doppler_lidar.py

@@ -126,6 +126,8 @@ def copy_file(self, valid_ind: list):
             "time",
             "wavelength",
             "polariser_bleed_through",
+            "ray_accumulation_time",
+            "pulses_per_ray",
             "elevation",
             "range",
         )
@@ -175,6 +177,14 @@ def fix_long_names(self):
             self.nc.variables[
                 "polariser_bleed_through"
             ].long_name = "Polariser bleed-through"
+        if "pulses_per_ray" in self.nc.variables:
+            self.nc.variables[
+                "pulses_per_ray"
+            ].long_name = "Pulses per ray"
+        if "ray_accumulation_time" in self.nc.variables:
+            self.nc.variables[
+                "ray_accumulation_time"
+            ].long_name = "Ray accumulation time"
         if "beta_cross" in self.nc.variables:
             self.nc.variables[
                 "beta_cross"

src/processing/instrument_process.py

@@ -756,9 +756,9 @@ def _fix_cl51_timestamps(filename: str, offset: datetime.timedelta) -> None:
 def _doppy_stare_to_nc(
     stare: doppy.product.Stare | doppy.product.StareDepol, filename: str
 ) -> None:
-    with doppy.netcdf.Dataset(filename) as nc:
+    with doppy.netcdf.Dataset(filename, format="NETCDF4_CLASSIC") as nc:
         nc.add_dimension("time")
-        nc.add_dimension("range")
+        nc.add_dimension("range",size=len(stare.radial_distance))
         nc.add_time(
             name="time",
             dimensions=("time",),
@@ -780,7 +780,7 @@ def _doppy_stare_to_nc(
             units="degrees",
             data=stare.elevation,
             dtype="f4",
-            long_name="elevation from horizontal",
+            long_name="Elevation from horizontal",
         )
         nc.add_variable(
             name="beta_raw",
@@ -795,7 +795,7 @@ def _doppy_stare_to_nc(
             units="sr-1 m-1",
             data=stare.beta,
             dtype="f4",
-            mask=stare.mask,
+            mask=stare.mask_beta,
         )
         nc.add_variable(
             name="v",
@@ -804,7 +804,7 @@ def _doppy_stare_to_nc(
             long_name="Doppler velocity",
             data=stare.radial_velocity,
             dtype="f4",
-            mask=stare.mask,
+            mask=stare.mask_radial_velocity,
         )
         nc.add_scalar_variable(
             name="wavelength",
@@ -828,7 +828,7 @@ def _doppy_stare_to_nc(
                 units="1",
                 data=stare.depolarisation,
                 dtype="f4",
-                mask=stare.mask | stare.mask_depolarisation,
+                mask=stare.mask_beta | stare.mask_depolarisation,
             )
             nc.add_variable(
                 name="beta_cross_raw",
@@ -843,7 +843,7 @@ def _doppy_stare_to_nc(
                 dimensions=("time", "range"),
                 units="sr-1 m-1",
                 data=stare.beta_cross,
-                mask=stare.mask | stare.mask_beta_cross,
+                mask=stare.mask_beta | stare.mask_beta_cross,
                 dtype="f4",
             )
             nc.add_scalar_variable(
@@ -855,14 +855,31 @@ def _doppy_stare_to_nc(
             )
         nc.add_attribute("serial_number", stare.system_id)
         nc.add_attribute("doppy_version", doppy.__version__)
+        match stare.ray_info:
+            case doppy.product.stare.RayAccumulationTime(value):
+                nc.add_scalar_variable(
+                    name="ray_accumulation_time",
+                    units="s",
+                    long_name="Ray accumulation time",
+                    data=value,
+                    dtype="f4",
+                )
+            case doppy.product.stare.PulsesPerRay(value):
+                nc.add_scalar_variable(
+                    name="pulses_per_ray",
+                    units="1",
+                    long_name="Pulses per ray",
+                    data=value,
+                    dtype="i4",
+                )
 
 
 def _doppy_wind_to_nc(
     wind: doppy.product.Wind, filename: str, options: doppy.product.WindOptions | None
 ) -> None:
-    with doppy.netcdf.Dataset(filename) as nc:
+    with doppy.netcdf.Dataset(filename,format="NETCDF4_CLASSIC") as nc:
         nc.add_dimension("time")
-        nc.add_dimension("height")
+        nc.add_dimension("height", size=len(wind.height))
         nc.add_time(
             name="time",
             dimensions=("time",),
@@ -929,9 +946,9 @@ def _doppy_wind_to_nc(
 def _doppy_wls70_wind_to_nc(
     wind: doppy.product.Wind, filename: str, options: doppy.product.WindOptions | None
 ) -> None:
-    with doppy.netcdf.Dataset(filename) as nc:
+    with doppy.netcdf.Dataset(filename,format="NETCDF4_CLASSIC") as nc:
         nc.add_dimension("time")
-        nc.add_dimension("height")
+        nc.add_dimension("height", size=len(wind.height))
         nc.add_time(
             name="time",
             dimensions=("time",),

src/processing/product.py

@@ -10,9 +10,10 @@
 from cloudnetpy.model_evaluation.products import product_resampling
 from cloudnetpy.products import generate_mwr_multi, generate_mwr_single
 from orbital_radar import Suborbital
+from cloudnetpy.products.epsilon import generate_epsilon_from_lidar
 from requests import HTTPError
 
-from processing import utils
+from processing import instrument, utils
 from processing.netcdf_comparer import NCDiff, nc_difference
 from processing.processor import ModelParams, Processor, ProductParams
 from processing.utils import SkipTaskError, Uuid
@@ -104,6 +105,8 @@ def process_product(processor: Processor, params: ProductParams, directory: Path
             new_file = _process_categorize(processor, params, uuid, directory)
         elif params.product.id == "cpr-simulation":
             new_file = _process_cpr_simulation(processor, params, uuid, directory)
+        elif params.product.id == "epsilon-lidar":
+            new_file = _process_epsilon_from_lidar(processor, params, uuid, directory)
         else:
             new_file = _process_level2(processor, params, uuid, directory)
     except CloudnetException as err:
@@ -174,7 +177,8 @@ def _generate_filename(params: ProductParams | ModelParams) -> str:
 def _process_mwrpy(
     processor: Processor, params: ProductParams, uuid: Uuid, directory: Path
 ) -> Path:
-    assert params.instrument is not None
+    if params.instrument is None:
+        raise RuntimeError("Instrument is None")
     payload = _get_payload(
         params,
         product_id="mwr-l1c",
@@ -217,6 +221,33 @@ def _process_cpr_simulation(
         str(categorize_file), str(output_file), mean_wind=6, uuid=uuid.volatile
     )
     utils.add_global_attributes(output_file)
+
+
+def _process_epsilon_from_lidar(
+    processor: Processor, params: ProductParams, uuid: Uuid, directory: Path
+) -> Path:
+    if params.instrument is None:
+        raise RuntimeError("Instrument is None")
+    payload_stare = _get_payload(
+        params,
+        product_id="doppler-lidar",
+        instrument_pid=params.instrument.pid,
+    )
+    metadata_stare = processor.md_api.get("api/files", payload_stare)
+    _check_response(metadata_stare, "doppler-lidar")
+
+    payload_wind = _get_payload(
+        params,
+        product_id="doppler-lidar-wind",
+    )
+    metadata_wind = processor.md_api.get("api/files", payload_wind)
+
+    _check_response(metadata_wind, "doppler-lidar-wind")
+
+    file_lidar, file_wind = processor.storage_api.download_products([metadata_stare[0],metadata_wind[0]], directory)
+
+    output_file = directory / "output.nc"
+    uuid.product = generate_epsilon_from_lidar(file_lidar, file_wind, str(output_file), uuid.volatile)
     return output_file