Update time coarsening configurations for 4-degree datasets and era5

scripts/data_process/combine_stats.py

@@ -1,4 +1,5 @@
 import dataclasses
+import logging
 import shutil
 import tempfile
 from typing import Dict, List
@@ -65,6 +66,16 @@ def open_datasets(roots: List[str], filename: str) -> List[xr.Dataset]:
     return datasets
 
 
+def _combined_stats_exist(output_directory: str, subdirectory: str) -> bool:
+    """Check if combined stats already exist."""
+    path = output_directory + f"/{subdirectory}/centering.nc"
+    if output_directory.startswith("gs:"):
+        fs = fsspec.filesystem("gs")
+        return fs.exists(path)
+    else:
+        return fsspec.filesystem("file").exists(path)
+
+
 def combine_stats(
     stats_roots: list[str],
     output_directory: str,
@@ -77,6 +88,13 @@ def combine_stats(
     Args:
         stats_roots: List of root directories with the stats to combine.
     """
+    if _combined_stats_exist(output_directory, subdirectory):
+        logging.info(
+            f"Combined stats already exist at "
+            f"{output_directory}/{subdirectory}. Skipping."
+        )
+        return
+
     with tempfile.TemporaryDirectory() as tmpdir, xr.set_options(keep_attrs=True):
         for filename in (
             "centering.nc",
@@ -148,6 +166,7 @@ def main(config_yaml: str):
     Arguments:
     config_yaml -- Path to the configuration file for the data processing pipeline.
     """
+    logging.basicConfig(level=logging.INFO)
     with open(config_yaml, "r") as f:
         config_data = yaml.load(f, Loader=yaml.CLoader)
     config = dacite.from_dict(data_class=Config, data=config_data)

scripts/data_process/configs/CM4-1pctCO2-atmosphere-4deg-8layer-140yr.yaml

@@ -108,3 +108,104 @@ dataset_computation:
       - temperature_of_soil_layer
     land_names_to_vertically_coarsen_by_sum:
       - total_moisture_content_of_soil_layer
+time_coarsen:
+  factor: 4
+  data_output_directory: gs://vcm-ml-intermediate/2025-06-18-CM4-1pctCO2-atmosphere-land-4deg-8layer-140yr-daily
+  stats_output_directory: gs://vcm-ml-intermediate/2025-06-18-CM4-1pctCO2-atmosphere-land-4deg-8layer-140yr-daily-stats
+  snapshot_names:
+    - PRESsfc
+    - PRMSL
+    - Q2m
+    - TMP2m
+    - TMP850
+    - UGRD10m
+    - VGRD10m
+    - air_temperature_0
+    - air_temperature_1
+    - air_temperature_2
+    - air_temperature_3
+    - air_temperature_4
+    - air_temperature_5
+    - air_temperature_6
+    - air_temperature_7
+    - column_soil_moisture
+    - eastward_wind_0
+    - eastward_wind_1
+    - eastward_wind_2
+    - eastward_wind_3
+    - eastward_wind_4
+    - eastward_wind_5
+    - eastward_wind_6
+    - eastward_wind_7
+    - h500
+    - northward_wind_0
+    - northward_wind_1
+    - northward_wind_2
+    - northward_wind_3
+    - northward_wind_4
+    - northward_wind_5
+    - northward_wind_6
+    - northward_wind_7
+    - sea_ice_fraction
+    - specific_humidity_0
+    - specific_humidity_1
+    - specific_humidity_2
+    - specific_humidity_3
+    - specific_humidity_4
+    - specific_humidity_5
+    - specific_humidity_6
+    - specific_humidity_7
+    - specific_total_water_0
+    - specific_total_water_1
+    - specific_total_water_2
+    - specific_total_water_3
+    - specific_total_water_4
+    - specific_total_water_5
+    - specific_total_water_6
+    - specific_total_water_7
+    - surface_snow_amount
+    - surface_snow_area_fraction
+    - surface_temperature
+    - temperature_of_soil_layer_0
+    - temperature_of_soil_layer_1
+    - temperature_of_soil_layer_2
+    - temperature_of_soil_layer_3
+    - total_land_column_energy_without_rivers
+    - total_moisture_content_of_soil_layer_0
+    - total_moisture_content_of_soil_layer_1
+    - total_moisture_content_of_soil_layer_2
+    - total_moisture_content_of_soil_layer_3
+    - total_water_path
+    - total_energy
+  window_names:
+    - DLWRFsfc
+    - DSWRFsfc
+    - DSWRFtoa
+    - GHTFLsfc
+    - LHTFLsfc
+    - PRATEsfc
+    - SHTFLsfc
+    - ULWRFsfc
+    - ULWRFtoa
+    - USWRFsfc
+    - USWRFtoa
+    - carbon_dioxide
+    - downward_longwave_radiative_flux
+    - eastward_surface_wind_stress
+    - net_surface_longwave_radiative_flux
+    - northward_surface_wind_stress
+    - tendency_of_total_water_path
+    - tendency_of_total_water_path_due_to_advection
+    - total_frozen_precipitation_rate
+    - total_precipitation_heat_flux
+    - total_vapor_sensible_heat_flux
+  constant_prefixes:
+    - HGTsfc
+    - ak_
+    - bk_
+    - glac_fraction
+    - lake_fraction
+    - land_fraction
+    - lat
+    - lon
+    - soil_fraction

scripts/data_process/configs/CM4-like-AM4-random-CO2-ensemble-atmosphere-4deg-8layer.yaml

@@ -115,3 +115,104 @@ dataset_computation:
       - temperature_of_soil_layer
     land_names_to_vertically_coarsen_by_sum:
       - total_moisture_content_of_soil_layer
+time_coarsen:
+  factor: 4
+  data_output_directory: gs://vcm-ml-intermediate/2025-12-02-CM4-like-AM4-random-CO2-4deg-daily
+  stats_output_directory: gs://vcm-ml-intermediate/2025-12-02-CM4-like-AM4-random-CO2-4deg-daily-stats
+  snapshot_names:
+    - PRESsfc
+    - PRMSL
+    - Q2m
+    - TMP2m
+    - TMP850
+    - UGRD10m
+    - VGRD10m
+    - air_temperature_0
+    - air_temperature_1
+    - air_temperature_2
+    - air_temperature_3
+    - air_temperature_4
+    - air_temperature_5
+    - air_temperature_6
+    - air_temperature_7
+    - column_soil_moisture
+    - eastward_wind_0
+    - eastward_wind_1
+    - eastward_wind_2
+    - eastward_wind_3
+    - eastward_wind_4
+    - eastward_wind_5
+    - eastward_wind_6
+    - eastward_wind_7
+    - h500
+    - northward_wind_0
+    - northward_wind_1
+    - northward_wind_2
+    - northward_wind_3
+    - northward_wind_4
+    - northward_wind_5
+    - northward_wind_6
+    - northward_wind_7
+    - sea_ice_fraction
+    - specific_humidity_0
+    - specific_humidity_1
+    - specific_humidity_2
+    - specific_humidity_3
+    - specific_humidity_4
+    - specific_humidity_5
+    - specific_humidity_6
+    - specific_humidity_7
+    - specific_total_water_0
+    - specific_total_water_1
+    - specific_total_water_2
+    - specific_total_water_3
+    - specific_total_water_4
+    - specific_total_water_5
+    - specific_total_water_6
+    - specific_total_water_7
+    - surface_snow_amount
+    - surface_snow_area_fraction
+    - surface_temperature
+    - temperature_of_soil_layer_0
+    - temperature_of_soil_layer_1
+    - temperature_of_soil_layer_2
+    - temperature_of_soil_layer_3
+    - total_land_column_energy_without_rivers
+    - total_moisture_content_of_soil_layer_0
+    - total_moisture_content_of_soil_layer_1
+    - total_moisture_content_of_soil_layer_2
+    - total_moisture_content_of_soil_layer_3
+    - total_water_path
+    - total_energy
+  window_names:
+    - DLWRFsfc
+    - DSWRFsfc
+    - DSWRFtoa
+    - GHTFLsfc
+    - LHTFLsfc
+    - PRATEsfc
+    - SHTFLsfc
+    - ULWRFsfc
+    - ULWRFtoa
+    - USWRFsfc
+    - USWRFtoa
+    - carbon_dioxide
+    - downward_longwave_radiative_flux
+    - eastward_surface_wind_stress
+    - net_surface_longwave_radiative_flux
+    - northward_surface_wind_stress
+    - tendency_of_total_water_path
+    - tendency_of_total_water_path_due_to_advection
+    - total_frozen_precipitation_rate
+    - total_precipitation_heat_flux
+    - total_vapor_sensible_heat_flux
+  constant_prefixes:
+    - HGTsfc
+    - ak_
+    - bk_
+    - glac_fraction
+    - lake_fraction
+    - land_fraction
+    - lat
+    - lon
+    - soil_fraction

scripts/data_process/configs/CM4-piControl-atmosphere-4deg-8layer-200yr.yaml

@@ -108,3 +108,104 @@ dataset_computation:
       - temperature_of_soil_layer
     land_names_to_vertically_coarsen_by_sum:
       - total_moisture_content_of_soil_layer
+time_coarsen:
+  factor: 4
+  data_output_directory: gs://vcm-ml-intermediate/2025-03-21-CM4-piControl-atmosphere-land-4deg-8layer-200yr-daily
+  stats_output_directory: gs://vcm-ml-intermediate/2025-03-21-CM4-piControl-atmosphere-land-4deg-8layer-200yr-daily-stats
+  snapshot_names:
+    - PRESsfc
+    - PRMSL
+    - Q2m
+    - TMP2m
+    - TMP850
+    - UGRD10m
+    - VGRD10m
+    - air_temperature_0
+    - air_temperature_1
+    - air_temperature_2
+    - air_temperature_3
+    - air_temperature_4
+    - air_temperature_5
+    - air_temperature_6
+    - air_temperature_7
+    - column_soil_moisture
+    - eastward_wind_0
+    - eastward_wind_1
+    - eastward_wind_2
+    - eastward_wind_3
+    - eastward_wind_4
+    - eastward_wind_5
+    - eastward_wind_6
+    - eastward_wind_7
+    - h500
+    - northward_wind_0
+    - northward_wind_1
+    - northward_wind_2
+    - northward_wind_3
+    - northward_wind_4
+    - northward_wind_5
+    - northward_wind_6
+    - northward_wind_7
+    - sea_ice_fraction
+    - specific_humidity_0
+    - specific_humidity_1
+    - specific_humidity_2
+    - specific_humidity_3
+    - specific_humidity_4
+    - specific_humidity_5
+    - specific_humidity_6
+    - specific_humidity_7
+    - specific_total_water_0
+    - specific_total_water_1
+    - specific_total_water_2
+    - specific_total_water_3
+    - specific_total_water_4
+    - specific_total_water_5
+    - specific_total_water_6
+    - specific_total_water_7
+    - surface_snow_amount
+    - surface_snow_area_fraction
+    - surface_temperature
+    - temperature_of_soil_layer_0
+    - temperature_of_soil_layer_1
+    - temperature_of_soil_layer_2
+    - temperature_of_soil_layer_3
+    - total_land_column_energy_without_rivers
+    - total_moisture_content_of_soil_layer_0
+    - total_moisture_content_of_soil_layer_1
+    - total_moisture_content_of_soil_layer_2
+    - total_moisture_content_of_soil_layer_3
+    - total_water_path
+    - total_energy
+  window_names:
+    - DLWRFsfc
+    - DSWRFsfc
+    - DSWRFtoa
+    - GHTFLsfc
+    - LHTFLsfc
+    - PRATEsfc
+    - SHTFLsfc
+    - ULWRFsfc
+    - ULWRFtoa
+    - USWRFsfc
+    - USWRFtoa
+    - carbon_dioxide
+    - downward_longwave_radiative_flux
+    - eastward_surface_wind_stress
+    - net_surface_longwave_radiative_flux
+    - northward_surface_wind_stress
+    - tendency_of_total_water_path
+    - tendency_of_total_water_path_due_to_advection
+    - total_frozen_precipitation_rate
+    - total_precipitation_heat_flux
+    - total_vapor_sensible_heat_flux
+  constant_prefixes:
+    - HGTsfc
+    - ak_
+    - bk_
+    - glac_fraction
+    - lake_fraction
+    - land_fraction
+    - lat
+    - lon
+    - soil_fraction