readme and logging

Author: nikoleskinen

scripts/monitor

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from monitoring.cli import main
+
+if __name__ == "__main__":
+    main()

src/monitoring/README.md

@@ -0,0 +1,29 @@
+# Monitoring
+
+## Usage
+
+```bash
+# Monitor current period by default
+./scripts/monitor {day,week,month,year,all}
+
+# Common options
+# start and stop arguments have format YYYY-MM-DD, YYYY-VV, YYYY-MM or YYYY
+# depending on the subcommand
+--start START               Monitor periods starting from START
+--stop STOP                 Monitor periods until STOP
+--product [PRODUCT ...]     Monitor only these products
+--site [SITE ...]           Monitor only these sites
+
+# Subcommand specific options
+## day
+--day [YYYY-MM-DD ...]
+
+## week
+--week [YYYY-VV ...]
+
+## month
+--month [YYYY-MM ...]
+
+## year
+--year [YYYY ...]
+```

src/monitoring/cli.py

@@ -26,8 +26,8 @@
 
 
 def main() -> None:
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
     args = _get_args()
-
     period_cls = period_cls_from_str(args.cmd)
     periods = build_periods(period_cls, args)
     periods_with_products = build_products(period_cls, periods, args.product)

src/monitoring/instrument/halo_doppler_lidar.py

@@ -5,6 +5,7 @@
 
 import matplotlib.pyplot as plt
 import numpy as np
+from cloudnet_api_client.client import APIClient
 from cloudnetpy.plotting.plotting import Dimensions
 from doppy.raw import HaloSysParams
 from doppy.raw.halo_bg import HaloBg
@@ -25,36 +26,50 @@
 from monitoring.product import MonitoringProduct, MonitoringVariable
 from monitoring.utils import (
     RawFilesDatePayload,
-    get_api_client,
     get_storage_api,
     instrument_uuid_to_pid,
 )
+from processing.storage_api import StorageApi
 
 
 def monitor(
-    period: PeriodType, product: MonitoringProduct, site: str, instrument_uuid: str
+    period: PeriodType,
+    product: MonitoringProduct,
+    site: str,
+    instrument_uuid: str,
+    api_client: APIClient,
+    storage_api: StorageApi,
 ) -> None:
     match product.id:
         case "halo-doppler-lidar_housekeeping":
-            monitor_housekeeping(period, product, site, instrument_uuid)
+            monitor_housekeeping(
+                period, product, site, instrument_uuid, api_client, storage_api
+            )
         case "halo-doppler-lidar_background":
-            monitor_background(period, product, site, instrument_uuid)
+            monitor_background(
+                period, product, site, instrument_uuid, api_client, storage_api
+            )
         case "halo-doppler-lidar_signal":
-            monitor_signal(period, product, site, instrument_uuid)
+            monitor_signal(
+                period, product, site, instrument_uuid, api_client, storage_api
+            )
 
 
 def monitor_housekeeping(
-    period: PeriodType, product: MonitoringProduct, site: str, instrument_uuid: str
+    period: PeriodType,
+    product: MonitoringProduct,
+    site: str,
+    instrument_uuid: str,
+    api_client: APIClient,
+    storage_api: StorageApi,
 ) -> None:
-    client = get_api_client()
-    storage_api = get_storage_api()
-    pid = instrument_uuid_to_pid(client, instrument_uuid)
+    pid = instrument_uuid_to_pid(api_client, instrument_uuid)
     date_opts: RawFilesDatePayload = {}
     if not isinstance(period, All):
         start, stop = period.to_interval_padded(days=31)
         date_opts = {"date_from": start, "date_to": stop}
 
-    records = client.raw_files(
+    records = api_client.raw_files(
         site_id=site,
         instrument_pid=pid,
         filename_prefix="system_parameters_",
@@ -117,22 +132,28 @@ def plot_housekeeping_variable(
     format_time_axis(ax)
     pretty_ax(ax, grid="y")
     fig_ = save_fig(fig)
-    return MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax]))
+    vis = MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax]))
+    plt.close(fig)
+    return vis
 
 
 def monitor_background(
-    period: PeriodType, product: MonitoringProduct, site: str, instrument_uuid: str
+    period: PeriodType,
+    product: MonitoringProduct,
+    site: str,
+    instrument_uuid: str,
+    api_client: APIClient,
+    storage_api: StorageApi,
 ) -> None:
-    client = get_api_client()
     storage_api = get_storage_api()
-    pid = instrument_uuid_to_pid(client, instrument_uuid)
+    pid = instrument_uuid_to_pid(api_client, instrument_uuid)
 
     date_opts: RawFilesDatePayload = {}
     if not isinstance(period, All):
         start, stop = period.to_interval_padded(days=1)
         date_opts = {"date_from": start, "date_to": stop}
 
-    records = client.raw_files(
+    records = api_client.raw_files(
         site_id=site,
         instrument_pid=pid,
         filename_prefix="Background_",
@@ -211,7 +232,9 @@ def plot_background_profile(
     format_time_axis(ax)
     pretty_ax_2d(ax)
     fig_ = save_fig(fig)
-    return MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax]))
+    vis = MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax]))
+    plt.close(fig)
+    return vis
 
 
 def plot_background_variance(
@@ -224,7 +247,9 @@ def plot_background_variance(
     format_time_axis(ax)
     pretty_ax(ax, grid="y")
     fig_ = save_fig(fig)
-    return MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax]))
+    vis = MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax]))
+    plt.close(fig)
+    return vis
 
 
 def plot_time_averaged_background_profile(
@@ -244,22 +269,27 @@ def plot_time_averaged_background_profile(
 
     pretty_ax(ax, grid="y")
     fig_ = save_fig(fig)
-    return MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax]))
+    vis = MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax]))
+    plt.close(fig)
+    return vis
 
 
 def monitor_signal(
-    period: PeriodType, product: MonitoringProduct, site: str, instrument_uuid: str
+    period: PeriodType,
+    product: MonitoringProduct,
+    site: str,
+    instrument_uuid: str,
+    api_client: APIClient,
+    storage_api: StorageApi,
 ) -> None:
-    client = get_api_client()
-    storage_api = get_storage_api()
-    pid = instrument_uuid_to_pid(client, instrument_uuid)
+    pid = instrument_uuid_to_pid(api_client, instrument_uuid)
 
     date_opts: RawFilesDatePayload = {}
     if not isinstance(period, All):
         start, stop = period.to_interval()
         date_opts = {"date_from": start, "date_to": stop}
 
-    records = client.raw_files(
+    records = api_client.raw_files(
         site_id=site,
         instrument_pid=pid,
         filename_suffix=".hpl",
@@ -353,7 +383,9 @@ def plot_radial_velocity_histogram(
     ax.set_ylabel("Count")
     pretty_ax(ax, grid="both")
     fig_ = save_fig(fig)
-    return MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax]))
+    vis = MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax]))
+    plt.close(fig)
+    return vis
 
 
 def _compute_radial_velocity_bins(raw: HaloHpl) -> list[float] | int:
@@ -382,4 +414,6 @@ def plot_signal_radial_velocity(
     ax.set_ylabel("Radial velocity")
     pretty_ax(ax)
     fig_ = save_fig(fig)
-    return MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax]))
+    vis = MonitoringVisualization(fig_.bytes, variable, Dimensions(fig, [ax]))
+    plt.close(fig)
+    return vis

src/monitoring/monitor.py

@@ -1,14 +1,20 @@
+import logging
+
 from monitoring.instrument.halo_doppler_lidar import (
     monitor as monitor_halo,
 )
 from monitoring.period import PeriodType
 from monitoring.product import MonitoringProduct
+from monitoring.utils import get_api_client, get_storage_api
 
 
 def monitor(
     period: PeriodType, product: MonitoringProduct, site: str, instrument_uuid: str
 ) -> None:
+    api_client = get_api_client()
+    storage_api = get_storage_api()
     if product.id.startswith("halo-doppler-lidar"):
-        monitor_halo(period, product, site, instrument_uuid)
+        monitor_halo(period, product, site, instrument_uuid, api_client, storage_api)
+        logging.info(f"Monitored {period!r} {product} {site}")
     else:
         raise ValueError(f"Unexpected product: '{product}'")

src/monitoring/period.py

@@ -57,6 +57,9 @@ def to_interval(self) -> tuple[datetime.date, datetime.date]:
     def to_interval_padded(self, days: int) -> tuple[datetime.date, datetime.date]:
         return pad_interval(self.to_interval(), days)
 
+    def __repr__(self) -> str:
+        return f"Day({self.date})"
+
 
 @dataclass(order=True, frozen=True)
 class Week:
@@ -99,6 +102,9 @@ def to_interval(self) -> tuple[datetime.date, datetime.date]:
     def to_interval_padded(self, days: int) -> tuple[datetime.date, datetime.date]:
         return pad_interval(self.to_interval(), days)
 
+    def __repr__(self) -> str:
+        return f"Week({self.year}-{self.week:02})"
+
 
 @dataclass(order=True, frozen=True)
 class Month:
@@ -139,6 +145,9 @@ def to_interval(self) -> tuple[datetime.date, datetime.date]:
     def to_interval_padded(self, days: int) -> tuple[datetime.date, datetime.date]:
         return pad_interval(self.to_interval(), days)
 
+    def __repr__(self) -> str:
+        return f"Month({self.year}-{self.month:02})"
+
 
 @dataclass(order=True, frozen=True)
 class Year:
@@ -173,12 +182,18 @@ def to_interval(self) -> tuple[datetime.date, datetime.date]:
     def to_interval_padded(self, days: int) -> tuple[datetime.date, datetime.date]:
         return pad_interval(self.to_interval(), days)
 
+    def __repr__(self) -> str:
+        return f"Year({self.year})"
+
 
 class All:
     @classmethod
     def to_str(cls) -> Literal["all"]:
         return "all"
 
+    def __repr__(self) -> str:
+        return f"All"
+
 
 PeriodType = PeriodProtocol | All