Skip to content

Commit 401b665

Browse files
borlowskyborlowsky
committed
moved some functions into client.py; returns now 3hr data; cronjob script
1 parent d097d4c commit 401b665

File tree

6 files changed

+151
-155
lines changed

6 files changed

+151
-155
lines changed

‎WWCS/dashboard/service/cronjob_OM.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
# Exit immediately if any command exits with a non-zero status
3+
set -e
4+
5+
# Exit immediately if a pipeline returns a non-zero status
6+
set -o pipefail
7+
8+
echo "=== CRON JOB ==="
9+
10+
cd /srv/shiny-server/dashboard/service
11+
12+
# Ensure the cronout folder exists
13+
mkdir -p cronout
14+
15+
echo "=== RETRIEVE EMCWF FORECASTS ==="
16+
17+
/home/wwcs/venv/bin/python3 get_open_meteo/get_open_meteo.py > cronout/open-meteo.out
18+
19+
/home/wwcs/venv/bin/python3 get_open_meteo/get_open_meteo_extended.py > cronout/open-meteo-ext.out
20+
21+
/home/wwcs/venv/bin/python3 get_open_meteo/get_open_meteo_grid.py > cronout/open-meteo-grd.out
22+
23+
/home/wwcs/venv/bin/python3 get_open_meteo/concatenate_open-meteo.py > cronout/open-meteo-concat.out
24+
25+
echo "=== GET OBSERVATIONS AND POSTPROCESS FORECASTS ==="
26+
27+
R CMD BATCH --no-save forecasts/EMOS.R cronout/emos.out
28+
29+
echo "=== PROCESS WEATHER PICTOGRAMS FROM FORECASTS ==="
30+
31+
R CMD BATCH --no-save forecasts/process_pictos.R cronout/pictos.out
32+
33+
R CMD BATCH --no-save forecasts/prepare_api_data.R cronout/api.out

‎WWCS/dashboard/service/get_open_meteo/client.py‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,32 @@
1+
from datetime import datetime, timedelta
2+
import pandas as pd
3+
import mysql.connector
4+
import numpy as np
5+
from openmeteo_sdk.Variable import Variable
6+
17
import openmeteo_requests
28
import requests_cache
39
import retry_requests
410

11+
from common import USERNAME, PASSWORD
12+
13+
def get_sites():
14+
with mysql.connector.connect(
15+
user=USERNAME,
16+
password=PASSWORD,
17+
host='127.0.0.1',
18+
database='SitesHumans',
19+
) as cnx:
20+
with cnx.cursor() as cursor:
21+
cursor.execute("SELECT siteID, latitude, longitude FROM Sites WHERE siteID NOT LIKE '%-S%'")
22+
return cursor.fetchall()
23+
24+
def enum_code_to_name(enum_cls, code: int):
25+
# enum members are stored as class attributes; reverse-lookup by value
26+
for name, value in enum_cls.__dict__.items():
27+
if value == code:
28+
return name
29+
return None
530

631
class Client:
732

@@ -10,6 +35,74 @@ def __init__(self):
1035
retry_session = retry_requests.retry(cache_session, retries=5, backoff_factor=0.2)
1136
self.client = openmeteo_requests.Client(session=retry_session)
1237

38+
def _ensemble_response_to_dataframe(response):
39+
"""Convert ensemble response to DataFrame with mean and std."""
40+
hourly = response.Hourly()
41+
42+
# Get time values
43+
time_values = pd.date_range(
44+
start=pd.to_datetime(hourly.Time(), unit="s", utc=True),
45+
end=pd.to_datetime(hourly.TimeEnd(), unit="s", utc=True),
46+
freq=pd.Timedelta(seconds=hourly.Interval()),
47+
inclusive="left"
48+
)
49+
50+
# Extract all temperature members
51+
hourly_variables = [hourly.Variables(i) for i in range(hourly.VariablesLength())]
52+
temp_vars = [v for v in hourly_variables
53+
if v.Variable() == Variable.temperature and v.Altitude() == 2]
54+
55+
# Stack all member values
56+
member_values = []
57+
for var in temp_vars:
58+
#member = var.EnsembleMember()
59+
values = var.ValuesAsNumpy()
60+
member_values.append(values)
61+
62+
# Calculate mean and std across ensemble members
63+
ensemble_array = np.stack(member_values) # Shape: (members, time)
64+
mean_values = np.mean(ensemble_array, axis=0)
65+
std_values = np.std(ensemble_array, axis=0)
66+
67+
# Build DataFrame
68+
df = pd.DataFrame({
69+
'time': time_values,
70+
'latitude': response.Latitude(),
71+
'longitude': response.Longitude(),
72+
'temperature_2m_mean': mean_values,
73+
'temperature_2m_std': std_values,
74+
})
75+
76+
# extract/return every 3rd hour
77+
return df[df["time"].dt.hour % 3 == 0]
78+
79+
def _response_to_dataframe(response):
80+
"""Convert openmeteo-requests response to pandas DataFrame."""
81+
hourly = response.Hourly()
82+
83+
time_values = pd.date_range(
84+
start=pd.to_datetime(hourly.Time(), unit="s", utc=True),
85+
end=pd.to_datetime(hourly.TimeEnd(), unit="s", utc=True),
86+
freq=pd.Timedelta(seconds=hourly.Interval()),
87+
inclusive="left"
88+
)
89+
90+
data = {
91+
'time': time_values,
92+
'latitude': response.Latitude(),
93+
'longitude': response.Longitude(),
94+
}
95+
96+
for i in range(hourly.VariablesLength()):
97+
var = hourly.Variables(i)
98+
var_name = enum_code_to_name(Variable, var.Variable())
99+
print(var_name)
100+
data[var_name] = var.ValuesAsNumpy()
101+
102+
# convert to panda and extract/return every 3rd hour
103+
df = pd.DataFrame(data)
104+
return df[df["time"].dt.hour % 3 == 0]
105+
13106
def ensemble(self, params: dict):
14107
url = "https://ensemble-api.open-meteo.com/v1/ensemble"
15108
responses = self.client.weather_api(url, params=params)
@@ -24,3 +117,10 @@ def forecast(self, params: dict):
24117
response = responses[0] # ! this is correct since we use only one model/ensemble !
25118
return response
26119

120+
def ensemble_df(self, params: dict):
121+
response = self.ensemble(params)
122+
return _ensemble_response_to_dataframe(response)
123+
124+
def forecast_df(self, params: dict):
125+
response = self.forecast(params)
126+
return _response_to_dataframe(response)

‎WWCS/dashboard/service/get_open_meteo/get_open_meteo.py‎

Lines changed: 8 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -1,104 +1,17 @@
1-
from datetime import datetime, timedelta
21
from pathlib import Path
2+
import netCDF4
33

4+
from datetime import datetime, timedelta
45
import pandas as pd
56
import mysql.connector
6-
import netCDF4
77
import numpy as np
88
from openmeteo_sdk.Variable import Variable
99

10-
from client import Client
11-
from common import USERNAME, PASSWORD
12-
13-
14-
client = Client()
15-
16-
def get_sites():
17-
with mysql.connector.connect(
18-
user=USERNAME,
19-
password=PASSWORD,
20-
host='127.0.0.1',
21-
database='SitesHumans',
22-
) as cnx:
23-
with cnx.cursor() as cursor:
24-
cursor.execute("SELECT siteID, latitude, longitude FROM Sites WHERE siteID NOT LIKE '%-S%'")
25-
return cursor.fetchall()
26-
27-
def ensemble_df(params: dict) -> pd.DataFrame:
28-
"""Get ensemble data and return DataFrame with mean and std."""
29-
response = client.ensemble(params)
30-
return _ensemble_response_to_dataframe(response)
31-
32-
def forecast_df(params: dict) -> pd.DataFrame:
33-
response = client.forecast(params)
34-
return _response_to_dataframe(response)
35-
36-
def _ensemble_response_to_dataframe(response) -> pd.DataFrame:
37-
"""Convert ensemble response to DataFrame with mean and std."""
38-
hourly = response.Hourly()
39-
40-
# Get time values
41-
time_values = pd.date_range(
42-
start=pd.to_datetime(hourly.Time(), unit="s", utc=True),
43-
end=pd.to_datetime(hourly.TimeEnd(), unit="s", utc=True),
44-
freq=pd.Timedelta(seconds=hourly.Interval()),
45-
inclusive="left"
46-
)
47-
48-
# Extract all temperature members
49-
hourly_variables = [hourly.Variables(i) for i in range(hourly.VariablesLength())]
50-
temp_vars = [v for v in hourly_variables
51-
if v.Variable() == Variable.temperature and v.Altitude() == 2]
52-
53-
# Stack all member values
54-
member_values = []
55-
for var in temp_vars:
56-
#member = var.EnsembleMember()
57-
values = var.ValuesAsNumpy()
58-
member_values.append(values)
59-
60-
# Calculate mean and std across ensemble members
61-
ensemble_array = np.stack(member_values) # Shape: (members, time)
62-
mean_values = np.mean(ensemble_array, axis=0)
63-
std_values = np.std(ensemble_array, axis=0)
64-
65-
# Build DataFrame
66-
df = pd.DataFrame({
67-
'time': time_values,
68-
'latitude': response.Latitude(),
69-
'longitude': response.Longitude(),
70-
'temperature_2m_mean': mean_values,
71-
'temperature_2m_std': std_values,
72-
})
73-
74-
return df
75-
76-
def _response_to_dataframe(response) -> pd.DataFrame:
77-
"""Convert openmeteo-requests response to pandas DataFrame."""
78-
hourly = response.Hourly()
79-
80-
time_values = pd.date_range(
81-
start=pd.to_datetime(hourly.Time(), unit="s", utc=True),
82-
end=pd.to_datetime(hourly.TimeEnd(), unit="s", utc=True),
83-
freq=pd.Timedelta(seconds=hourly.Interval()),
84-
inclusive="left"
85-
)
86-
87-
data = {
88-
'time': time_values,
89-
'latitude': response.Latitude(),
90-
'longitude': response.Longitude(),
91-
}
92-
93-
for i in range(hourly.VariablesLength()):
94-
var = hourly.Variables(i)
95-
var_name = f"variable_{i}"
96-
if i == 0:
97-
var_name = 'temperature_2m'
98-
data[var_name] = var.ValuesAsNumpy()
99-
100-
return pd.DataFrame(data)
10+
import client
11+
12+
10113

14+
om_client = client.Client()
10215

10316
def dataframe_to_netcdf(df: pd.DataFrame, filename: str, date_str: str):
10417
"""
@@ -170,7 +83,7 @@ def dataframe_to_netcdf(df: pd.DataFrame, filename: str, date_str: str):
17083
today = datetime.today().date()
17184
dates = [d.strftime("%Y-%m-%d") for d in pd.date_range(today - timedelta(days=total_days), today)]
17285

173-
sites = get_sites()
86+
sites = client.get_sites()
17487

17588
outdir = Path('ifsdata')
17689
outdir.mkdir(exist_ok=True)
@@ -190,7 +103,7 @@ def dataframe_to_netcdf(df: pd.DataFrame, filename: str, date_str: str):
190103
continue
191104

192105
# Use ensemble API instead of forecast
193-
df = ensemble_df({
106+
df = om_client.ensemble_df({
194107
'latitude': lat,
195108
'longitude': lon,
196109
'start_date': date_str,

‎WWCS/dashboard/service/get_open_meteo/get_open_meteo_extended.py‎

Lines changed: 6 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -1,65 +1,15 @@
1-
from datetime import datetime, timedelta
21
from pathlib import Path
2+
import netCDF4
33

4+
from datetime import datetime, timedelta
45
import pandas as pd
56
import mysql.connector
6-
import netCDF4
77
import numpy as np
88
from openmeteo_sdk.Variable import Variable
99

10-
from client import Client
11-
from common import USERNAME, PASSWORD
12-
13-
14-
client = Client()
15-
16-
def get_sites():
17-
with mysql.connector.connect(
18-
user=USERNAME,
19-
password=PASSWORD,
20-
host='127.0.0.1',
21-
database='SitesHumans',
22-
) as cnx:
23-
with cnx.cursor() as cursor:
24-
cursor.execute("SELECT siteID, latitude, longitude FROM Sites WHERE siteID NOT LIKE '%-S%'")
25-
return cursor.fetchall()
26-
27-
def forecast_df(params: dict) -> pd.DataFrame:
28-
response = client.forecast(params)
29-
return _response_to_dataframe(response)
30-
31-
def enum_code_to_name(enum_cls, code: int):
32-
# enum members are stored as class attributes; reverse-lookup by value
33-
for name, value in enum_cls.__dict__.items():
34-
if value == code:
35-
return name
36-
return None
37-
38-
def _response_to_dataframe(response) -> pd.DataFrame:
39-
"""Convert openmeteo-requests response to pandas DataFrame."""
40-
hourly = response.Hourly()
41-
42-
time_values = pd.date_range(
43-
start=pd.to_datetime(hourly.Time(), unit="s", utc=True),
44-
end=pd.to_datetime(hourly.TimeEnd(), unit="s", utc=True),
45-
freq=pd.Timedelta(seconds=hourly.Interval()),
46-
inclusive="left"
47-
)
48-
49-
data = {
50-
'time': time_values,
51-
'latitude': response.Latitude(),
52-
'longitude': response.Longitude(),
53-
}
54-
55-
for i in range(hourly.VariablesLength()):
56-
var = hourly.Variables(i)
57-
var_name = enum_code_to_name(Variable, var.Variable())
58-
print(var_name)
59-
data[var_name] = var.ValuesAsNumpy()
60-
61-
return pd.DataFrame(data)
10+
import client
6211

12+
om_client = client.Client()
6313

6414
def dataframe_to_netcdf(df: pd.DataFrame, filename: str, date_str: str):
6515
"""
@@ -167,7 +117,7 @@ def dataframe_to_netcdf(df: pd.DataFrame, filename: str, date_str: str):
167117
today = datetime.today().date()
168118
dates = [d.strftime("%Y-%m-%d") for d in pd.date_range(today - timedelta(days=total_days), today)]
169119

170-
sites = get_sites()
120+
sites = client.get_sites()
171121

172122
outdir = Path('ifsdata')
173123
outdir.mkdir(exist_ok=True)
@@ -187,7 +137,7 @@ def dataframe_to_netcdf(df: pd.DataFrame, filename: str, date_str: str):
187137
continue
188138

189139
# Use forecast API
190-
df = forecast_df({
140+
df = om_client.forecast_df({
191141
'latitude': lat,
192142
'longitude': lon,
193143
'start_date': date_str,

‎WWCS/dashboard/service/get_open_meteo/get_open_meteo_grid.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,16 @@
88
import yaml
99
from openmeteo_sdk.Variable import Variable
1010

11-
from client import Client
11+
import client
1212

1313

1414
#ROOT_PATH = Path("/home/wwcs/wwcs/WWCS")
15-
ROOT_PATH = Path("/home/boris/Documents/PV_Taj/wwcs/WWCS_repo/wwcs/WWCS")
15+
ROOT_PATH = Path("/home/boris/wwcs/WWCS_repo/wwcs/WWCS")
1616
#ROOT_PATH = Path("/home/jdavid/sandboxes/Caritas/wwcs/WWCS")
1717
CONFIG_PATH = ROOT_PATH / "config.yaml"
1818
DATA_PATH = ROOT_PATH / "dashboard" / "ifsdata"
1919

20-
client = Client()
20+
om_client = client.Client()
2121

2222
# ================
2323
# helper functions
@@ -54,7 +54,7 @@ def download_point(lat, lon, date_string):
5454
"""
5555

5656
# Create a fresh session every time (no caching)
57-
response = client.ensemble({
57+
response = om_client.ensemble({
5858
"latitude": lat,
5959
"longitude": lon,
6060
"hourly": "temperature_2m",

‎WWCS/dashboard/service/get_open_meteo/learn.py‎

Whitespace-only changes.

0 commit comments

Comments
 (0)