From 87586941bdc52d8570a2b1a0f58e606cabad2169 Mon Sep 17 00:00:00 2001 From: Luis Alvergue Date: Mon, 21 Jul 2025 16:12:14 +0000 Subject: [PATCH 1/9] chore(5-min-viz): remove unused placeholders and borders --- .../districts/templates/districts/district.html | 7 +------ .../pems_web/districts/templates/districts/index.html | 10 +--------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/pems_web/src/pems_web/districts/templates/districts/district.html b/pems_web/src/pems_web/districts/templates/districts/district.html index 43695912..c7fe2374 100644 --- a/pems_web/src/pems_web/districts/templates/districts/district.html +++ b/pems_web/src/pems_web/districts/templates/districts/district.html @@ -5,13 +5,8 @@ {% endblock headline %} {% block districts-content %} -
-
-

Form

-
-
-
+
From 555757009423c49f0b65da5db50aecedf481da4b Mon Sep 17 00:00:00 2001 From: Luis Alvergue Date: Tue, 22 Jul 2025 15:16:33 -0500 Subject: [PATCH 2/9] feat(5-min-viz): add map and station summary --- .../apps/stations/app_stations.py | 9 +++++++- .../components/map_station_summary.py | 23 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 pems_streamlit/src/pems_streamlit/components/map_station_summary.py diff --git a/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py b/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py index 547dfaad..b1d1c8ac 100644 --- a/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py +++ b/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py @@ -5,6 +5,8 @@ from pems_data import ServiceFactory +from pems_streamlit.components.map_station_summary import map_station_summary + FACTORY = ServiceFactory() STATIONS = FACTORY.stations_service() S3 = FACTORY.s3_source @@ -47,13 +49,18 @@ def main(): district_number = query_params.get("district_number", "") df_station_metadata = load_station_metadata(district_number) - st.dataframe(df_station_metadata, use_container_width=True) + + map_placeholder = st.empty() station = st.selectbox( "Station", df_station_metadata["STATION_ID"], ) + with map_placeholder: + df_selected_station = df_station_metadata.query("STATION_ID == @station") + map_station_summary(df_selected_station) + days = st.multiselect("Days", get_available_days()) station_data_button = st.button("Load Station Data", type="primary") diff --git a/pems_streamlit/src/pems_streamlit/components/map_station_summary.py b/pems_streamlit/src/pems_streamlit/components/map_station_summary.py new file mode 100644 index 00000000..a232d8f3 --- /dev/null +++ b/pems_streamlit/src/pems_streamlit/components/map_station_summary.py @@ -0,0 +1,23 @@ +import pandas as pd +import streamlit as st + + +def map_station_summary(df_station_metadata: pd.DataFrame): + + map_col, info_col = st.columns([0.6, 0.4]) + + with map_col: + map_df = df_station_metadata.rename(columns={"LATITUDE": "latitude", "LONGITUDE": "longitude"}) + map_df_cleaned = map_df.dropna(subset=["latitude", "longitude"]) + st.map(map_df_cleaned[["latitude", "longitude"]], height=265) + + with info_col: + with st.container(border=True): + st.markdown(f"**Station {df_station_metadata['STATION_ID'].item()} - {df_station_metadata['NAME'].item()}**") + st.markdown( + f"{df_station_metadata["FREEWAY"].item()} - {df_station_metadata["DIRECTION"].item()}, {df_station_metadata["CITY_NAME"].item()}" + ) + st.markdown(f"**County** {df_station_metadata["COUNTY_NAME"].item()}") + st.markdown(f"**District** {df_station_metadata["DISTRICT"].item()}") + st.markdown(f"**Absolute Post Mile** {df_station_metadata["ABSOLUTE_POSTMILE"].item()}") + st.markdown(f"**Lanes** {df_station_metadata["PHYSICAL_LANES"].item()}") From bcbaf98045efd7391f5458d513c45bf8f07145da Mon Sep 17 00:00:00 2001 From: Luis Alvergue Date: Tue, 29 Jul 2025 11:40:14 +0000 Subject: [PATCH 3/9] feat(5-min-viz): add 5-minute traffic data visualization --- pems_streamlit/pyproject.toml | 1 + .../apps/stations/app_stations.py | 14 ++++ .../components/plot_5_min_traffic_data.py | 72 +++++++++++++++++++ 3 files changed, 87 insertions(+) create mode 100644 pems_streamlit/src/pems_streamlit/components/plot_5_min_traffic_data.py diff --git a/pems_streamlit/pyproject.toml b/pems_streamlit/pyproject.toml index e6ceb4ef..83db6625 100644 --- a/pems_streamlit/pyproject.toml +++ b/pems_streamlit/pyproject.toml @@ -8,6 +8,7 @@ dependencies = [ # local package reference # a wheel for this package is built during Docker build "pems_data", + "plotly==6.2.0", "streamlit==1.45.1", ] diff --git a/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py b/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py index b1d1c8ac..c984dcb5 100644 --- a/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py +++ b/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py @@ -6,6 +6,7 @@ from pems_data import ServiceFactory from pems_streamlit.components.map_station_summary import map_station_summary +from pems_streamlit.components.plot_5_min_traffic_data import plot_5_min_traffic_data FACTORY = ServiceFactory() STATIONS = FACTORY.stations_service() @@ -57,6 +58,14 @@ def main(): df_station_metadata["STATION_ID"], ) + quantity = st.multiselect("Quantity", ["VOLUME_SUM", "OCCUPANCY_AVG", "SPEED_FIVE_MINS"]) + + num_lanes = int(df_station_metadata[df_station_metadata["STATION_ID"] == station]["PHYSICAL_LANES"].iloc[0]) + lane = st.multiselect( + "Lane", + list(range(1, num_lanes + 1)), + ) + with map_placeholder: df_selected_station = df_station_metadata.query("STATION_ID == @station") map_station_summary(df_selected_station) @@ -67,7 +76,12 @@ def main(): if station_data_button: df_station_data = load_station_data(station) + filtered_df = df_station_data[ + (df_station_data["SAMPLE_TIMESTAMP"].dt.day.isin(days)) & (df_station_data["LANE"].isin(lane)) + ] st.dataframe(df_station_data, use_container_width=True) + filtered_df_sorted = filtered_df.sort_values(by="SAMPLE_TIMESTAMP") + plot_5_min_traffic_data(filtered_df_sorted, quantity, lane) if __name__ == "__main__": diff --git a/pems_streamlit/src/pems_streamlit/components/plot_5_min_traffic_data.py b/pems_streamlit/src/pems_streamlit/components/plot_5_min_traffic_data.py new file mode 100644 index 00000000..e839d3de --- /dev/null +++ b/pems_streamlit/src/pems_streamlit/components/plot_5_min_traffic_data.py @@ -0,0 +1,72 @@ +import pandas as pd +import plotly.graph_objs as go +import streamlit as st + +QUANTITY_CONFIG = { + "VOLUME_SUM": {"name": "Volume (veh/hr)"}, + "OCCUPANCY_AVG": {"name": "Occupancy (%)"}, + "SPEED_FIVE_MINS": {"name": "Speed (mph)"}, +} + + +def plot_5_min_traffic_data(df_station_data: pd.DataFrame, quantities: list, lanes: list): + fig = go.Figure() + + layout_updates = { + "xaxis": dict(title="Time of Day"), + "legend": dict(orientation="h", yanchor="top", y=-0.3, xanchor="center", x=0.5), + } + + # One quantity selected + if len(quantities) == 1: + qty_key = quantities[0] + qty_name = QUANTITY_CONFIG[qty_key]["name"] + + for lane in lanes: + df_lane = df_station_data[df_station_data["LANE"] == lane] + fig.add_trace( + go.Scatter( + x=df_lane["SAMPLE_TIMESTAMP"], + y=df_lane[qty_key], + mode="lines", + name=f"Lane {lane} {qty_name.split(' ')[0]}", + ) + ) + + layout_updates["title"] = dict(text=f"{qty_name}", x=0.5, xanchor="center") + layout_updates["yaxis"] = dict(title=f"{qty_name}", side="left") + + # Two quantities selected + elif len(quantities) == 2: + left_qty_key, right_qty_key = quantities[0], quantities[1] + left_qty_name = QUANTITY_CONFIG[left_qty_key]["name"] + right_qty_name = QUANTITY_CONFIG[right_qty_key]["name"] + + for lane in lanes: + df_lane = df_station_data[df_station_data["LANE"] == lane] + fig.add_trace( + go.Scatter( + x=df_lane["SAMPLE_TIMESTAMP"], + y=df_lane[left_qty_key], + mode="lines", + name=f"Lane {lane} {left_qty_name.split(' ')[0]}", + ) + ) + fig.add_trace( + go.Scatter( + x=df_lane["SAMPLE_TIMESTAMP"], + y=df_lane[right_qty_key], + mode="lines", + name=f"Lane {lane} {right_qty_name.split(' ')[0]}", + yaxis="y2", + ) + ) + + # Create layout for two axes + layout_updates["title"] = dict(text=f"{left_qty_name} vs. {right_qty_name}", x=0.5, xanchor="center") + layout_updates["yaxis"] = dict(title=f"{left_qty_name}", side="left") + layout_updates["yaxis2"] = dict(title=f"{right_qty_name}", side="right", overlaying="y") + + fig.update_layout(**layout_updates) + + st.plotly_chart(fig, use_container_width=True) From 835fa6ea23e41e7e376b7079aa440e498359d289 Mon Sep 17 00:00:00 2001 From: Luis Alvergue Date: Tue, 29 Jul 2025 19:02:25 +0000 Subject: [PATCH 4/9] feat(5-min-viz): add checks to visualization ensure that the required parameters are set before proceeding with the visualization --- .../apps/stations/app_stations.py | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py b/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py index c984dcb5..ce5cc907 100644 --- a/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py +++ b/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py @@ -75,13 +75,24 @@ def main(): station_data_button = st.button("Load Station Data", type="primary") if station_data_button: - df_station_data = load_station_data(station) - filtered_df = df_station_data[ - (df_station_data["SAMPLE_TIMESTAMP"].dt.day.isin(days)) & (df_station_data["LANE"].isin(lane)) - ] - st.dataframe(df_station_data, use_container_width=True) - filtered_df_sorted = filtered_df.sort_values(by="SAMPLE_TIMESTAMP") - plot_5_min_traffic_data(filtered_df_sorted, quantity, lane) + error_messages = [] + if len(quantity) == 0 or len(quantity) > 2: + error_messages.append("- Please select one or two quantities to proceed.") + if not lane: + error_messages.append("- Please select at least one lane to proceed.") + if not days: + error_messages.append("- Please select at least one day to proceed.") + if error_messages: + full_error_message = "\n".join(error_messages) + st.error(full_error_message) + else: + df_station_data = load_station_data(station) + filtered_df = df_station_data[ + (df_station_data["SAMPLE_TIMESTAMP"].dt.day.isin(days)) & (df_station_data["LANE"].isin(lane)) + ] + st.dataframe(df_station_data, use_container_width=True) + filtered_df_sorted = filtered_df.sort_values(by="SAMPLE_TIMESTAMP") + plot_5_min_traffic_data(filtered_df_sorted, quantity, lane) if __name__ == "__main__": From 9ea94a621492a83e998967fc8c1490af5598492c Mon Sep 17 00:00:00 2001 From: Luis Alvergue Date: Tue, 29 Jul 2025 19:25:06 +0000 Subject: [PATCH 5/9] chore(5-min-viz): remove data table the data table is not needed in the UI at this point --- pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py b/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py index ce5cc907..0f4596d8 100644 --- a/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py +++ b/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py @@ -90,7 +90,6 @@ def main(): filtered_df = df_station_data[ (df_station_data["SAMPLE_TIMESTAMP"].dt.day.isin(days)) & (df_station_data["LANE"].isin(lane)) ] - st.dataframe(df_station_data, use_container_width=True) filtered_df_sorted = filtered_df.sort_values(by="SAMPLE_TIMESTAMP") plot_5_min_traffic_data(filtered_df_sorted, quantity, lane) From c3392d55dcbfd681096b08a665da9dba42e45038 Mon Sep 17 00:00:00 2001 From: Luis Alvergue Date: Tue, 29 Jul 2025 19:40:57 +0000 Subject: [PATCH 6/9] fix(5-min-viz): remove stale elements from previous run remove "greyed out" plot and error messages from the previous run. to ensure only one plot or error message is shown, draw it inside a dedicated placeholder. --- .../src/pems_streamlit/apps/stations/app_stations.py | 9 +++++++-- .../pems_streamlit/components/plot_5_min_traffic_data.py | 3 +-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py b/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py index 0f4596d8..32cdec56 100644 --- a/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py +++ b/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py @@ -74,6 +74,9 @@ def main(): station_data_button = st.button("Load Station Data", type="primary") + error_placeholder = st.empty() + plot_placeholder = st.empty() + if station_data_button: error_messages = [] if len(quantity) == 0 or len(quantity) > 2: @@ -84,14 +87,16 @@ def main(): error_messages.append("- Please select at least one day to proceed.") if error_messages: full_error_message = "\n".join(error_messages) - st.error(full_error_message) + error_placeholder.error(full_error_message) else: df_station_data = load_station_data(station) filtered_df = df_station_data[ (df_station_data["SAMPLE_TIMESTAMP"].dt.day.isin(days)) & (df_station_data["LANE"].isin(lane)) ] filtered_df_sorted = filtered_df.sort_values(by="SAMPLE_TIMESTAMP") - plot_5_min_traffic_data(filtered_df_sorted, quantity, lane) + + fig = plot_5_min_traffic_data(filtered_df_sorted, quantity, lane) + plot_placeholder.plotly_chart(fig, use_container_width=True) if __name__ == "__main__": diff --git a/pems_streamlit/src/pems_streamlit/components/plot_5_min_traffic_data.py b/pems_streamlit/src/pems_streamlit/components/plot_5_min_traffic_data.py index e839d3de..e2f22c78 100644 --- a/pems_streamlit/src/pems_streamlit/components/plot_5_min_traffic_data.py +++ b/pems_streamlit/src/pems_streamlit/components/plot_5_min_traffic_data.py @@ -1,6 +1,5 @@ import pandas as pd import plotly.graph_objs as go -import streamlit as st QUANTITY_CONFIG = { "VOLUME_SUM": {"name": "Volume (veh/hr)"}, @@ -69,4 +68,4 @@ def plot_5_min_traffic_data(df_station_data: pd.DataFrame, quantities: list, lan fig.update_layout(**layout_updates) - st.plotly_chart(fig, use_container_width=True) + return fig From 36fd70923bfb551ca03156797bdd60ad91093acd Mon Sep 17 00:00:00 2001 From: Luis Alvergue Date: Tue, 29 Jul 2025 21:11:38 +0000 Subject: [PATCH 7/9] chore(5-min-viz): organize streamlit app place app_stations.py under districts to match pems_web and to use the more accurate streamlit app name "districts--stations" --- .../src/pems_streamlit/apps/{stations => districts}/__init__.py | 0 .../pems_streamlit/apps/{stations => districts}/app_stations.py | 0 .../src/pems_web/districts/templates/districts/district.html | 2 +- pems_web/src/pems_web/districts/templates/districts/index.html | 2 +- 4 files changed, 2 insertions(+), 2 deletions(-) rename pems_streamlit/src/pems_streamlit/apps/{stations => districts}/__init__.py (100%) rename pems_streamlit/src/pems_streamlit/apps/{stations => districts}/app_stations.py (100%) diff --git a/pems_streamlit/src/pems_streamlit/apps/stations/__init__.py b/pems_streamlit/src/pems_streamlit/apps/districts/__init__.py similarity index 100% rename from pems_streamlit/src/pems_streamlit/apps/stations/__init__.py rename to pems_streamlit/src/pems_streamlit/apps/districts/__init__.py diff --git a/pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py b/pems_streamlit/src/pems_streamlit/apps/districts/app_stations.py similarity index 100% rename from pems_streamlit/src/pems_streamlit/apps/stations/app_stations.py rename to pems_streamlit/src/pems_streamlit/apps/districts/app_stations.py diff --git a/pems_web/src/pems_web/districts/templates/districts/district.html b/pems_web/src/pems_web/districts/templates/districts/district.html index c7fe2374..c26c8ea2 100644 --- a/pems_web/src/pems_web/districts/templates/districts/district.html +++ b/pems_web/src/pems_web/districts/templates/districts/district.html @@ -9,7 +9,7 @@
diff --git a/pems_web/src/pems_web/districts/templates/districts/index.html b/pems_web/src/pems_web/districts/templates/districts/index.html index dbba2000..025073a9 100644 --- a/pems_web/src/pems_web/districts/templates/districts/index.html +++ b/pems_web/src/pems_web/districts/templates/districts/index.html @@ -16,7 +16,7 @@ {% block districts-content %}
-
From 0b22d3aec3f0a2ce81f145c6ede117985df28f4c Mon Sep 17 00:00:00 2001 From: Luis Alvergue Date: Thu, 31 Jul 2025 13:53:27 +0000 Subject: [PATCH 8/9] chore(cache): increase cache duration for station data setting the cache to 1 hour seems like a good balance between performance and data freshness. --- pems_data/src/pems_data/services/stations.py | 2 +- .../src/pems_streamlit/apps/districts/app_stations.py | 2 +- tests/pytest/pems_data/services/test_stations.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pems_data/src/pems_data/services/stations.py b/pems_data/src/pems_data/services/stations.py index efa818ef..e6eda50a 100644 --- a/pems_data/src/pems_data/services/stations.py +++ b/pems_data/src/pems_data/services/stations.py @@ -80,7 +80,7 @@ def get_imputed_agg_5min(self, station_id: str) -> pd.DataFrame: value (pandas.DataFrame): The station's data as a DataFrame. """ - cache_opts = {"key": self._build_cache_key("imputed", "agg", "5m", "station", station_id), "ttl": 300} # 5 minutes + cache_opts = {"key": self._build_cache_key("imputed", "agg", "5m", "station", station_id), "ttl": 3600} # 1 hour columns = [ "STATION_ID", "LANE", diff --git a/pems_streamlit/src/pems_streamlit/apps/districts/app_stations.py b/pems_streamlit/src/pems_streamlit/apps/districts/app_stations.py index 32cdec56..b3325aaf 100644 --- a/pems_streamlit/src/pems_streamlit/apps/districts/app_stations.py +++ b/pems_streamlit/src/pems_streamlit/apps/districts/app_stations.py @@ -34,7 +34,7 @@ def match(m: re.Match): return S3.get_prefixes(pattern, initial_prefix=STATIONS.imputation_detector_agg_5min, match_func=match) -@st.cache_data(ttl=300) # Cache for 5 minutes +@st.cache_data(ttl=3600) # Cache for 1 hour def load_station_data(station_id: str) -> pd.DataFrame: """ Loads station data for a specific station. diff --git a/tests/pytest/pems_data/services/test_stations.py b/tests/pytest/pems_data/services/test_stations.py index 32328e56..ca9507fe 100644 --- a/tests/pytest/pems_data/services/test_stations.py +++ b/tests/pytest/pems_data/services/test_stations.py @@ -74,6 +74,6 @@ def test_get_imputed_agg_5min(self, service: StationsService, data_source: IData cache_opts = data_source.read.call_args.kwargs["cache_opts"] assert station_id in cache_opts["key"] - assert cache_opts["ttl"] == 300 + assert cache_opts["ttl"] == 3600 pd.testing.assert_frame_equal(result, df) From 108056cdad5aa881722cf7f53ff6ef11311b46c1 Mon Sep 17 00:00:00 2001 From: Luis Alvergue Date: Thu, 31 Jul 2025 13:55:42 +0000 Subject: [PATCH 9/9] chore(5-min-viz): make components a Python subpackage --- pems_streamlit/src/pems_streamlit/components/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 pems_streamlit/src/pems_streamlit/components/__init__.py diff --git a/pems_streamlit/src/pems_streamlit/components/__init__.py b/pems_streamlit/src/pems_streamlit/components/__init__.py new file mode 100644 index 00000000..e69de29b