From c55f7f866db41e8ff4886248a068a7a5ecc19942 Mon Sep 17 00:00:00 2001 From: Nat DeFries <42820733+nmdefries@users.noreply.github.com> Date: Wed, 27 Mar 2024 19:10:32 -0400 Subject: [PATCH 1/9] calculate min_issue in metadata table --- src/acquisition/covidcast/database.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/acquisition/covidcast/database.py b/src/acquisition/covidcast/database.py index 871061b81..3f3752199 100644 --- a/src/acquisition/covidcast/database.py +++ b/src/acquisition/covidcast/database.py @@ -473,6 +473,7 @@ def compute_covidcast_meta(self, table_name=None, n_threads=None): ROUND(AVG(`value`),7) AS `mean_value`, ROUND(STD(`value`),7) AS `stdev_value`, MAX(`value_updated_timestamp`) AS `last_update`, + MIN(`issue`) as `min_issue`, MAX(`issue`) as `max_issue`, MIN(`lag`) as `min_lag`, MAX(`lag`) as `max_lag` From cf9dc4d54ceb68a9682ae99a544e871468b6a39e Mon Sep 17 00:00:00 2001 From: Nat DeFries <42820733+nmdefries@users.noreply.github.com> Date: Wed, 27 Mar 2024 19:10:45 -0400 Subject: [PATCH 2/9] describe min_issue in docs and update example return value --- docs/api/covidcast_meta.md | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/api/covidcast_meta.md b/docs/api/covidcast_meta.md index 08c3990c8..47a9a98cd 100644 --- a/docs/api/covidcast_meta.md +++ b/docs/api/covidcast_meta.md @@ -38,6 +38,7 @@ None required. | `epidata[].max_value` | maximum value | float | | `epidata[].mean_value` | mean of value | float | | `epidata[].stdev_value` | standard deviation of value | float | +| `epidata[].min_issue` | oldest date data was issued (e.g., 20200710) | integer | | `epidata[].max_issue` | most recent date data was issued (e.g., 20200710) | integer | | `epidata[].min_lag` | smallest lag from observation to issue, in `time_type` units | integer | | `epidata[].max_lag` | largest lag from observation to issue, in `time_type` units | integer | @@ -51,19 +52,22 @@ https://api.delphi.cmu.edu/epidata/covidcast_meta/ { "result": 1, "epidata": [ - { + { "data_source": "doctor-visits", "signal": "smoothed_adj_cli", - "last_update": 1592707979, - "stdev_value": 2.6647410028331, - "num_locations": 2500, "time_type": "day", - "max_value": 87.190476, - "mean_value": 1.4439366759191, "geo_type": "county", - "min_value": 0, - "max_time": 20200617, - "min_time": 20200201 + "min_time": 20200201, + "max_time": 20240321, + "num_locations": 2586, + "min_value": 0.0, + "max_value": 87.670832, + "mean_value": 2.4972178, + "stdev_value": 3.6433783, + "last_update": 1711412545, + "max_issue": 20240325, + "min_lag": 2, + "max_lag": 129 }, ... ], From 39289e119f139fa0fa2caac3bc98b1bba03726a8 Mon Sep 17 00:00:00 2001 From: Nat DeFries <42820733+nmdefries@users.noreply.github.com> Date: Thu, 28 Mar 2024 12:53:52 -0400 Subject: [PATCH 3/9] update tests to expect min_issue --- .../acquisition/covidcast/test_covidcast_meta_caching.py | 1 + integrations/client/test_delphi_epidata.py | 1 + integrations/server/test_covidcast_meta.py | 1 + 3 files changed, 3 insertions(+) diff --git a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py index 30b6bbcf2..70b5f120f 100644 --- a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py +++ b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py @@ -137,6 +137,7 @@ def test_caching(self): 'max_value': 1, 'mean_value': 1, 'stdev_value': 0, + 'min_issue': 20200422, 'max_issue': 20200423, 'min_lag': 0, 'max_lag': 1, diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 2af923b2b..a78f85948 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -319,6 +319,7 @@ def test_covidcast_meta(self): mean_value=12., max_value=22., stdev_value=8.1649658, # population stdev, not sample, which is 10. + min_issue=DEFAULT_ISSUE, max_issue=DEFAULT_ISSUE + 2, min_lag=0, max_lag=0, # we didn't set lag when inputting data diff --git a/integrations/server/test_covidcast_meta.py b/integrations/server/test_covidcast_meta.py index 857422a41..e082afef3 100644 --- a/integrations/server/test_covidcast_meta.py +++ b/integrations/server/test_covidcast_meta.py @@ -129,6 +129,7 @@ def insert_placeholder_data(self): 'mean_value': 15, 'stdev_value': 5, 'last_update': 123, + 'min_issue': 1, 'max_issue': 2, 'min_lag': 0, 'max_lag': 0, From 5f64b54eb0d448a2e6415a992ef546723f77f73c Mon Sep 17 00:00:00 2001 From: Nat DeFries <42820733+nmdefries@users.noreply.github.com> Date: Thu, 9 May 2024 17:58:34 -0400 Subject: [PATCH 4/9] insert one row at a time --- integrations/client/test_delphi_epidata.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index a78f85948..bcafd5801 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -283,15 +283,14 @@ def test_covidcast_meta(self): # 1st issue: 0 10 20 # 2nd issue: 1 11 21 # 3rd issue: 2 12 22 - rows = [ - CovidcastTestRow.make_default_row( - time_value=DEFAULT_TIME_VALUE + t, - issue=DEFAULT_ISSUE + i, - value=t*10 + i - ) - for i in range(3) for t in range(3) - ] - self._insert_rows(rows) + for i in range(3): + for t in range(3): + row = CovidcastTestRow.make_default_row( + time_value=DEFAULT_TIME_VALUE + t, + issue=DEFAULT_ISSUE + i, + value=t*10 + i + ) + self._insert_rows(row) # cache it update_covidcast_meta_cache(args=None) From 44d1f5e55d5a6239a0c62c933c6247f9d35889ba Mon Sep 17 00:00:00 2001 From: Nat DeFries <42820733+nmdefries@users.noreply.github.com> Date: Thu, 9 May 2024 18:04:47 -0400 Subject: [PATCH 5/9] insert row as a len-1 list --- integrations/client/test_delphi_epidata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index bcafd5801..6d0869a3d 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -290,7 +290,7 @@ def test_covidcast_meta(self): issue=DEFAULT_ISSUE + i, value=t*10 + i ) - self._insert_rows(row) + self._insert_rows([row]) # cache it update_covidcast_meta_cache(args=None) From f994cdffba4d1f736bfda99889d301dcae793f00 Mon Sep 17 00:00:00 2001 From: Nat DeFries <42820733+nmdefries@users.noreply.github.com> Date: Thu, 9 May 2024 18:27:59 -0400 Subject: [PATCH 6/9] use last row for metadata --- integrations/client/test_delphi_epidata.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 6d0869a3d..3942710c6 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -307,10 +307,10 @@ def test_covidcast_meta(self): del response['epidata'][0]['last_update'] expected = dict( - data_source=rows[0].source, - signal=rows[0].signal, - time_type=rows[0].time_type, - geo_type=rows[0].geo_type, + data_source=row.source, + signal=row.signal, + time_type=row.time_type, + geo_type=row.geo_type, min_time=DEFAULT_TIME_VALUE, max_time=DEFAULT_TIME_VALUE + 2, num_locations=1, From b388cdb32ca599e8a405aaec5681cda3e81e2018 Mon Sep 17 00:00:00 2001 From: nmdefries <42820733+nmdefries@users.noreply.github.com> Date: Fri, 10 May 2024 17:56:46 -0400 Subject: [PATCH 7/9] add old fake row Co-authored-by: george --- docs/api/covidcast_meta.md | 1 + integrations/client/test_delphi_epidata.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/docs/api/covidcast_meta.md b/docs/api/covidcast_meta.md index 47a9a98cd..dc88284ee 100644 --- a/docs/api/covidcast_meta.md +++ b/docs/api/covidcast_meta.md @@ -65,6 +65,7 @@ https://api.delphi.cmu.edu/epidata/covidcast_meta/ "mean_value": 2.4972178, "stdev_value": 3.6433783, "last_update": 1711412545, + "min_issue": 20200507, "max_issue": 20240325, "min_lag": 2, "max_lag": 129 diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 3942710c6..87ab99e7f 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -291,6 +291,11 @@ def test_covidcast_meta(self): value=t*10 + i ) self._insert_rows([row]) + self._insert_rows([CovidCastTestRow.make_default_row( + time_value=DEFAULT_TIME_VALUE-1, + issue=DEFAULT_ISSUE, + value=12 + )]) # cache it update_covidcast_meta_cache(args=None) From baf920e461605b90c62568fd1b49d6a05114bc9c Mon Sep 17 00:00:00 2001 From: Nat DeFries <42820733+nmdefries@users.noreply.github.com> Date: Fri, 10 May 2024 19:20:14 -0400 Subject: [PATCH 8/9] CovidcastTestRow spelling; new min_time --- integrations/client/test_delphi_epidata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 87ab99e7f..846871da4 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -291,7 +291,7 @@ def test_covidcast_meta(self): value=t*10 + i ) self._insert_rows([row]) - self._insert_rows([CovidCastTestRow.make_default_row( + self._insert_rows([CovidcastTestRow.make_default_row( time_value=DEFAULT_TIME_VALUE-1, issue=DEFAULT_ISSUE, value=12 @@ -316,7 +316,7 @@ def test_covidcast_meta(self): signal=row.signal, time_type=row.time_type, geo_type=row.geo_type, - min_time=DEFAULT_TIME_VALUE, + min_time=DEFAULT_TIME_VALUE - 1, max_time=DEFAULT_TIME_VALUE + 2, num_locations=1, min_value=2., From a62b773bd472104988522f376a556bd337f598ec Mon Sep 17 00:00:00 2001 From: Nat DeFries <42820733+nmdefries@users.noreply.github.com> Date: Fri, 10 May 2024 19:43:43 -0400 Subject: [PATCH 9/9] new stddev --- integrations/client/test_delphi_epidata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 846871da4..0cbda279d 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -322,7 +322,7 @@ def test_covidcast_meta(self): min_value=2., mean_value=12., max_value=22., - stdev_value=8.1649658, # population stdev, not sample, which is 10. + stdev_value=7.0710678, # population stdev, not sample, which is 10. min_issue=DEFAULT_ISSUE, max_issue=DEFAULT_ISSUE + 2, min_lag=0,