Skip to content

Refactor covid_hosp auto columns #1203

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 22 commits into from
Jul 24, 2023
Merged

Refactor covid_hosp auto columns #1203

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
1fb30c7
Account for dynamic YAML in covid_hosp tests
rzats Jun 21, 2023
7324875
Refactor covid_hosp database files
rzats Jun 21, 2023
d4d233f
Refactor covid_hosp network files
rzats Jun 21, 2023
70aa3d1
Don't forget a newline
rzats Jun 21, 2023
904a370
Simplify update.py structure
rzats Jun 26, 2023
53a9bb5
Move Columndef
rzats Jun 26, 2023
4027ea1
Move METADATA_ID to property, be stricter with unsubclassed instances
rzats Jun 26, 2023
c50d4ce
Also be stricter with default args in constructor
rzats Jun 26, 2023
bfd00c3
Specify issue column in YAML file
rzats Jun 26, 2023
d3d77a3
Another round of changes
rzats Jun 30, 2023
6ed19b0
Update database constructor
rzats Jul 5, 2023
6e9f0b3
Update tests
rzats Jul 5, 2023
83f4888
Update tests & TestDatabase class
rzats Jul 7, 2023
a10a872
Clean up logger usage
rzats Jul 7, 2023
9f18b73
Review fixes
rzats Jul 14, 2023
fcc1983
Further review fixes
rzats Jul 14, 2023
5b134d3
Simplify CovidHospTestCase
rzats Jul 18, 2023
c08517b
formatting+cleanup in src/acquisition/covid_hosp/common/test_utils.py
melange396 Jul 18, 2023
4fd29aa
Move utils-related classes
rzats Jul 18, 2023
c1b34f9
Merge branch 'rzatserkovnyi/covid-hosp-auto-refactor' of https://gith…
rzats Jul 18, 2023
3a920ae
renamed Utils class to TypeUtils. also removed some related unused c…
melange396 Jul 18, 2023
882f50c
more TypeUtils name updates
melange396 Jul 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 25 additions & 36 deletions integrations/acquisition/covid_hosp/facility/test_scenarios.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,15 @@

# standard library
import unittest
from unittest.mock import MagicMock
from unittest.mock import patch

# first party
from delphi.epidata.acquisition.covid_hosp.common.database import Database
from delphi.epidata.acquisition.covid_hosp.facility.database import Database
from delphi.epidata.acquisition.covid_hosp.common.network import Network
from delphi.epidata.acquisition.covid_hosp.common.test_utils import UnitTestUtils
from delphi.epidata.acquisition.covid_hosp.common.utils import Utils
from delphi.epidata.client.delphi_epidata import Epidata
from delphi.epidata.acquisition.covid_hosp.facility.update import Update
from delphi.epidata.common.covid_hosp.covid_hosp_schema_io import CovidHospSomething
import delphi.operations.secrets as secrets

# third party
Expand All @@ -35,7 +37,7 @@ def setUp(self):
secrets.db.epi = ('user', 'pass')

# clear relevant tables
with Database.connect() as db:
with Database().connect() as db:
with db.new_cursor() as cur:
cur.execute('truncate table covid_hosp_facility')
cur.execute('truncate table covid_hosp_facility_key')
Expand All @@ -45,22 +47,17 @@ def setUp(self):
def test_acquire_dataset(self):
"""Acquire a new dataset."""

# only mock out network calls to external hosts
mock_network = MagicMock()
mock_network.fetch_metadata.return_value = \
self.test_utils.load_sample_metadata()
mock_network.fetch_dataset.return_value = \
self.test_utils.load_sample_dataset()

# make sure the data does not yet exist
with self.subTest(name='no data yet'):
response = Epidata.covid_hosp_facility(
'450822', Epidata.range(20200101, 20210101))
self.assertEqual(response['result'], -2, response)

# acquire sample data into local database
with self.subTest(name='first acquisition'):
acquired = Update.run(network=mock_network)
with self.subTest(name='first acquisition'), \
patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()), \
patch.object(Network, 'fetch_dataset', return_value=self.test_utils.load_sample_dataset()):
acquired = Utils.update_dataset(Database)
self.assertTrue(acquired)

# make sure the data now exists
Expand Down Expand Up @@ -89,12 +86,14 @@ def test_acquire_dataset(self):
else:
self.assertEqual(row[k], v, f"row[{k}] is {row[k]} not {v}")

# expect 113 fields per row (114 database columns, except `id`)
self.assertEqual(len(row), 113)
# Expect len(row) to equal the amount of dynamic columns + one extra issue column
self.assertEqual(len(row), len(list(CovidHospSomething().columns('covid_hosp_facility'))) + 1)

# re-acquisition of the same dataset should be a no-op
with self.subTest(name='second acquisition'):
acquired = Update.run(network=mock_network)
with self.subTest(name='second acquisition'), \
patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()), \
patch.object(Network, 'fetch_dataset', return_value=self.test_utils.load_sample_dataset()):
acquired = Utils.update_dataset(Database)
self.assertFalse(acquired)

# make sure the data still exists
Expand All @@ -108,16 +107,11 @@ def test_acquire_dataset(self):
def test_facility_lookup(self):
"""Lookup facilities using various filters."""

# only mock out network calls to external hosts
mock_network = MagicMock()
mock_network.fetch_metadata.return_value = \
self.test_utils.load_sample_metadata()
mock_network.fetch_dataset.return_value = \
self.test_utils.load_sample_dataset()

# acquire sample data into local database
with self.subTest(name='first acquisition'):
acquired = Update.run(network=mock_network)
with self.subTest(name='first acquisition'), \
patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()), \
patch.object(Network, 'fetch_dataset', return_value=self.test_utils.load_sample_dataset()):
acquired = Utils.update_dataset(Database)
self.assertTrue(acquired)

# texas ground truth, sorted by `hospital_pk`
Expand Down Expand Up @@ -181,16 +175,11 @@ def test_facility_lookup(self):
response = Epidata.covid_hosp_facility_lookup(state='not a state')
self.assertEqual(response['result'], -2)

# update facility info
mock_network = MagicMock()
mock_network.fetch_metadata.return_value = \
self.test_utils.load_sample_metadata('metadata_update_facility.csv')
mock_network.fetch_dataset.return_value = \
self.test_utils.load_sample_dataset('dataset_update_facility.csv')

# acquire sample data into local database
with self.subTest(name='second acquisition'):
acquired = Update.run(network=mock_network)
# acquire sample data into local database with updated facility info
with self.subTest(name='second acquisition'), \
patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata('metadata_update_facility.csv')), \
patch.object(Network, 'fetch_dataset', return_value=self.test_utils.load_sample_dataset('dataset_update_facility.csv')):
acquired = Utils.update_dataset(Database)
self.assertTrue(acquired)

texas_hospitals[1]['zip'] = '88888'
Expand Down
21 changes: 10 additions & 11 deletions integrations/acquisition/covid_hosp/state_daily/test_scenarios.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@
from delphi.epidata.acquisition.covid_hosp.state_daily.database import Database
from delphi.epidata.acquisition.covid_hosp.common.test_utils import UnitTestUtils
from delphi.epidata.client.delphi_epidata import Epidata
from delphi.epidata.acquisition.covid_hosp.state_daily.update import Update
from delphi.epidata.acquisition.covid_hosp.state_daily.network import Network
from delphi.epidata.acquisition.covid_hosp.common.network import Network
from delphi.epidata.acquisition.covid_hosp.common.utils import Utils
from delphi.epidata.common.covid_hosp.covid_hosp_schema_io import CovidHospSomething
import delphi.operations.secrets as secrets

# py3tester coverage target (equivalent to `import *`)
Expand All @@ -39,7 +39,7 @@ def setUp(self):
secrets.db.epi = ('user', 'pass')

# clear relevant tables
with Database.connect() as db:
with Database().connect() as db:
with db.new_cursor() as cur:
cur.execute('truncate table covid_hosp_state_daily')
cur.execute('truncate table covid_hosp_state_timeseries')
Expand All @@ -62,7 +62,7 @@ def test_acquire_dataset(self):
self.test_utils.load_sample_dataset("dataset0.csv"), # first dataset for 3/15
self.test_utils.load_sample_dataset()] # second dataset for 3/15
) as mock_fetch:
acquired = Update.run()
acquired = Utils.update_dataset(Database)
self.assertTrue(acquired)
self.assertEqual(mock_fetch_meta.call_count, 1)

Expand All @@ -82,8 +82,8 @@ def test_acquire_dataset(self):
self.assertAlmostEqual(actual, expected)
self.assertIsNone(row['critical_staffing_shortage_today_no'])

# expect 61 fields per row (62 database columns, except `id`) # TODO: ??? this is wrong!
self.assertEqual(len(row), 118)
# Expect len(row) to equal the amount of dynamic columns + one extra issue column
self.assertEqual(len(row), len(list(CovidHospSomething().columns('state_daily'))) + 1)

with self.subTest(name='all date batches acquired'):
response = Epidata.covid_hosp('WY', Epidata.range(20200101, 20210101), issues=20210313)
Expand All @@ -93,7 +93,7 @@ def test_acquire_dataset(self):
with self.subTest(name='second acquisition'), \
patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()) as mock_fetch_meta, \
patch.object(Network, 'fetch_dataset', return_value=self.test_utils.load_sample_dataset()) as mock_fetch:
acquired = Update.run()
acquired = Utils.update_dataset(Database)
self.assertFalse(acquired)

# make sure the data still exists
Expand All @@ -114,18 +114,17 @@ def test_acquire_specific_issue(self):

# acquire sample data into local database
# mock out network calls to external hosts
with Database.connect() as db:
with Database().connect() as db:
pre_max_issue = db.get_max_issue()
self.assertEqual(pre_max_issue, pd.Timestamp('1900-01-01 00:00:00'))
with self.subTest(name='first acquisition'), \
patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()) as mock_fetch_meta, \
patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()), \
patch.object(Network, 'fetch_dataset', side_effect=[self.test_utils.load_sample_dataset("dataset0.csv")]
) as mock_fetch:
acquired = Utils.update_dataset(Database,
Network,
date(2021, 3, 12),
date(2021, 3, 14))
with Database.connect() as db:
with Database().connect() as db:
post_max_issue = db.get_max_issue()
self.assertEqual(post_max_issue, pd.Timestamp('2021-03-13 00:00:00'))
self.assertTrue(acquired)
42 changes: 20 additions & 22 deletions integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,15 @@
# standard library
import unittest
from unittest.mock import MagicMock
from unittest.mock import patch

# first party
from delphi.epidata.acquisition.covid_hosp.common.database import Database
from delphi.epidata.acquisition.covid_hosp.state_timeseries.database import Database
from delphi.epidata.acquisition.covid_hosp.common.network import Network
from delphi.epidata.acquisition.covid_hosp.common.test_utils import UnitTestUtils
from delphi.epidata.acquisition.covid_hosp.common.utils import Utils
from delphi.epidata.client.delphi_epidata import Epidata
from delphi.epidata.acquisition.covid_hosp.state_timeseries.update import Update
from delphi.epidata.common.covid_hosp.covid_hosp_schema_io import CovidHospSomething
import delphi.operations.secrets as secrets

# third party
Expand All @@ -35,7 +38,7 @@ def setUp(self):
secrets.db.epi = ('user', 'pass')

# clear relevant tables
with Database.connect() as db:
with Database().connect() as db:
with db.new_cursor() as cur:
cur.execute('truncate table covid_hosp_state_daily')
cur.execute('truncate table covid_hosp_state_timeseries')
Expand All @@ -45,21 +48,16 @@ def setUp(self):
def test_acquire_dataset(self):
"""Acquire a new dataset."""

# only mock out network calls to external hosts
mock_network = MagicMock()
mock_network.fetch_metadata.return_value = \
self.test_utils.load_sample_metadata()
mock_network.fetch_dataset.return_value = \
self.test_utils.load_sample_dataset()

# make sure the data does not yet exist
with self.subTest(name='no data yet'):
response = Epidata.covid_hosp('MA', Epidata.range(20200101, 20210101))
self.assertEqual(response['result'], -2)

# acquire sample data into local database
with self.subTest(name='first acquisition'):
acquired = Update.run(network=mock_network)
with self.subTest(name='first acquisition'), \
patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()), \
patch.object(Network, 'fetch_dataset', return_value=self.test_utils.load_sample_dataset()):
acquired = Utils.update_dataset(Database)
self.assertTrue(acquired)

# make sure the data now exists
Expand All @@ -78,12 +76,14 @@ def test_acquire_dataset(self):
self.assertAlmostEqual(actual, expected)
self.assertIsNone(row['critical_staffing_shortage_today_no'])

# expect 61 fields per row (62 database columns, except `id`) # TODO: ??? this is wrong!
self.assertEqual(len(row), 118)
# Expect len(row) to equal the amount of dynamic columns + one extra issue column
self.assertEqual(len(row), len(list(CovidHospSomething().columns('state_timeseries'))) + 1)

# re-acquisition of the same dataset should be a no-op
with self.subTest(name='second acquisition'):
acquired = Update.run(network=mock_network)
with self.subTest(name='second acquisition'), \
patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()), \
patch.object(Network, 'fetch_dataset', return_value=self.test_utils.load_sample_dataset()):
acquired = Utils.update_dataset(Database)
self.assertFalse(acquired)

# make sure the data still exists
Expand All @@ -93,13 +93,11 @@ def test_acquire_dataset(self):
self.assertEqual(len(response['epidata']), 1)

# acquire new data into local database
with self.subTest(name='first acquisition'):
with self.subTest(name='updated acquisition'), \
patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata("metadata2.csv")), \
patch.object(Network, 'fetch_dataset', return_value=self.test_utils.load_sample_dataset("dataset2.csv")):
# acquire new data with 3/16 issue date
mock_network.fetch_metadata.return_value = \
self.test_utils.load_sample_metadata("metadata2.csv")
mock_network.fetch_dataset.return_value = \
self.test_utils.load_sample_dataset("dataset2.csv")
acquired = Update.run(network=mock_network)
acquired = Utils.update_dataset(Database)
self.assertTrue(acquired)

with self.subTest(name='as_of checks'):
Expand Down
6 changes: 3 additions & 3 deletions integrations/server/test_covid_hosp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ def setUp(self):
secrets.db.epi = ('user', 'pass')

# clear relevant tables
with Database.connect() as db:
with Database().connect() as db:
with db.new_cursor() as cur:
cur.execute('truncate table covid_hosp_state_daily')
cur.execute('truncate table covid_hosp_state_timeseries')
Expand All @@ -45,7 +45,7 @@ def insert_daily(self, cur, issue, value):
def test_query_by_issue(self):
"""Query with and without specifying an issue."""

with Database.connect() as db:
with Database().connect() as db:
with db.new_cursor() as cur:
# inserting out of order to test server-side order by
# also inserting two for 20201201 to test tiebreaker.
Expand Down Expand Up @@ -91,7 +91,7 @@ def test_query_by_issue(self):


def test_query_by_as_of(self):
with Database.connect() as db:
with Database().connect() as db:
with db.new_cursor() as cur:
self.insert_timeseries(cur, 20201101, 0)
self.insert_daily(cur, 20201102, 1)
Expand Down
Loading