diff --git a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py index 30b6bbcf2..2495a2000 100644 --- a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py +++ b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py @@ -2,17 +2,10 @@ # standard library import json -import unittest - -# third party -import mysql.connector -import requests # first party from delphi_utils import Nans -from delphi.epidata.client.delphi_epidata import Epidata -import delphi.operations.secrets as secrets -import delphi.epidata.acquisition.covidcast.database as live +from delphi.epidata.common.covidcast_test_base import CovidcastTestBase from delphi.epidata.maintenance.covidcast_meta_cache_updater import main # py3tester coverage target (equivalent to `import *`) @@ -21,88 +14,26 @@ 'covidcast_meta_cache_updater' ) -# use the local instance of the Epidata API -BASE_URL = 'http://delphi_web_epidata/epidata' - -class CovidcastMetaCacheTests(unittest.TestCase): +class CovidcastMetaCacheTests(CovidcastTestBase): """Tests covidcast metadata caching.""" - def setUp(self): - """Perform per-test setup.""" - - # connect to the `epidata` database - cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='covid') - cur = cnx.cursor() - - # clear all tables - cur.execute("truncate table epimetric_load") - cur.execute("truncate table epimetric_full") - cur.execute("truncate table epimetric_latest") - cur.execute("truncate table geo_dim") - cur.execute("truncate table signal_dim") - # reset the `covidcast_meta_cache` table (it should always have one row) - cur.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') - cnx.commit() - cur.close() - - # make connection and cursor available to test cases - self.cnx = cnx - self.cur = cnx.cursor() - - # use the local instance of the epidata database - secrets.db.host = 'delphi_database_epidata' - secrets.db.epi = ('user', 'pass') - - epidata_cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='epidata') - epidata_cur = epidata_cnx.cursor() - - epidata_cur.execute("DELETE FROM `api_user`") - epidata_cur.execute('INSERT INTO `api_user`(`api_key`, `email`) VALUES("key", "email")') - epidata_cnx.commit() - epidata_cur.close() - epidata_cnx.close() - - # use the local instance of the Epidata API - Epidata.BASE_URL = BASE_URL - Epidata.auth = ('epidata', 'key') - - def tearDown(self): - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - - @staticmethod - def _make_request(): - params = {'cached': 'true'} - response = requests.get(f"{Epidata.BASE_URL}/covidcast_meta", params=params, auth=Epidata.auth) - response.raise_for_status() - return response.json() - def test_caching(self): """Populate, query, cache, query, and verify the cache.""" # insert dummy data - self.cur.execute(f''' + self._db._cursor.execute(''' INSERT INTO `signal_dim` (`signal_key_id`, `source`, `signal`) VALUES (42, 'src', 'sig'); ''') - self.cur.execute(f''' + self._db._cursor.execute(''' INSERT INTO `geo_dim` (`geo_key_id`, `geo_type`, `geo_value`) VALUES - (96, 'state', 'pa'), + (96, 'state', 'pa'), (97, 'state', 'wa'); ''') - self.cur.execute(f''' + self._db._cursor.execute(f''' INSERT INTO `epimetric_latest` (`epimetric_id`, `signal_key_id`, `geo_key_id`, `time_type`, `time_value`, `value_updated_timestamp`, @@ -115,14 +46,11 @@ def test_caching(self): (16, 42, 97, 'day', 20200422, 789, 1, 2, 3, 20200423, 1, {Nans.NOT_MISSING}, {Nans.NOT_MISSING}, {Nans.NOT_MISSING}) ''') - self.cnx.commit() + self._db._connection.commit() # make sure the live utility is serving something sensible - cvc_database = live.Database() - cvc_database.connect() - epidata1 = cvc_database.compute_covidcast_meta() - cvc_database.disconnect(False) - self.assertEqual(len(epidata1),1) + epidata1 = self._db.compute_covidcast_meta() + self.assertEqual(len(epidata1), 1) self.assertEqual(epidata1, [ { 'data_source': 'src', @@ -142,11 +70,11 @@ def test_caching(self): 'max_lag': 1, } ]) - epidata1={'result':1, 'message':'success', 'epidata':epidata1} + epidata1 = {'result': 1, 'message': 'success', 'epidata': epidata1} # make sure the API covidcast_meta is still blank, since it only serves # the cached version and we haven't cached anything yet - epidata2 = Epidata.covidcast_meta() + epidata2 = self.epidata_client.covidcast_meta() self.assertEqual(epidata2['result'], -2, json.dumps(epidata2)) # update the cache @@ -154,21 +82,21 @@ def test_caching(self): main(args) # fetch the cached version - epidata3 = Epidata.covidcast_meta() + epidata3 = self.epidata_client.covidcast_meta() # cached version should now equal live version self.assertEqual(epidata1, epidata3) # insert dummy data timestamped as of now - self.cur.execute(''' + self._db._cursor.execute(''' update covidcast_meta_cache set timestamp = UNIX_TIMESTAMP(NOW()), epidata = '[{"hello": "world"}]' ''') - self.cnx.commit() + self._db._connection.commit() # fetch the cached version (manually) - epidata4 = self._make_request() + epidata4 = self._make_request(endpoint="covidcast_meta", json=True, params={'cached': 'true'}, auth=self.epidata_client.auth, raise_for_status=True) # make sure the cache was actually served self.assertEqual(epidata4, { @@ -180,15 +108,15 @@ def test_caching(self): }) # insert dummy data timestamped as 2 hours old - self.cur.execute(''' + self._db._cursor.execute(''' update covidcast_meta_cache set timestamp = UNIX_TIMESTAMP(NOW()) - 3600 * 2, epidata = '[{"hello": "world"}]' ''') - self.cnx.commit() + self._db._connection.commit() # fetch the cached version (manually) - epidata5 = self._make_request() + epidata5 = self._make_request(endpoint="covidcast_meta", json=True, params={'cached': 'true'}, auth=self.epidata_client.auth, raise_for_status=True) # make sure the cache was returned anyhow self.assertEqual(epidata4, epidata5) diff --git a/integrations/acquisition/covidcast/test_csv_uploading.py b/integrations/acquisition/covidcast/test_csv_uploading.py index dab35f414..5f11dbb99 100644 --- a/integrations/acquisition/covidcast/test_csv_uploading.py +++ b/integrations/acquisition/covidcast/test_csv_uploading.py @@ -3,80 +3,23 @@ # standard library from datetime import date import os -import unittest -import argparse # third party -import mysql.connector import pandas as pd import numpy as np # first party from delphi_utils import Nans -from delphi.epidata.client.delphi_epidata import Epidata from delphi.epidata.acquisition.covidcast.csv_to_database import main, get_argument_parser -import delphi.operations.secrets as secrets +from delphi.epidata.common.covidcast_test_base import CovidcastTestBase # py3tester coverage target (equivalent to `import *`) __test_target__ = 'delphi.epidata.acquisition.covidcast.csv_to_database' -class CsvUploadingTests(unittest.TestCase): +class CsvUploadingTests(CovidcastTestBase): """Tests covidcast CSV uploading.""" - def setUp(self): - """Perform per-test setup.""" - - # connect to the `epidata` database and clear the `covidcast` table - cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='covid') - cur = cnx.cursor() - - # clear all tables - cur.execute("truncate table epimetric_load") - cur.execute("truncate table epimetric_full") - cur.execute("truncate table epimetric_latest") - cur.execute("truncate table geo_dim") - cur.execute("truncate table signal_dim") - # reset the `covidcast_meta_cache` table (it should always have one row) - cur.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') - - cnx.commit() - cur.close() - - # make connection and cursor available to test cases - self.cnx = cnx - self.cur = cnx.cursor() - - # use the local instance of the epidata database - secrets.db.host = 'delphi_database_epidata' - secrets.db.epi = ('user', 'pass') - - epidata_cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='epidata') - epidata_cur = epidata_cnx.cursor() - - epidata_cur.execute("DELETE FROM `api_user`") - epidata_cur.execute('INSERT INTO `api_user`(`api_key`, `email`) VALUES("key", "email")') - epidata_cnx.commit() - epidata_cur.close() - epidata_cnx.close() - - # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' - Epidata.auth = ('epidata', 'key') - - def tearDown(self): - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - @staticmethod def apply_lag(expected_epidata): expected_issue_day=date.today() @@ -91,11 +34,11 @@ def apply_lag(expected_epidata): return expected_epidata def verify_timestamps_and_defaults(self): - self.cur.execute(''' + self._db._cursor.execute(''' select value_updated_timestamp from epimetric_full UNION ALL select value_updated_timestamp from epimetric_latest''') - for (value_updated_timestamp,) in self.cur: + for (value_updated_timestamp,) in self._db._cursor: self.assertGreater(value_updated_timestamp, 0) def test_uploading(self): @@ -130,16 +73,16 @@ def test_uploading(self): # upload CSVs main(args) - response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') + response = self.epidata_client.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') - expected_values = pd.concat([values, pd.DataFrame({ "geo_type": "state", "source": "src-name", "time_type": "day", "time_value": [20200419] * 3, "signal": [signal_name] * 3, "direction": [None] * 3})], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records") + expected_values = pd.concat([values, pd.DataFrame({"geo_type": "state", "source": "src-name", "time_type": "day", "time_value": [20200419] * 3, "signal": [signal_name] * 3, "direction": [None] * 3})], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records") expected_response = {'result': 1, 'epidata': self.apply_lag(expected_values), 'message': 'success'} self.assertEqual(response, expected_response) self.verify_timestamps_and_defaults() # Verify that files were archived - path = data_dir + f'/archive/successful/src-name/20200419_state_test.csv.gz' + path = data_dir + '/archive/successful/src-name/20200419_state_test.csv.gz' self.assertIsNotNone(os.stat(path)) self.tearDown() @@ -158,7 +101,7 @@ def test_uploading(self): # upload CSVs main(args) - response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') + response = self.epidata_client.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_values = pd.concat([values, pd.DataFrame({ "geo_type": "state", @@ -195,7 +138,7 @@ def test_uploading(self): # upload CSVs main(args) - response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') + response = self.epidata_client.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_response = {'epidata': [], 'result': -2, 'message': 'no results'} @@ -220,7 +163,7 @@ def test_uploading(self): # upload CSVs main(args) - response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') + response = self.epidata_client.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_values_df = pd.concat([values, pd.DataFrame({ "geo_type": "state", @@ -256,7 +199,7 @@ def test_uploading(self): # upload CSVs main(args) - response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') + response = self.epidata_client.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_values = pd.concat([values, pd.DataFrame({ "geo_type": "state", @@ -290,7 +233,7 @@ def test_uploading(self): # upload CSVs main(args) - response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') + response = self.epidata_client.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_response = {'epidata': [], 'result': -2, 'message': 'no results'} diff --git a/integrations/acquisition/covidcast/test_db.py b/integrations/acquisition/covidcast/test_db.py index 7b9d80770..780b318ee 100644 --- a/integrations/acquisition/covidcast/test_db.py +++ b/integrations/acquisition/covidcast/test_db.py @@ -1,13 +1,13 @@ from delphi_utils import Nans from delphi.epidata.acquisition.covidcast.database import DBLoadStateException -from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow +from delphi.epidata.common.covidcast_test_base import CovidcastTestBase, CovidcastTestRow # all the Nans we use here are just one value, so this is a shortcut to it: nmv = Nans.NOT_MISSING.value -class TestTest(CovidcastBase): +class TestDatabase(CovidcastTestBase): def _find_matches_for_row(self, row): # finds (if existing) row from both history and latest views that matches long-key of provided CovidcastTestRow diff --git a/integrations/acquisition/covidcast/test_delete_batch.py b/integrations/acquisition/covidcast/test_delete_batch.py index 4624df27c..c68b8a2b6 100644 --- a/integrations/acquisition/covidcast/test_delete_batch.py +++ b/integrations/acquisition/covidcast/test_delete_batch.py @@ -6,39 +6,16 @@ from os import path # first party -import delphi.operations.secrets as secrets -from delphi.epidata.acquisition.covidcast.database import Database -from delphi.epidata.acquisition.covidcast.test_utils import covidcast_rows_from_args +from delphi.epidata.common.covidcast_test_base import covidcast_rows_from_args, CovidcastTestBase # py3tester coverage target (equivalent to `import *`) __test_target__ = 'delphi.epidata.acquisition.covidcast.database' Example = namedtuple("example", "given expected") -class DeleteBatch(unittest.TestCase): +class DeleteBatch(CovidcastTestBase): """Tests batch deletions""" - - def setUp(self): - """Perform per-test setup.""" - - # use the local instance of the epidata database - secrets.db.host = 'delphi_database_epidata' - secrets.db.epi = ('user', 'pass') - - # will use secrets as set above - self._db = Database() - self._db.connect() - - for table in "epimetric_load epimetric_latest epimetric_full geo_dim signal_dim".split(): - self._db._cursor.execute(f"TRUNCATE TABLE {table}") - - - def tearDown(self): - """Perform per-test teardown.""" - self._db.disconnect(False) - del self._db - @unittest.skip("Database user would require FILE privileges") def test_delete_from_file(self): self._test_delete_batch(path.join(path.dirname(__file__), "delete_batch.csv")) diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 2af923b2b..8b57accaf 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -10,10 +10,8 @@ from aiohttp.client_exceptions import ClientResponseError # third party -import delphi.operations.secrets as secrets from delphi.epidata.maintenance.covidcast_meta_cache_updater import main as update_covidcast_meta_cache -from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow, FIPS, MSA -from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.common.covidcast_test_base import CovidcastTestBase, CovidcastTestRow, FIPS, MSA from delphi_utils import Nans # py3tester coverage target @@ -21,31 +19,10 @@ # all the Nans we use here are just one value, so this is a shortcut to it: nmv = Nans.NOT_MISSING.value -def fake_epidata_endpoint(func): - """This can be used as a decorator to enable a bogus Epidata endpoint to return 404 responses.""" - def wrapper(*args): - Epidata.BASE_URL = 'http://delphi_web_epidata/fake_epidata' - func(*args) - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' - return wrapper -class DelphiEpidataPythonClientTests(CovidcastBase): +class DelphiEpidataPythonClientTests(CovidcastTestBase): """Tests the Python client.""" - def localSetUp(self): - """Perform per-test setup.""" - - # reset the `covidcast_meta_cache` table (it should always have one row) - self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') - - # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' - Epidata.auth = ('epidata', 'key') - - # use the local instance of the epidata database - secrets.db.host = 'delphi_database_epidata' - secrets.db.epi = ('user', 'pass') - def test_covidcast(self): """Test that the covidcast endpoint returns expected data.""" @@ -60,7 +37,7 @@ def test_covidcast(self): with self.subTest(name='request two signals'): # fetch data - response = Epidata.covidcast( + response = self.epidata_client.covidcast( **self.params_from_row(rows[0], signals=[rows[0].signal, rows[-1].signal]) ) @@ -79,7 +56,7 @@ def test_covidcast(self): with self.subTest(name='request two signals with tree format'): # fetch data - response = Epidata.covidcast( + response = self.epidata_client.covidcast( **self.params_from_row(rows[0], signals=[rows[0].signal, rows[-1].signal], format='tree') ) @@ -101,7 +78,7 @@ def test_covidcast(self): with self.subTest(name='request most recent'): # fetch data, without specifying issue or lag - response_1 = Epidata.covidcast( + response_1 = self.epidata_client.covidcast( **self.params_from_row(rows[0]) ) @@ -116,7 +93,7 @@ def test_covidcast(self): with self.subTest(name='request as-of a date'): # fetch data, specifying as_of - response_1a = Epidata.covidcast( + response_1a = self.epidata_client.covidcast( **self.params_from_row(rows[0], as_of=rows[1].issue) ) @@ -140,8 +117,8 @@ def test_covidcast(self): with self.subTest(name='request a range of issues'): # fetch data, specifying issue range, not lag - response_2 = Epidata.covidcast( - **self.params_from_row(rows[0], issues=Epidata.range(rows[0].issue, rows[1].issue)) + response_2 = self.epidata_client.covidcast( + **self.params_from_row(rows[0], issues=self.epidata_client.range(rows[0].issue, rows[1].issue)) ) expected = [ @@ -158,7 +135,7 @@ def test_covidcast(self): with self.subTest(name='request at a given lag'): # fetch data, specifying lag, not issue range - response_3 = Epidata.covidcast( + response_3 = self.epidata_client.covidcast( **self.params_from_row(rows[0], lag=2) ) @@ -173,7 +150,7 @@ def test_covidcast(self): with self.subTest(name='long request'): # fetch data, without specifying issue or lag # TODO should also trigger a post but doesn't due to the 414 issue - response_1 = Epidata.covidcast( + response_1 = self.epidata_client.covidcast( **self.params_from_row(rows[0], signals='sig'*1000) ) @@ -185,7 +162,7 @@ def test_covidcast(self): def test_request_method(self, get, post): """Test that a GET request is default and POST is used if a 414 is returned.""" with self.subTest(name='get request'): - Epidata.covidcast('src', 'sig', 'day', 'county', 20200414, '01234') + self.epidata_client.covidcast('src', 'sig', 'day', 'county', 20200414, '01234') get.assert_called_once() post.assert_not_called() with self.subTest(name='post request'): @@ -193,7 +170,7 @@ def test_request_method(self, get, post): mock_response = MagicMock() mock_response.status_code = 414 get.return_value = mock_response - Epidata.covidcast('src', 'sig', 'day', 'county', 20200414, '01234') + self.epidata_client.covidcast('src', 'sig', 'day', 'county', 20200414, '01234') get.assert_called_once() post.assert_called_once() @@ -204,7 +181,7 @@ def test_retry_request(self, get): mock_response = MagicMock() mock_response.status_code = 200 get.side_effect = [JSONDecodeError('Expecting value', "", 0), mock_response] - response = Epidata._request("") + response = self.epidata_client._request("") self.assertEqual(get.call_count, 2) self.assertEqual(response, mock_response.json()) @@ -215,7 +192,7 @@ def test_retry_request(self, get): get.side_effect = [JSONDecodeError('Expecting value', "", 0), JSONDecodeError('Expecting value', "", 0), mock_response] - response = Epidata._request("") + response = self.epidata_client._request("") self.assertEqual(get.call_count, 2) # 2 from previous test + 2 from this one self.assertEqual(response, {'result': 0, 'message': 'error: Expecting value: line 1 column 1 (char 0)'} @@ -240,7 +217,7 @@ def test_geo_value(self): ] def fetch(geo): - return Epidata.covidcast( + return self.epidata_client.covidcast( **self.params_from_row(rows[0], geo_value=geo) ) @@ -297,7 +274,7 @@ def test_covidcast_meta(self): update_covidcast_meta_cache(args=None) # fetch data - response = Epidata.covidcast_meta() + response = self.epidata_client.covidcast_meta() # make sure "last updated" time is recent: updated_time = response['epidata'][0]['last_update'] @@ -343,9 +320,9 @@ def test_async_epidata(self): ] self._insert_rows(rows) - test_output = Epidata.async_epidata([ - self.params_from_row(rows[0], source='covidcast'), - self.params_from_row(rows[1], source='covidcast') + test_output = self.epidata_client.async_epidata([ + self.params_from_row(rows[0], source="covidcast"), + self.params_from_row(rows[1], source="covidcast") ]*12, batch_size=10) responses = [i[0]["epidata"] for i in test_output] # check response is same as standard covidcast call (minus fields omitted by the api.php endpoint), @@ -354,17 +331,16 @@ def test_async_epidata(self): self.assertEqual( responses, [ - [{k: row[k] for k in row.keys() - ignore_fields} for row in Epidata.covidcast(**self.params_from_row(rows[0]))["epidata"]], - [{k: row[k] for k in row.keys() - ignore_fields} for row in Epidata.covidcast(**self.params_from_row(rows[1]))["epidata"]], + [{k: row[k] for k in row.keys() - ignore_fields} for row in self.epidata_client.covidcast(**self.params_from_row(rows[0]))["epidata"]], + [{k: row[k] for k in row.keys() - ignore_fields} for row in self.epidata_client.covidcast(**self.params_from_row(rows[1]))["epidata"]], ]*12 ) - @fake_epidata_endpoint def test_async_epidata_fail(self): + self.epidata_client.BASE_URL = "http://delphi_web_epidata/fake_epidata" with pytest.raises(ClientResponseError, match="404, message='NOT FOUND'"): - Epidata.async_epidata([ + self.epidata_client.async_epidata([ { - 'source': 'covidcast', 'data_source': 'src', 'signals': 'sig', 'time_type': 'day', @@ -373,3 +349,4 @@ def test_async_epidata_fail(self): 'time_values': '20200414' } ]) + self.epidata_client.BASE_URL = "http://delphi_web_epidata/epidata" diff --git a/integrations/server/test_api_keys.py b/integrations/server/test_api_keys.py index 27b992f17..91fb11a57 100644 --- a/integrations/server/test_api_keys.py +++ b/integrations/server/test_api_keys.py @@ -1,8 +1,6 @@ """Integration tests for the API Keys""" -import requests - # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class APIKeysTets(DelphiTestBase): @@ -11,16 +9,11 @@ class APIKeysTets(DelphiTestBase): def localSetUp(self): self.role_name = "cdc" - def _make_request(self, endpoint: str, params: dict = {}, auth: tuple = None): - url = f"{self.epidata_client.BASE_URL}/{endpoint}" - response = requests.get(url, params=params, auth=auth) - return response - def test_public_route(self): """Test public route""" status_codes = set() for _ in range(10): - status_codes.add(self._make_request("version").status_code) + status_codes.add(self._make_request(endpoint="version").status_code) self.assertEqual(status_codes, {200}) def test_no_multiples_data_source(self): @@ -105,7 +98,7 @@ def test_multiples_allowed_signal_three_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request("covidcast", params=params).status_code) + status_codes.add(self._make_request(params=params).status_code) self.assertEqual(status_codes, {401}) def test_multiples_mixed_allowed_signal_three_multiples(self): @@ -119,7 +112,7 @@ def test_multiples_mixed_allowed_signal_three_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request("covidcast", params=params).status_code) + status_codes.add(self._make_request(params=params).status_code) self.assertEqual(status_codes, {401}) def test_multiples_mixed_allowed_signal_api_key(self): @@ -134,7 +127,7 @@ def test_multiples_mixed_allowed_signal_api_key(self): status_codes = set() for _ in range(10): status_codes.add( - self._make_request("covidcast", params=params, auth=self.epidata_client.auth).status_code + self._make_request(params=params, auth=self.epidata_client.auth).status_code ) self.assertEqual(status_codes, {200}) @@ -150,7 +143,7 @@ def test_multiples_allowed_signal_api_key(self): status_codes = set() for _ in range(10): status_codes.add( - self._make_request("covidcast", params=params, auth=self.epidata_client.auth).status_code + self._make_request(params=params, auth=self.epidata_client.auth).status_code ) self.assertEqual(status_codes, {200}) @@ -166,7 +159,7 @@ def test_no_multiples_allowed_signal_api_key(self): status_codes = set() for _ in range(10): status_codes.add( - self._make_request("covidcast", params=params, auth=self.epidata_client.auth).status_code + self._make_request(params=params, auth=self.epidata_client.auth).status_code ) self.assertEqual(status_codes, {200}) @@ -183,7 +176,7 @@ def test_no_multiples_allowed_signal_bad_api_key(self): for _ in range(10): status_codes.add( self._make_request( - "covidcast", params=params, auth=("bad_key", "bad_email") + params=params, auth=("bad_key", "bad_email") ).status_code ) self.assertEqual(status_codes, {200}) @@ -193,7 +186,7 @@ def test_restricted_endpoint_no_key(self): params = {"regions": "1as", "epiweeks": "202020"} status_codes = set() for _ in range(10): - status_codes.add(self._make_request("cdc", params=params).status_code) + status_codes.add(self._make_request(params=params, endpoint="cdc").status_code) self.assertEqual(status_codes, {401}) def test_restricted_endpoint_invalid_key(self): @@ -205,7 +198,7 @@ def test_restricted_endpoint_invalid_key(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request("cdc", params=params).status_code) + status_codes.add(self._make_request(params=params, endpoint="cdc").status_code) self.assertEqual(status_codes, {401}) def test_restricted_endpoint_no_roles_key(self): @@ -217,7 +210,7 @@ def test_restricted_endpoint_no_roles_key(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request("cdc", params=params).status_code) + status_codes.add(self._make_request(params=params, endpoint="cdc").status_code) self.assertEqual(status_codes, {401}) def test_restricted_endpoint_valid_roles_key(self): @@ -229,5 +222,5 @@ def test_restricted_endpoint_valid_roles_key(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request("cdc", params=params).status_code) + status_codes.add(self._make_request(params=params, endpoint="cdc").status_code) self.assertEqual(status_codes, {200}) diff --git a/integrations/server/test_cdc.py b/integrations/server/test_cdc.py index d468bd162..7f60ddca1 100644 --- a/integrations/server/test_cdc.py +++ b/integrations/server/test_cdc.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class CdcTest(DelphiTestBase): diff --git a/integrations/server/test_covidcast.py b/integrations/server/test_covidcast.py index 8b7fc6f52..fd824ad74 100644 --- a/integrations/server/test_covidcast.py +++ b/integrations/server/test_covidcast.py @@ -1,31 +1,19 @@ """Integration tests for the `covidcast` endpoint.""" -# standard library -from typing import Callable import unittest - -# third party import mysql.connector # first party from delphi_utils import Nans -from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow, FIPS, MSA -from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.common.covidcast_test_base import CovidcastTestBase, CovidcastTestRow, FIPS, MSA -class CovidcastTests(CovidcastBase): +class CovidcastTests(CovidcastTestBase): """Tests the `covidcast` endpoint.""" - def localSetUp(self): - """Perform per-test setup.""" - self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') - def request_based_on_row(self, row: CovidcastTestRow, **kwargs): params = self.params_from_row(row, endpoint='covidcast', **kwargs) - # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' - Epidata.auth = ('epidata', 'key') - response = Epidata.covidcast(**params) + response = self.epidata_client.covidcast(**params) return response @@ -79,7 +67,7 @@ def _insert_placeholder_set_five(self): for i in [4, 5, 6] ] self._insert_rows(rows) - return rows + return rows def _insert_placeholder_set_with_weeks(self): rows = [ @@ -118,11 +106,11 @@ def test_round_trip(self): # # insert placeholder data # self.cur.execute(f''' # INSERT INTO - # `covidcast` (`id`, `source`, `signal`, `time_type`, `geo_type`, - # `time_value`, `geo_value`, `value_updated_timestamp`, - # `value`, `stderr`, `sample_size`, `direction_updated_timestamp`, + # `covidcast` (`id`, `source`, `signal`, `time_type`, `geo_type`, + # `time_value`, `geo_value`, `value_updated_timestamp`, + # `value`, `stderr`, `sample_size`, `direction_updated_timestamp`, # `direction`, `issue`, `lag`, `is_latest_issue`, `missing_value`, - # `missing_stderr`,`missing_sample_size`) + # `missing_stderr`,`missing_sample_size`) # VALUES # (0, 'src', 'sig', 'day', 'county', 20200414, '01234', # 123, 1.5, 2.5, 3.5, 456, 4, 20200414, 0, 1, @@ -373,7 +361,7 @@ def test_unique_key_constraint(self): self._insert_placeholder_set_one() # succeed to insert different placeholder data under a different time_type - self._insert_placeholder_set_one(time_type='week') + self._insert_placeholder_set_one() def test_nullable_columns(self): """Missing values should be surfaced as null.""" diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index 206ed3c54..497a65471 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -6,52 +6,18 @@ # third party from more_itertools import windowed -import requests import pandas as pd from delphi.epidata.maintenance.covidcast_meta_cache_updater import main as update_cache -from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow +from delphi.epidata.common.covidcast_test_base import CovidcastTestBase, CovidcastTestRow -# use the local instance of the Epidata API -BASE_URL = "http://delphi_web_epidata/epidata/covidcast" -BASE_URL_OLD = "http://delphi_web_epidata/epidata/api.php" -AUTH = ('epidata', 'key') - -class CovidcastEndpointTests(CovidcastBase): +class CovidcastEndpointTests(CovidcastTestBase): """Tests the `covidcast/*` endpoint.""" def localSetUp(self): """Perform per-test setup.""" - # reset the `covidcast_meta_cache` table (it should always have one row) - self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') - - cur = self._db._cursor - # NOTE: we must specify the db schema "epidata" here because the cursor/connection are bound to schema "covid" - cur.execute("TRUNCATE TABLE epidata.api_user") - cur.execute("TRUNCATE TABLE epidata.user_role") - cur.execute("TRUNCATE TABLE epidata.user_role_link") - cur.execute("INSERT INTO epidata.api_user (api_key, email) VALUES ('quidel_key', 'quidel_email')") - cur.execute("INSERT INTO epidata.user_role (name) VALUES ('quidel')") - cur.execute( - "INSERT INTO epidata.user_role_link (user_id, role_id) SELECT api_user.id, user_role.id FROM epidata.api_user JOIN epidata.user_role WHERE api_key='quidel_key' and user_role.name='quidel'" - ) - cur.execute("INSERT INTO epidata.api_user (api_key, email) VALUES ('key', 'email')") - - def _fetch(self, endpoint="/", is_compatibility=False, auth=AUTH, **params): - # make the request - if is_compatibility: - url = BASE_URL_OLD - # only set endpoint if it's not already set - # only set endpoint if it's not already set - params.setdefault("endpoint", "covidcast") - if params.get("source"): - params.setdefault("data_source", params.get("source")) - else: - url = f"{BASE_URL}{endpoint}" - response = requests.get(url, params=params, auth=auth) - response.raise_for_status() - return response.json() + self.role_name = "quidel" def _diff_rows(self, rows: Sequence[float]): return [ @@ -61,7 +27,7 @@ def _diff_rows(self, rows: Sequence[float]): def _smooth_rows(self, rows: Sequence[float]): return [ - sum(e)/len(e) if None not in e else None + sum(e)/len(e) if None not in e else None for e in windowed(rows, 7) ] @@ -72,11 +38,16 @@ def test_basic(self): self._insert_rows(rows) with self.subTest("validation"): - out = self._fetch("/") + out = self._make_request(auth=self.epidata_client.auth, json=True, raise_for_status=True) self.assertEqual(out["result"], -1) with self.subTest("simple"): - out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*") + params = { + "signal": first.signal_pair(), + "geo": first.geo_pair(), + "time": "day:*" + } + out = self._make_request(auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(len(out["epidata"]), len(rows)) def test_basic_restricted_source(self): @@ -84,21 +55,26 @@ def test_basic_restricted_source(self): rows = [CovidcastTestRow.make_default_row(time_value=2020_04_01 + i, value=i, source="quidel") for i in range(10)] first = rows[0] self._insert_rows(rows) + params = { + "signal": first.signal_pair(), + "geo": first.geo_pair(), + "time": "day:*" + } with self.subTest("validation"): - out = self._fetch("/") + out = self._make_request(auth=self.epidata_client.auth, json=True, raise_for_status=True) self.assertEqual(out["result"], -1) with self.subTest("no_roles"): - out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*") + out = self._make_request(auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(len(out["epidata"]), 0) with self.subTest("no_api_key"): - out = self._fetch("/", auth=None, signal=first.signal_pair(), geo=first.geo_pair(), time="day:*") + out = self._make_request(json=True, raise_for_status=True, params=params) self.assertEqual(len(out["epidata"]), 0) with self.subTest("quidel_role"): - out = self._fetch("/", auth=("epidata", "quidel_key"), signal=first.signal_pair(), geo=first.geo_pair(), time="day:*") + out = self._make_request(json=True, raise_for_status=True, params=params, auth=("epidata", "quidel_key")) self.assertEqual(len(out["epidata"]), len(rows)) def test_compatibility(self): @@ -106,18 +82,23 @@ def test_compatibility(self): rows = [CovidcastTestRow.make_default_row(source="src", signal="sig", time_value=2020_04_01 + i, value=i) for i in range(10)] first = rows[0] self._insert_rows(rows) + params = { + "signal": first.signal_pair(), + "geo": first.geo_pair(), + "time": "day:*", + } with self.subTest("validation"): - out = self._fetch("/", is_compatibility=True) + out = self._make_request(json=True, raise_for_status=True, is_compatibility=True) self.assertEqual(out["result"], -1) with self.subTest("simple"): - out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + out = self._make_request(params=params, json=True, raise_for_status=True, is_compatibility=True) self.assertEqual(out["epidata"], [row.as_api_compatibility_row_dict() for row in rows]) with self.subTest("same contents sans excluded columns"): - compat = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) - regular = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*") + compat = self._make_request(params=params, json=True, raise_for_status=True, is_compatibility=True) + regular = self._make_request(params=params, json=True, raise_for_status=True) # Remove keys from the regular row which are excluded in the compat rows for row in regular['epidata']: for key in ['source', 'geo_type', 'time_type']: @@ -129,17 +110,22 @@ def test_compatibility_restricted_source(self): rows = [CovidcastTestRow.make_default_row(time_value=2020_04_01 + i, value=i, source="quidel") for i in range(10)] first = rows[0] self._insert_rows(rows) + params = { + "signal": first.signal_pair(), + "geo": first.geo_pair(), + "time": "day:*", + } with self.subTest("no_roles"): - out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + out = self._make_request(params=params, json=True, raise_for_status=True, is_compatibility=True) self.assertTrue("epidata" not in out) with self.subTest("no_api_key"): - out = self._fetch("/", auth=None, signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + out = self._make_request(params=params, json=True, raise_for_status=True, is_compatibility=True) self.assertTrue("epidata" not in out) with self.subTest("quidel_role"): - out = self._fetch("/", auth=("epidata", "quidel_key"), signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + out = self._make_request(params=params, auth=("epidata", "quidel_key"), json=True, raise_for_status=True, is_compatibility=True) self.assertEqual(out["epidata"], [row.as_api_compatibility_row_dict() for row in rows]) def test_trend(self): @@ -152,8 +138,14 @@ def test_trend(self): ref = rows[num_rows // 2] self._insert_rows(rows) - out = self._fetch("/trend", signal=first.signal_pair(), geo=first.geo_pair(), date=last.time_value, window="20200401-20201212", basis=ref.time_value) - + params = { + "signal": first.signal_pair(), + "geo": first.geo_pair(), + "date": last.time_value, + "window": "20200401-20201212", + "basis": ref.time_value + } + out = self._make_request(endpoint="covidcast/trend", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(out["result"], 1) self.assertEqual(len(out["epidata"]), 1) @@ -177,7 +169,6 @@ def test_trend(self): self.assertEqual(trend["max_value"], last.value) self.assertEqual(trend["max_trend"], "steady") - def test_trendseries(self): """Request a signal from the /trendseries endpoint.""" @@ -187,7 +178,14 @@ def test_trendseries(self): last = rows[-1] self._insert_rows(rows) - out = self._fetch("/trendseries", signal=first.signal_pair(), geo=first.geo_pair(), date=last.time_value, window="20200401-20200410", basis=1) + params = { + "signal": first.signal_pair(), + "geo": first.geo_pair(), + "date": last.time_value, + "window": "20200401-20200410", + "basis": 1 + } + out = self._make_request(endpoint="covidcast/trendseries", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(out["result"], 1) self.assertEqual(len(out["epidata"]), 3) @@ -244,7 +242,6 @@ def match_row(trend, row): self.assertEqual(trend["max_value"], first.value) self.assertEqual(trend["max_trend"], "decreasing") - def test_csv(self): """Request a signal from the /csv endpoint.""" @@ -252,17 +249,18 @@ def test_csv(self): first = rows[0] self._insert_rows(rows) - response = requests.get( - f"{BASE_URL}/csv", - params=dict(signal=first.signal_pair(), start_day="2020-04-01", end_day="2020-12-12", geo_type=first.geo_type), - ) - response.raise_for_status() + params = { + "signal": first.signal_pair(), + "start_day": "2020-04-01", + "end_day": "2020-12-12", + "geo_type": first.geo_type + } + response = self._make_request(endpoint="covidcast/csv", raise_for_status=True, params=params) out = response.text df = pd.read_csv(StringIO(out), index_col=0) self.assertEqual(df.shape, (len(rows), 10)) self.assertEqual(list(df.columns), ["geo_value", "signal", "time_value", "issue", "lag", "value", "stderr", "sample_size", "geo_type", "data_source"]) - def test_backfill(self): """Request a signal from the /backfill endpoint.""" @@ -274,7 +272,13 @@ def test_backfill(self): self._insert_rows([*issue_0, *issue_1, *last_issue]) first = issue_0[0] - out = self._fetch("/backfill", signal=first.signal_pair(), geo=first.geo_pair(), time="day:20200401-20201212", anchor_lag=3) + params = { + "signal": first.signal_pair(), + "geo": first.geo_pair(), + "time": "day:20200401-20201212", + "anchor_lag": 3 + } + out = self._make_request(endpoint="covidcast/backfill", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(out["result"], 1) df = pd.DataFrame(out["epidata"]) self.assertEqual(len(df), 3 * num_rows) # num issues @@ -304,7 +308,7 @@ def test_meta(self): update_cache(args=None) with self.subTest("plain"): - out = self._fetch("/meta") + out = self._make_request(endpoint="covidcast/meta", auth=self.epidata_client.auth, json=True, raise_for_status=True) self.assertEqual(len(out), 1) data_source = out[0] self.assertEqual(data_source["source"], first.source) @@ -322,20 +326,26 @@ def test_meta(self): self.assertEqual(stats_g["mean"], sum(r.value for r in rows) / len(rows)) with self.subTest("filtered"): - out = self._fetch("/meta", signal=f"{first.source}:*") + params = { + "signal": f"{first.source}:*" + } + out = self._make_request(endpoint="covidcast/meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(len(out), 1) data_source = out[0] self.assertEqual(data_source["source"], first.source) self.assertEqual(len(data_source["signals"]), 1) stats = data_source["signals"][0] self.assertEqual(stats["source"], first.source) - out = self._fetch("/meta", signal=f"{first.source}:X") + params = { + "signal": f"{first.source}:X" + } + out = self._make_request(endpoint="covidcast/meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(len(out), 0) def test_meta_restricted(self): """Request 'restricted' signals from the /meta endpoint.""" # NOTE: this method is nearly identical to ./test_covidcast_meta.py:test_restricted_sources() - # ...except the self._fetch() methods are different, as is the format of those methods' outputs + # ...except the self._make_request() methods are different, as is the format of those methods' outputs # (the other covidcast_meta endpoint uses APrinter, this one returns its own unadulterated json). # additionally, the sample data used here must match entries (that is, named sources and signals) # from covidcast_utils.model.data_sources (the `data_sources` variable from file @@ -354,12 +364,15 @@ def test_meta_restricted(self): update_cache(args=None) # verify unauthenticated (no api key) or unauthorized (user w/o privilege) only see metadata for one source - self.assertEqual(len(self._fetch("/meta", auth=None)), 1) - self.assertEqual(len(self._fetch("/meta", auth=AUTH)), 1) + unauthenticated_request = self._make_request(endpoint="covidcast/meta", json=True, raise_for_status=True) + unauthorized_request = self._make_request(endpoint="covidcast/meta", auth=self.epidata_client.auth, json=True, raise_for_status=True) + self.assertEqual(len(unauthenticated_request), 1) + self.assertEqual(len(unauthorized_request), 1) # verify authorized user sees metadata for both sources qauth = ('epidata', 'quidel_key') - self.assertEqual(len(self._fetch("/meta", auth=qauth)), 2) + authorized_request = self._make_request(endpoint="covidcast/meta", auth=qauth, json=True, raise_for_status=True) + self.assertEqual(len(authorized_request), 2) def test_coverage(self): """Request a signal from the /coverage endpoint.""" @@ -371,17 +384,34 @@ def test_coverage(self): first = rows[0] with self.subTest("default"): - out = self._fetch("/coverage", signal=first.signal_pair(), geo_type=first.geo_type, latest=dates[-1], format="json") + params = { + "signal": first.signal_pair(), + "geo_type": first.geo_type, + "latest": dates[-1], + "format": "json" + } + out = self._make_request(endpoint="covidcast/coverage", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(len(out), len(num_geos_per_date)) self.assertEqual([o["time_value"] for o in out], dates) self.assertEqual([o["count"] for o in out], num_geos_per_date) with self.subTest("specify window"): - out = self._fetch("/coverage", signal=first.signal_pair(), geo_type=first.geo_type, window=f"{dates[0]}-{dates[1]}", format="json") + params = { + "signal": first.signal_pair(), + "geo_type": first.geo_type, + "window": f"{dates[0]}-{dates[1]}", + "format": "json" + } + out = self._make_request("covidcast/coverage", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(len(out), 2) self.assertEqual([o["time_value"] for o in out], dates[:2]) self.assertEqual([o["count"] for o in out], num_geos_per_date[:2]) with self.subTest("invalid geo_type"): - out = self._fetch("/coverage", signal=first.signal_pair(), geo_type="doesnt_exist", format="json") + params = { + "signal": first.signal_pair(), + "geo_type": "doesnt_exist", + "format": "json" + } + out = self._make_request(endpoint="covidcast/coverage", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(len(out), 0) diff --git a/integrations/server/test_covidcast_meta.py b/integrations/server/test_covidcast_meta.py index 857422a41..e79ec523c 100644 --- a/integrations/server/test_covidcast_meta.py +++ b/integrations/server/test_covidcast_meta.py @@ -1,24 +1,12 @@ """Integration tests for the `covidcast_meta` endpoint.""" -# standard library -import unittest - -# third party -import mysql.connector -import requests - #first party from delphi_utils import Nans -from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow +from delphi.epidata.common.covidcast_test_base import CovidcastTestBase, CovidcastTestRow from delphi.epidata.maintenance.covidcast_meta_cache_updater import main as update_cache -import delphi.operations.secrets as secrets -# use the local instance of the Epidata API -BASE_URL = 'http://delphi_web_epidata/epidata' -AUTH = ('epidata', 'key') - -class CovidcastMetaTests(CovidcastBase): +class CovidcastMetaTests(CovidcastTestBase): """Tests the `covidcast_meta` endpoint.""" src_sig_lookups = { @@ -52,63 +40,23 @@ class CovidcastMetaTests(CovidcastBase): def localSetUp(self): """Perform per-test setup.""" - # connect to the `epidata` database and clear the `covidcast` table - cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='covid') - cur = cnx.cursor() - - # clear all tables - cur.execute("truncate table epimetric_load") - cur.execute("truncate table epimetric_full") - cur.execute("truncate table epimetric_latest") - cur.execute("truncate table geo_dim") - cur.execute("truncate table signal_dim") - # reset the `covidcast_meta_cache` table (it should always have one row) - cur.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') - - # NOTE: we must specify the db schema "epidata" here because the cursor/connection are bound to schema "covid" - cur.execute("TRUNCATE TABLE epidata.api_user") - cur.execute("TRUNCATE TABLE epidata.user_role") - cur.execute("TRUNCATE TABLE epidata.user_role_link") - cur.execute("INSERT INTO epidata.api_user (api_key, email) VALUES ('quidel_key', 'quidel_email')") - cur.execute("INSERT INTO epidata.user_role (name) VALUES ('quidel')") - cur.execute( - "INSERT INTO epidata.user_role_link (user_id, role_id) SELECT api_user.id, user_role.id FROM epidata.api_user JOIN epidata.user_role WHERE api_key='quidel_key' and user_role.name='quidel'" - ) - cur.execute("INSERT INTO epidata.api_user (api_key, email) VALUES ('key', 'email')") + self.role_name = "quidel" # populate dimension tables for (src,sig) in self.src_sig_lookups: - cur.execute(''' + self._db._cursor.execute(''' INSERT INTO `signal_dim` (`signal_key_id`, `source`, `signal`) VALUES (%d, '%s', '%s'); ''' % ( self.src_sig_lookups[(src,sig)], src, sig )) for (gt,gv) in self.geo_lookups: - cur.execute(''' + self._db._cursor.execute(''' INSERT INTO `geo_dim` (`geo_key_id`, `geo_type`, `geo_value`) VALUES (%d, '%s', '%s'); ''' % ( self.geo_lookups[(gt,gv)], gt, gv )) - cnx.commit() - cur.close() + self._db._connection.commit() # initialize counter for tables without non-autoincrement id self.id_counter = 666 - # make connection and cursor available to test cases - self.cnx = cnx - self.cur = cnx.cursor() - - # use the local instance of the epidata database - secrets.db.host = 'delphi_database_epidata' - secrets.db.epi = ('user', 'pass') - - - def localTearDown(self): - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() def insert_placeholder_data(self): expected = [] @@ -135,26 +83,19 @@ def insert_placeholder_data(self): }) for tv in (1, 2): for gv, v in zip(('geo1', 'geo2'), (10, 20)): - self.cur.execute(self.template % ( + self._db._cursor.execute(self.template % ( self._get_id(), self.src_sig_lookups[(src,sig)], self.geo_lookups[(gt,gv)], tt, tv, v, tv, # re-use time value for issue Nans.NOT_MISSING, Nans.NOT_MISSING, Nans.NOT_MISSING )) - self.cnx.commit() + self._db._connection.commit() update_cache(args=None) return expected def _get_id(self): self.id_counter += 1 return self.id_counter - - @staticmethod - def _fetch(auth=AUTH, **kwargs): - params = kwargs.copy() - response = requests.get(f"{BASE_URL}/covidcast_meta", params=params, auth=auth) - response.raise_for_status() - return response.json() def test_round_trip(self): """Make a simple round-trip with some sample data.""" @@ -163,7 +104,7 @@ def test_round_trip(self): expected = self.insert_placeholder_data() # make the request - response = self._fetch() + response = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True) # assert that the right data came back self.assertEqual(response, { @@ -185,12 +126,15 @@ def test_restricted_sources(self): update_cache(args=None) # verify unauthenticated (no api key) or unauthorized (user w/o privilege) only see metadata for one source - self.assertEqual(len(self._fetch(auth=None)['epidata']), 1) - self.assertEqual(len(self._fetch(auth=AUTH)['epidata']), 1) + unauthenticated_request = self._make_request(endpoint="covidcast_meta", json=True, raise_for_status=True) + self.assertEqual(len(unauthenticated_request['epidata']), 1) + unauthorized_request = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True) + self.assertEqual(len(unauthorized_request['epidata']), 1) # verify authorized user sees metadata for both sources qauth = ('epidata', 'quidel_key') - self.assertEqual(len(self._fetch(auth=qauth)['epidata']), 2) + authorized_request = self._make_request(endpoint="covidcast_meta", auth=qauth, json=True, raise_for_status=True) + self.assertEqual(len(authorized_request['epidata']), 2) def test_filter(self): """Test filtering options some sample data.""" @@ -198,63 +142,99 @@ def test_filter(self): # insert placeholder data and accumulate expected results (in sort order) expected = self.insert_placeholder_data() - res = self._fetch() + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True) self.assertEqual(res['result'], 1) self.assertEqual(len(res['epidata']), len(expected)) # time types - res = self._fetch(time_types='day') + params = { + "time_types": "day" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertEqual(len(res['epidata']), sum([1 for s in expected if s['time_type'] == 'day'])) - res = self._fetch(time_types='day,week') + params = { + "time_types": "day,week" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertTrue(isinstance(res['epidata'], list)) self.assertEqual(len(res['epidata']), len(expected)) - res = self._fetch(time_types='sec') + params = { + "time_types": "sec" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], -2) # geo types - res = self._fetch(geo_types='hrr') + params = { + "geo_types": "hrr" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertTrue(isinstance(res['epidata'], list)) self.assertEqual(len(res['epidata']), sum([1 for s in expected if s['geo_type'] == 'hrr'])) - res = self._fetch(geo_types='hrr,msa') + params = { + "geo_types": "hrr,msa" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertTrue(isinstance(res['epidata'], list)) self.assertEqual(len(res['epidata']), len(expected)) - res = self._fetch(geo_types='state') + params = { + "geo_types": "state" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], -2) # signals - res = self._fetch(signals='src1:sig1') + params = { + "signals": "src1:sig1" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertTrue(isinstance(res['epidata'], list)) self.assertEqual(len(res['epidata']), sum([1 for s in expected if s['data_source'] == 'src1' and s['signal'] == 'sig1'])) - res = self._fetch(signals='src1') + params = { + "signals": "src1" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertTrue(isinstance(res['epidata'], list)) self.assertEqual(len(res['epidata']), sum([1 for s in expected if s['data_source'] == 'src1'])) - res = self._fetch(signals='src1:*') + params = { + "signals": "src1:*" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertTrue(isinstance(res['epidata'], list)) self.assertEqual(len(res['epidata']), sum([1 for s in expected if s['data_source'] == 'src1'])) - res = self._fetch(signals='src1:src4') + params = { + "signals": "src1:src4" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], -2) - res = self._fetch(signals='src1:*,src2:*') + params = { + "signals": "src1:*,src2:*" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertTrue(isinstance(res['epidata'], list)) self.assertEqual(len(res['epidata']), len(expected)) # filter fields - res = self._fetch(fields='data_source,min_time') + params = { + "fields": "data_source,min_time" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertEqual(len(res['epidata']), len(expected)) self.assertTrue('data_source' in res['epidata'][0]) @@ -262,8 +242,10 @@ def test_filter(self): self.assertFalse('max_time' in res['epidata'][0]) self.assertFalse('signal' in res['epidata'][0]) - res = self._fetch(fields='xx') + params = { + "fields": "xx" + } + res = self._make_request(endpoint="covidcast_meta", auth=self.epidata_client.auth, json=True, raise_for_status=True, params=params) self.assertEqual(res['result'], 1) self.assertEqual(len(res['epidata']), len(expected)) self.assertEqual(res['epidata'][0], {}) - diff --git a/integrations/server/test_delphi.py b/integrations/server/test_delphi.py index fc3e3bc7e..36e31e2b2 100644 --- a/integrations/server/test_delphi.py +++ b/integrations/server/test_delphi.py @@ -1,7 +1,7 @@ import json # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class DelphiTest(DelphiTestBase): diff --git a/integrations/server/test_dengue_nowcast.py b/integrations/server/test_dengue_nowcast.py index 79f9765f4..8762472b8 100644 --- a/integrations/server/test_dengue_nowcast.py +++ b/integrations/server/test_dengue_nowcast.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class DengueNowcastTest(DelphiTestBase): diff --git a/integrations/server/test_dengue_sensors.py b/integrations/server/test_dengue_sensors.py index 55d103367..424bc24d6 100644 --- a/integrations/server/test_dengue_sensors.py +++ b/integrations/server/test_dengue_sensors.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class DengueSensorsTest(DelphiTestBase): diff --git a/integrations/server/test_ecdc_ili.py b/integrations/server/test_ecdc_ili.py index 5fe24bcd3..e49e0bcfa 100644 --- a/integrations/server/test_ecdc_ili.py +++ b/integrations/server/test_ecdc_ili.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class EcdcIliTest(DelphiTestBase): diff --git a/integrations/server/test_flusurv.py b/integrations/server/test_flusurv.py index 33f0f00b8..586b28d1c 100644 --- a/integrations/server/test_flusurv.py +++ b/integrations/server/test_flusurv.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class FlusurvTest(DelphiTestBase): diff --git a/integrations/server/test_fluview.py b/integrations/server/test_fluview.py index 48d9585fd..eb6c1ec0f 100644 --- a/integrations/server/test_fluview.py +++ b/integrations/server/test_fluview.py @@ -1,50 +1,16 @@ """Integration tests for the `fluview` endpoint.""" -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.common.delphi_test_base import DelphiTestBase -class FluviewTests(unittest.TestCase): +class FluviewTests(DelphiTestBase): """Tests the `fluview` endpoint.""" - @classmethod - def setUpClass(cls): - """Perform one-time setup.""" - - # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' - Epidata.auth = ('epidata', 'key') - - def setUp(self): + def localSetUp(self): """Perform per-test setup.""" - # connect to the `epidata` database and clear the `fluview` table - cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='epidata') - cur = cnx.cursor() - cur.execute('truncate table fluview') - cur.execute('delete from api_user') - cur.execute('insert into api_user(api_key, email) values ("key", "email")') - cnx.commit() - cur.close() - - # make connection and cursor available to test cases - self.cnx = cnx - self.cur = cnx.cursor() - - def tearDown(self): - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() + self.truncate_tables_list = ["fluview"] def test_round_trip(self): """Make a simple round-trip with some sample data.""" @@ -62,7 +28,7 @@ def test_round_trip(self): self.cnx.commit() # make the request - response = Epidata.fluview('nat', 202020) + response = self.epidata_client.fluview('nat', 202020) # assert that the right data came back self.assertEqual(response, { diff --git a/integrations/server/test_fluview_clinical.py b/integrations/server/test_fluview_clinical.py index 8ebccfc9e..300e0cb73 100644 --- a/integrations/server/test_fluview_clinical.py +++ b/integrations/server/test_fluview_clinical.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class FluviewClinicalTest(DelphiTestBase): diff --git a/integrations/server/test_fluview_meta.py b/integrations/server/test_fluview_meta.py index 6f81c1859..b23d79f9d 100644 --- a/integrations/server/test_fluview_meta.py +++ b/integrations/server/test_fluview_meta.py @@ -1,50 +1,15 @@ """Integration tests for the `fluview_meta` endpoint.""" -# standard library -import unittest - -# third party -import mysql.connector - # first party -from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.common.delphi_test_base import DelphiTestBase -class FluviewMetaTests(unittest.TestCase): +class FluviewMetaTests(DelphiTestBase): """Tests the `fluview_meta` endpoint.""" - @classmethod - def setUpClass(cls): - """Perform one-time setup.""" - - # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' - Epidata.auth = ('epidata', 'key') - - def setUp(self): + def localSetUp(self): """Perform per-test setup.""" - - # connect to the `epidata` database and clear the `fluview` table - cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='epidata') - cur = cnx.cursor() - cur.execute('truncate table fluview') - cur.execute('delete from api_user') - cur.execute('insert into api_user(api_key, email) values ("key", "email")') - cnx.commit() - cur.close() - - # make connection and cursor available to test cases - self.cnx = cnx - self.cur = cnx.cursor() - - def tearDown(self): - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() + self.truncate_tables_list = ["fluview"] def test_round_trip(self): """Make a simple round-trip with some sample data.""" @@ -64,7 +29,7 @@ def test_round_trip(self): self.cnx.commit() # make the request - response = Epidata.fluview_meta() + response = self.epidata_client.fluview_meta() # assert that the right data came back self.assertEqual(response, { diff --git a/integrations/server/test_gft.py b/integrations/server/test_gft.py index 3f2a68526..80a0dc984 100644 --- a/integrations/server/test_gft.py +++ b/integrations/server/test_gft.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class GftTest(DelphiTestBase): diff --git a/integrations/server/test_ght.py b/integrations/server/test_ght.py index 67b135aef..19be5b6a1 100644 --- a/integrations/server/test_ght.py +++ b/integrations/server/test_ght.py @@ -1,5 +1,6 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase + class GhtTest(DelphiTestBase): diff --git a/integrations/server/test_kcdc_ili.py b/integrations/server/test_kcdc_ili.py index aab40baa6..48c1e5160 100644 --- a/integrations/server/test_kcdc_ili.py +++ b/integrations/server/test_kcdc_ili.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class KcdcIliTest(DelphiTestBase): diff --git a/integrations/server/test_meta.py b/integrations/server/test_meta.py index 295a5cd98..e0cd1013f 100644 --- a/integrations/server/test_meta.py +++ b/integrations/server/test_meta.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class MetaTest(DelphiTestBase): diff --git a/integrations/server/test_nidss_dengue.py b/integrations/server/test_nidss_dengue.py index 679937ea6..1b08048ce 100644 --- a/integrations/server/test_nidss_dengue.py +++ b/integrations/server/test_nidss_dengue.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class NiddsDengueTest(DelphiTestBase): diff --git a/integrations/server/test_nidss_flu.py b/integrations/server/test_nidss_flu.py index 0b13ee67f..678017502 100644 --- a/integrations/server/test_nidss_flu.py +++ b/integrations/server/test_nidss_flu.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class NiddsFluTest(DelphiTestBase): diff --git a/integrations/server/test_norostat.py b/integrations/server/test_norostat.py index a7866a91d..bfc2c2b80 100644 --- a/integrations/server/test_norostat.py +++ b/integrations/server/test_norostat.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class NorostatTest(DelphiTestBase): diff --git a/integrations/server/test_nowcast.py b/integrations/server/test_nowcast.py index 2b48dd0da..38897f0ae 100644 --- a/integrations/server/test_nowcast.py +++ b/integrations/server/test_nowcast.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class NowcastTest(DelphiTestBase): diff --git a/integrations/server/test_paho_dengue.py b/integrations/server/test_paho_dengue.py index bbe8953f8..911986897 100644 --- a/integrations/server/test_paho_dengue.py +++ b/integrations/server/test_paho_dengue.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class PahoDengueTest(DelphiTestBase): diff --git a/integrations/server/test_quidel.py b/integrations/server/test_quidel.py index 696a7ee41..30902eeae 100644 --- a/integrations/server/test_quidel.py +++ b/integrations/server/test_quidel.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class QuidelTest(DelphiTestBase): diff --git a/integrations/server/test_sensors.py b/integrations/server/test_sensors.py index 835b53893..c5afa329e 100644 --- a/integrations/server/test_sensors.py +++ b/integrations/server/test_sensors.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class SensorsTest(DelphiTestBase): diff --git a/integrations/server/test_signal_dashboard.py b/integrations/server/test_signal_dashboard.py index 1aed3eec6..fefca1a81 100644 --- a/integrations/server/test_signal_dashboard.py +++ b/integrations/server/test_signal_dashboard.py @@ -1,10 +1,12 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class SignalDashboardTest(DelphiTestBase): """Basic integration tests for signal_dashboard_coverage and signal_dashboard_status endpints.""" + # NOTE: In all other tests localSetUp() method was used. But it is not applicable for this test + # due to order of commands, so thats why method reload + calling super was required. def setUp(self) -> None: """Perform per-test setup.""" @@ -24,7 +26,7 @@ def setUp(self) -> None: def test_signal_dashboard_coverage(self): """Basic integration test for signal_dashboard_coverage endpoint""" - + response = self.epidata_client._request("signal_dashboard_coverage") self.assertEqual( response, diff --git a/integrations/server/test_twitter.py b/integrations/server/test_twitter.py index 7ba162480..18891c009 100644 --- a/integrations/server/test_twitter.py +++ b/integrations/server/test_twitter.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class TwitterTest(DelphiTestBase): diff --git a/integrations/server/test_wiki.py b/integrations/server/test_wiki.py index d53cef0c9..21638b43e 100644 --- a/integrations/server/test_wiki.py +++ b/integrations/server/test_wiki.py @@ -1,5 +1,5 @@ # first party -from delphi.epidata.common.integration_test_base_class import DelphiTestBase +from delphi.epidata.common.delphi_test_base import DelphiTestBase class WikiTest(DelphiTestBase): diff --git a/src/acquisition/covidcast/test_utils.py b/src/common/covidcast_test_base.py similarity index 60% rename from src/acquisition/covidcast/test_utils.py rename to src/common/covidcast_test_base.py index 5a978f8cd..1ab58a911 100644 --- a/src/acquisition/covidcast/test_utils.py +++ b/src/common/covidcast_test_base.py @@ -1,17 +1,15 @@ from dataclasses import fields from datetime import date from typing import Any, Dict, Iterable, List, Optional, Sequence -import unittest import pandas as pd -from redis import Redis - -from delphi_utils import Nans -from delphi.epidata.common.covidcast_row import CovidcastRow from delphi.epidata.acquisition.covidcast.database import Database -from delphi.epidata.server._config import REDIS_HOST, REDIS_PASSWORD +from delphi.epidata.common.covidcast_row import CovidcastRow from delphi.epidata.server.utils.dates import day_to_time_value, time_value_to_day import delphi.operations.secrets as secrets +from delphi_utils import Nans + +from delphi.epidata.common.delphi_test_base import DelphiTestBase # all the Nans we use here are just one value, so this is a shortcut to it: nmv = Nans.NOT_MISSING.value @@ -19,8 +17,14 @@ # TODO replace these real geo_values with fake values, and use patch and mock to mock the return values of # delphi_utils.geomap.GeoMapper().get_geo_values(geo_type) in parse_geo_sets() of _params.py -FIPS = ['04019', '19143', '29063', '36083'] # Example list of valid FIPS codes as strings -MSA = ['40660', '44180', '48620', '49420'] # Example list of valid MSA codes as strings +FIPS = [ + "04019", + "19143", + "29063", + "36083", +] # Example list of valid FIPS codes as strings +MSA = ["40660", "44180", "48620", "49420"] # Example list of valid MSA codes as strings + class CovidcastTestRow(CovidcastRow): @staticmethod @@ -51,31 +55,89 @@ def __post_init__(self): if isinstance(self.issue, date): self.issue = day_to_time_value(self.issue) if isinstance(self.value_updated_timestamp, date): - self.value_updated_timestamp = day_to_time_value(self.value_updated_timestamp) + self.value_updated_timestamp = day_to_time_value( + self.value_updated_timestamp + ) def _sanitize_fields(self, extra_checks: bool = True): if self.issue and self.issue < self.time_value: self.issue = self.time_value if self.issue: - self.lag = (time_value_to_day(self.issue) - time_value_to_day(self.time_value)).days + self.lag = ( + time_value_to_day(self.issue) - time_value_to_day(self.time_value) + ).days else: self.lag = None # This sanity checking is already done in CsvImporter, but it's here so the testing class gets it too. if pd.isna(self.value) and self.missing_value == Nans.NOT_MISSING: - self.missing_value = Nans.NOT_APPLICABLE.value if extra_checks else Nans.OTHER.value + self.missing_value = ( + Nans.NOT_APPLICABLE.value if extra_checks else Nans.OTHER.value + ) if pd.isna(self.stderr) and self.missing_stderr == Nans.NOT_MISSING: - self.missing_stderr = Nans.NOT_APPLICABLE.value if extra_checks else Nans.OTHER.value + self.missing_stderr = ( + Nans.NOT_APPLICABLE.value if extra_checks else Nans.OTHER.value + ) if pd.isna(self.sample_size) and self.missing_sample_size == Nans.NOT_MISSING: - self.missing_sample_size = Nans.NOT_APPLICABLE.value if extra_checks else Nans.OTHER.value + self.missing_sample_size = ( + Nans.NOT_APPLICABLE.value if extra_checks else Nans.OTHER.value + ) return self -def covidcast_rows_from_args(sanitize_fields: bool = False, test_mode: bool = True, **kwargs: Dict[str, Iterable]) -> List[CovidcastTestRow]: +class CovidcastTestBase(DelphiTestBase): + def setUp(self): + # use the local test instance of the database + secrets.db.host = 'delphi_database_epidata' + secrets.db.epi = ('user', 'pass') + + self._db = Database() + self._db.connect() + + # empty all of the data tables + for ( + table + ) in "epimetric_load epimetric_latest epimetric_full geo_dim signal_dim".split(): + self._db._cursor.execute(f"TRUNCATE TABLE {table};") + + # reset the `covidcast_meta_cache` table (it should always have one row) + self._db._cursor.execute( + 'update covidcast_meta_cache set timestamp = 0, epidata = "[]"' + ) + self._db._connection.commit() + super().setUp() + + def localTearDown(self): + self._db.disconnect(False) + del self._db + + def _insert_rows(self, rows: Sequence[CovidcastTestRow]): + # inserts rows into the database using the full acquisition process, including 'dbjobs' load into history & latest tables + n = self._db.insert_or_update_bulk(rows) + print(f"{n} rows added to load table & dispatched to v4 schema") + # NOTE: this isnt expressly needed for our test cases, but would be if using external access (like through client lib) to ensure changes are visible outside of this db session + self._db._connection.commit() + + def params_from_row(self, row: CovidcastTestRow, **kwargs): + ret = { + "data_source": row.source, + "signals": row.signal, + "time_type": row.time_type, + "geo_type": row.geo_type, + "time_values": row.time_value, + "geo_value": row.geo_value, + } + ret.update(kwargs) + return ret + + +def covidcast_rows_from_args( + sanitize_fields: bool = False, test_mode: bool = True, **kwargs: Dict[str, Iterable] +) -> List[CovidcastTestRow]: """A convenience constructor for test rows. Example: @@ -89,12 +151,22 @@ def covidcast_rows_from_args(sanitize_fields: bool = False, test_mode: bool = Tr assert len(set(len(lst) for lst in kwargs.values())) == 1 if sanitize_fields: - return [CovidcastTestRow.make_default_row(**_kwargs)._sanitize_fields(extra_checks=test_mode) for _kwargs in transpose_dict(kwargs)] + return [ + CovidcastTestRow.make_default_row(**_kwargs)._sanitize_fields( + extra_checks=test_mode + ) + for _kwargs in transpose_dict(kwargs) + ] else: - return [CovidcastTestRow.make_default_row(**_kwargs) for _kwargs in transpose_dict(kwargs)] + return [ + CovidcastTestRow.make_default_row(**_kwargs) + for _kwargs in transpose_dict(kwargs) + ] -def covidcast_rows_from_records(records: Iterable[dict], sanity_check: bool = False) -> List[CovidcastTestRow]: +def covidcast_rows_from_records( + records: Iterable[dict], sanity_check: bool = False +) -> List[CovidcastTestRow]: """A convenience constructor. Default is different from from_args, because from_records is usually called on faux-API returns in tests, @@ -103,36 +175,60 @@ def covidcast_rows_from_records(records: Iterable[dict], sanity_check: bool = Fa You can use csv.DictReader before this to read a CSV file. """ records = list(records) - return [CovidcastTestRow.make_default_row(**record) if not sanity_check else CovidcastTestRow.make_default_row(**record)._sanitize_fields() for record in records] + return [ + CovidcastTestRow.make_default_row(**record) + if not sanity_check + else CovidcastTestRow.make_default_row(**record)._sanitize_fields() + for record in records + ] -def covidcast_rows_as_dicts(rows: Iterable[CovidcastTestRow], ignore_fields: Optional[List[str]] = None) -> List[dict]: +def covidcast_rows_as_dicts( + rows: Iterable[CovidcastTestRow], ignore_fields: Optional[List[str]] = None +) -> List[dict]: return [row.as_dict(ignore_fields=ignore_fields) for row in rows] -def covidcast_rows_as_dataframe(rows: Iterable[CovidcastTestRow], ignore_fields: Optional[List[str]] = None) -> pd.DataFrame: +def covidcast_rows_as_dataframe( + rows: Iterable[CovidcastTestRow], ignore_fields: Optional[List[str]] = None +) -> pd.DataFrame: if ignore_fields is None: ignore_fields = [] - columns = [field.name for field in fields(CovidcastTestRow) if field.name not in ignore_fields] + columns = [ + field.name + for field in fields(CovidcastTestRow) + if field.name not in ignore_fields + ] if rows: - df = pd.concat([row.as_dataframe(ignore_fields=ignore_fields) for row in rows], ignore_index=True) + df = pd.concat( + [row.as_dataframe(ignore_fields=ignore_fields) for row in rows], + ignore_index=True, + ) return df[columns] else: return pd.DataFrame(columns=columns) def covidcast_rows_as_api_row_df(rows: Iterable[CovidcastTestRow]) -> pd.DataFrame: - return covidcast_rows_as_dataframe(rows, ignore_fields=CovidcastTestRow._api_row_ignore_fields) + return covidcast_rows_as_dataframe( + rows, ignore_fields=CovidcastTestRow._api_row_ignore_fields + ) -def covidcast_rows_as_api_compatibility_row_df(rows: Iterable[CovidcastTestRow]) -> pd.DataFrame: - return covidcast_rows_as_dataframe(rows, ignore_fields=CovidcastTestRow._api_row_compatibility_ignore_fields) +def covidcast_rows_as_api_compatibility_row_df( + rows: Iterable[CovidcastTestRow], +) -> pd.DataFrame: + return covidcast_rows_as_dataframe( + rows, ignore_fields=CovidcastTestRow._api_row_compatibility_ignore_fields + ) def covidcast_rows_as_db_row_df(rows: Iterable[CovidcastTestRow]) -> pd.DataFrame: - return covidcast_rows_as_dataframe(rows, ignore_fields=CovidcastTestRow._db_row_ignore_fields) + return covidcast_rows_as_dataframe( + rows, ignore_fields=CovidcastTestRow._db_row_ignore_fields + ) def transpose_dict(d: Dict[Any, List[Any]]) -> List[Dict[Any, Any]]: @@ -145,65 +241,11 @@ def transpose_dict(d: Dict[Any, List[Any]]) -> List[Dict[Any, Any]]: return [dict(zip(d.keys(), values)) for values in zip(*d.values())] -def assert_frame_equal_no_order(df1: pd.DataFrame, df2: pd.DataFrame, index: List[str], **kwargs: Any) -> None: +def assert_frame_equal_no_order( + df1: pd.DataFrame, df2: pd.DataFrame, index: List[str], **kwargs: Any +) -> None: """Assert that two DataFrames are equal, ignoring the order of rows.""" # Remove any existing index. If it wasn't named, drop it. Set a new index and sort it. df1 = df1.reset_index().drop(columns="index").set_index(index).sort_index() df2 = df2.reset_index().drop(columns="index").set_index(index).sort_index() pd.testing.assert_frame_equal(df1, df2, **kwargs) - - -class CovidcastBase(unittest.TestCase): - def setUp(self): - # use the local test instance of the database - secrets.db.host = 'delphi_database_epidata' - secrets.db.epi = ('user', 'pass') - - self._db = Database() - self._db.connect() - - # empty all of the data tables - for table in "epimetric_load epimetric_latest epimetric_full geo_dim signal_dim".split(): - self._db._cursor.execute(f"TRUNCATE TABLE {table};") - self.localSetUp() - self._db._connection.commit() - - # clear all rate-limiting info from redis - r = Redis(host=REDIS_HOST, password=REDIS_PASSWORD) - for k in r.keys("LIMITER/*"): - r.delete(k) - - - def tearDown(self): - # close and destroy conenction to the database - self.localTearDown() - self._db.disconnect(False) - del self._db - - def localSetUp(self): - # stub; override in subclasses to perform custom setup. - # runs after tables have been truncated but before database changes have been committed - pass - - def localTearDown(self): - # stub; override in subclasses to perform custom teardown. - # runs after database changes have been committed - pass - - def _insert_rows(self, rows: Sequence[CovidcastTestRow]): - # inserts rows into the database using the full acquisition process, including 'dbjobs' load into history & latest tables - n = self._db.insert_or_update_bulk(rows) - print(f"{n} rows added to load table & dispatched to v4 schema") - self._db._connection.commit() # NOTE: this isnt expressly needed for our test cases, but would be if using external access (like through client lib) to ensure changes are visible outside of this db session - - def params_from_row(self, row: CovidcastTestRow, **kwargs): - ret = { - 'data_source': row.source, - 'signals': row.signal, - 'time_type': row.time_type, - 'geo_type': row.geo_type, - 'time_values': row.time_value, - 'geo_value': row.geo_value, - } - ret.update(kwargs) - return ret diff --git a/src/common/delphi_test_base.py b/src/common/delphi_test_base.py new file mode 100644 index 000000000..540d9f321 --- /dev/null +++ b/src/common/delphi_test_base.py @@ -0,0 +1,120 @@ +# standard library +from __future__ import annotations +import unittest +from typing import Optional + + +# third party +import mysql.connector +import requests + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class DelphiTestBase(unittest.TestCase): + """Basic integration test class""" + + def __init__(self, methodName: str = "runTest") -> None: + super().__init__(methodName) + self.delete_from_tables_list = [] + self.truncate_tables_list = [] + self.create_tables_list = [] + self.role_name = None + self.epidata_client = Epidata + self.epidata_client.BASE_URL = "http://delphi_web_epidata/epidata" + self.epidata_client.auth = ("epidata", "key") + + def create_key_with_role(self, cur, role_name: str): + cur.execute( + f'INSERT INTO `api_user`(`api_key`, `email`) VALUES("{role_name}_key", "{role_name}_email")' + ) + cur.execute(f'INSERT INTO `user_role`(`name`) VALUES("{role_name}")') + cur.execute( + f'INSERT INTO `user_role_link`(`user_id`, `role_id`) SELECT `api_user`.`id`, `user_role`.`id` FROM `api_user` JOIN `user_role` WHERE `api_user`.`api_key`="{role_name}_key" AND `user_role`.`name`="{role_name}"' + ) + + def _make_request( + self, + endpoint: str = "covidcast", + auth: Optional[tuple | str] = None, + json: bool = False, + raise_for_status: bool = False, + params: dict = {}, + is_compatibility: bool = False + ): + url = f"{self.epidata_client.BASE_URL}/{endpoint}" + if is_compatibility: + url = "http://delphi_web_epidata/epidata/api.php" + params.setdefault("endpoint", "covidcast") + if params.get("source"): + params.setdefault("data_source", params.get("source")) + response = requests.get( + url, params=params, auth=auth + ) + if raise_for_status: + response.raise_for_status() + if json: + return response.json() + return response + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect( + user="user", + password="pass", + host="delphi_database_epidata", + database="epidata", + ) + cur = cnx.cursor() + + cur.execute("DELETE FROM `api_user`") + cur.execute("TRUNCATE TABLE `user_role`") + cur.execute("TRUNCATE TABLE `user_role_link`") + cur.execute( + 'INSERT INTO `api_user`(`api_key`, `email`) VALUES ("key", "email")' + ) + + self.localSetUp() + + for stmt in self.create_tables_list: + cur.execute(stmt) + + for table_name in self.delete_from_tables_list: + cur.execute(f"DELETE FROM `{table_name}`") + + for table_name in self.truncate_tables_list: + cur.execute(f"TRUNCATE TABLE `{table_name}`") + + if self.role_name: + self.create_key_with_role(cur, self.role_name) + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + def localSetUp(self): + # stub; override in subclasses to perform custom setup. + # runs after user/api_key tables have been truncated, but before test-specific tables are created/deleted/truncated and before database changes have been committed + pass + + def localTearDown(self): + # stub; override in subclasses to perform custom teardown. + # runs after database changes have been committed + pass + + @staticmethod + def _clear_limits() -> None: + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.localTearDown() + self.cur.close() + self.cnx.close() + self._clear_limits() diff --git a/tests/common/test_covidcast_row.py b/tests/common/test_covidcast_row.py index 273596ebb..77e2e441e 100644 --- a/tests/common/test_covidcast_row.py +++ b/tests/common/test_covidcast_row.py @@ -5,20 +5,21 @@ from delphi_utils.nancodes import Nans from delphi.epidata.common.covidcast_row import CovidcastRow, set_df_dtypes -from delphi.epidata.acquisition.covidcast.test_utils import ( +from delphi.epidata.common.covidcast_test_base import ( CovidcastTestRow, covidcast_rows_as_api_compatibility_row_df, covidcast_rows_as_api_row_df, covidcast_rows_from_args, transpose_dict, + MSA, + CovidcastTestBase ) -from delphi.epidata.acquisition.covidcast.test_utils import MSA # py3tester coverage target (equivalent to `import *`) __test_target__ = "delphi.epidata.common.covidcast_row" -class TestCovidcastRows(unittest.TestCase): +class TestCovidcastRows(CovidcastTestBase): expected_df = set_df_dtypes( DataFrame( { diff --git a/tests/server/test_params.py b/tests/server/test_params.py index cd287445c..c4058ad1a 100644 --- a/tests/server/test_params.py +++ b/tests/server/test_params.py @@ -28,7 +28,7 @@ from delphi.epidata.server._exceptions import ( ValidationFailedException, ) -from delphi.epidata.acquisition.covidcast.test_utils import FIPS, MSA +from delphi.epidata.common.covidcast_test_base import FIPS, MSA # py3tester coverage target __test_target__ = "delphi.epidata.server._params" diff --git a/tests/server/test_query.py b/tests/server/test_query.py index 95b21a55a..614e4fb22 100644 --- a/tests/server/test_query.py +++ b/tests/server/test_query.py @@ -19,7 +19,7 @@ TimeSet, SourceSignalSet, ) -from delphi.epidata.acquisition.covidcast.test_utils import FIPS, MSA +from delphi.epidata.common.covidcast_test_base import FIPS, MSA # py3tester coverage target __test_target__ = "delphi.epidata.server._query"