refactor(covid_hosp test): simplify

Author: dshemetov

integrations/acquisition/covid_hosp/state_daily/test_scenarios.py

@@ -47,6 +47,17 @@ def setUp(self):
         cur.execute('delete from api_user')
         cur.execute('insert into api_user(api_key, email) values("key", "email")')
 
+  def get_modified_dataset(self, critical_staffing_shortage_today_yes, reporting_cutoff_start):
+    """Get a simplified version of a test dataset.
+
+    Only WY data is modified. The issue date is specified in the metadata file.
+    """
+    df = self.test_utils.load_sample_dataset()
+    df_new = pd.DataFrame(df[df["state"] == "WY"], columns=df.columns).reset_index(drop=True)
+    df_new["critical_staffing_shortage_today_yes"] = critical_staffing_shortage_today_yes
+    df_new["reporting_cutoff_start"] = reporting_cutoff_start
+    return df_new
+
   def test_acquire_dataset(self):
     """Acquire a new dataset."""
 
@@ -58,12 +69,13 @@ def test_acquire_dataset(self):
 
       # acquire sample data into local database
       # mock out network calls to external hosts
+      # issues: 3/13, 3/15
       with self.subTest(name='first acquisition'), \
           patch.object(Network, 'fetch_metadata',
                        return_value=self.test_utils.load_sample_metadata("metadata.csv")) as mock_fetch_meta, \
           patch.object(Network, 'fetch_dataset', side_effect=[
-            self.test_utils.load_sample_dataset("dataset0.csv"), # dataset for 3/13
-            self.test_utils.load_sample_dataset("dataset0.csv")  # dataset for 3/15
+            self.test_utils.load_sample_dataset(),
+            self.test_utils.load_sample_dataset()
           ]) as mock_fetch:
         acquired = Update.run()
         self.assertTrue(acquired)
@@ -78,7 +90,7 @@ def test_acquire_dataset(self):
         self.assertEqual(row['state'], 'WY')
         self.assertEqual(row['date'], 20201209)
         self.assertEqual(row['issue'], 20210315) # include today's data by default
-        self.assertEqual(row['critical_staffing_shortage_today_yes'], 5)
+        self.assertEqual(row['critical_staffing_shortage_today_yes'], 8)
         self.assertEqual(row['total_patients_hospitalized_confirmed_influenza_covid_coverage'], 56)
         self.assertIsNone(row['critical_staffing_shortage_today_no'])
 
@@ -90,12 +102,13 @@ def test_acquire_dataset(self):
         self.assertEqual(response['result'], 1)
 
       # re-acquisition of the same dataset should be a no-op
+      # issues: 3/13, 3/15
       with self.subTest(name='second acquisition'), \
           patch.object(Network, 'fetch_metadata',
                        return_value=self.test_utils.load_sample_metadata("metadata.csv")) as mock_fetch_meta, \
           patch.object(Network, 'fetch_dataset', side_effect=[
-            self.test_utils.load_sample_dataset("dataset0.csv"),
-            self.test_utils.load_sample_dataset("dataset0.csv")
+            self.test_utils.load_sample_dataset(),
+            self.test_utils.load_sample_dataset()
           ]) as mock_fetch:
         acquired = Update.run()
         self.assertFalse(acquired)
@@ -111,8 +124,10 @@ def test_acquire_dataset(self):
           patch.object(Network, 'fetch_metadata',
                       return_value=self.test_utils.load_sample_metadata("metadata2.csv")) as mock_fetch_meta, \
           patch.object(Network, 'fetch_dataset', side_effect=[
-            self.test_utils.load_sample_dataset("dataset1.csv"),
-            self.test_utils.load_sample_dataset("dataset2.csv"),
+            self.get_modified_dataset(critical_staffing_shortage_today_yes = 9, reporting_cutoff_start="2020-12-09"),
+            self.get_modified_dataset(critical_staffing_shortage_today_yes = 10, reporting_cutoff_start="2020-12-09"),
+            self.get_modified_dataset(critical_staffing_shortage_today_yes = 11, reporting_cutoff_start="2020-12-10"),
+            self.get_modified_dataset(critical_staffing_shortage_today_yes = 12, reporting_cutoff_start="2020-12-10"),
           ]) as mock_fetch:
         acquired = Update.run()
         self.assertTrue(acquired)
@@ -124,19 +139,21 @@ def test_acquire_dataset(self):
         self.assertEqual(response['result'], 1)
         self.assertEqual(len(response['epidata']), 2)
 
-        row = response['epidata'][0] # data from 03-15 00:01AM, dataset1.csv
+        # should have data from 03-15 00:00:01AM
+        row = response['epidata'][0]
         self.assertEqual(row['state'], 'WY')
         self.assertEqual(row['date'], 20201209)
         self.assertEqual(row['issue'], 20210315) # include today's data by default
-        self.assertEqual(row['critical_staffing_shortage_today_yes'], 8)
+        self.assertEqual(row['critical_staffing_shortage_today_yes'], 10)
         self.assertEqual(row['total_patients_hospitalized_confirmed_influenza_covid_coverage'], 56)
         self.assertIsNone(row['critical_staffing_shortage_today_no'])
 
-        row = response['epidata'][1] # data from 03-16 00:00AM, dataset1.csv
+        # should have data from 03-16 00:00:01AM, dataset4.csv
+        row = response['epidata'][1]
         self.assertEqual(row['state'], 'WY')
         self.assertEqual(row['date'], 20201210)
         self.assertEqual(row['issue'], 20210316) # include today's data by default
-        self.assertEqual(row['critical_staffing_shortage_today_yes'], 10)
+        self.assertEqual(row['critical_staffing_shortage_today_yes'], 12)
         self.assertEqual(row['total_patients_hospitalized_confirmed_influenza_covid_coverage'], 56)
         self.assertIsNone(row['critical_staffing_shortage_today_no'])
 
@@ -164,7 +181,7 @@ def test_acquire_specific_issue(self):
     self.assertEqual(pre_max_issue, pd.Timestamp('1900-01-01 00:00:00'))
     with self.subTest(name='first acquisition'), \
          patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()) as mock_fetch_meta, \
-         patch.object(Network, 'fetch_dataset', side_effect=[self.test_utils.load_sample_dataset("dataset0.csv")]
+         patch.object(Network, 'fetch_dataset', side_effect=[self.test_utils.load_sample_dataset()]
                       ) as mock_fetch:
       acquired = Utils.update_dataset(Database,
                                       Network,

testdata/acquisition/covid_hosp/state_daily/dataset.csv

@@ -51,4 +51,4 @@ MI,30,129,4,32,127,4,41,159,23598,163,18003,163,3812,159,376,163,162,159,9,159,9
 MN,21,116,2,26,111,2,63,138,10358,139,7558,139,1516,138,182,139,70,138,3,138,2,138,806,139,346,138,355,139,1490,138,1358,139,26,138,21,138,1019,139,0.7296775439273991,139,7558,10358,0.2082417582417582,138,1516,7280,0.151630326065213,138,1516,9998,0.365364308342133,138,346,947,0.7909715407262021,139,806,1019,2020-12-09,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56
 MO,47,78,16,62,63,16,22,137,17433,141,13521,141,2611,137,315,141,239,137,5,137,18,137,1615,141,645,137,604,141,2546,137,2307,141,65,137,26,137,1931,141,0.7755980037859233,141,13521,17433,0.1964487247009254,137,2611,13291,0.1523959610109146,137,2611,17133,0.3456591639871382,137,645,1866,0.8363542206110823,141,1615,1931,2020-12-09,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56
 MS,21,85,2,30,76,2,12,106,8799,108,5637,108,1254,106,142,108,30,106,3,106,5,106,718,108,338,106,263,108,1234,106,1066,108,20,106,5,106,881,108,0.6406409819297647,108,5637,8799,0.2250134577426879,106,1254,5573,0.143922873866636,106,1254,8713,0.3953216374269006,106,338,855,0.8149829738933031,108,718,881,2020-12-09,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56
-WY,8,,2,7,22,2,5,29,1729,31,856,31,198,29,26,31,15,29,0,29,0,29,58,31,32,29,32,31,196,29,189,31,2,29,2,29,137,31,0.4950838635049161,31,856,1729,0.2362768496420047,29,198,838,0.1172985781990521,29,198,1688,0.2519685039370078,29,32,127,0.4233576642335766,31,58,137,2020/12/09,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56
+WY,8,,2,7,22,2,5,29,1729,31,856,31,198,29,26,31,15,29,0,29,0,29,58,31,32,29,32,31,196,29,189,31,2,29,2,29,137,31,0.4950838635049161,31,856,1729,0.2362768496420047,29,198,838,0.1172985781990521,29,198,1688,0.2519685039370078,29,32,127,0.4233576642335766,31,58,137,2020-12-09,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56