add extra duplication checks and check if tables exists

Author: cchuong

src/acquisition/rvdss/pull_historic.py

@@ -491,13 +491,15 @@ def fetch_one_season_from_report(url):
             # The columns have the region information (i.e Pr tests, meaning this columns has the tests for the prairies)
 
             if "reporting laboratory" in str(table.columns):
+               respiratory_detection_table_exists = True
                respiratory_detection_table = create_detections_table(table,modified_date,current_week,current_week_end,season[0])
                respiratory_detection_table = respiratory_detection_table.set_index(['epiweek', 'time_value', 'issue', 'geo_type', 'geo_value'])
             elif "number" in caption.text.lower():
                number_table_exists = True
                number_detections_table = create_number_detections_table(table,modified_date,season[0])
                number_detections_table = number_detections_table.set_index(['epiweek', 'time_value', 'issue', 'geo_type', 'geo_value'])
             elif "positive" in caption.text.lower():
+               positive_table_exists = True
                flu = " influenza" in caption.text.lower()
 
                # tables are missing week 53
@@ -531,20 +533,21 @@ def fetch_one_season_from_report(url):
 
         # Check if the indices are already in the season table
         # If not, add the weeks tables into the season table
-
+                
         # check for deduplication pandas
-        if not number_detections_table.index.isin(all_number_tables.index).any():
-            all_number_tables=pd.concat([all_number_tables,number_detections_table])
-        
-        if not respiratory_detection_table.index.isin(all_respiratory_detection_tables.index).any():
-              all_respiratory_detection_tables= pd.concat([all_respiratory_detection_tables,respiratory_detection_table])
-
-        if not combined_positive_tables.index.isin(all_positive_tables.index).any():
-            all_positive_tables=pd.concat([all_positive_tables,combined_positive_tables])
-
+        if respiratory_detection_table_exists:
+            if not respiratory_detection_table.index.isin(all_respiratory_detection_tables.index).any():
+                  all_respiratory_detection_tables= pd.concat([all_respiratory_detection_tables,respiratory_detection_table])
+            del respiratory_detection_table
+        if positive_table_exists:
+            if not combined_positive_tables.index.isin(all_positive_tables.index).any():
+                all_positive_tables=pd.concat([all_positive_tables,combined_positive_tables])
+            del combined_positive_tables
+            del pos_table
         if number_table_exists:
             if not number_detections_table.index.isin(all_number_tables.index).any():
                 all_number_tables=pd.concat([all_number_tables,number_detections_table])
+            del number_detections_table
 
     return {
         "respiratory_detection": all_respiratory_detection_tables,

src/acquisition/rvdss/run.py

@@ -14,7 +14,6 @@
 from delphi.epidata.acquisition.rvdss.database import respiratory_detections_cols, pct_positive_cols, detections_counts_cols, expected_table_names, expected_columns, get_num_rows, update
 
 def update_current_data():
-
     ## Check if data for current update date has already been fetched
     headers = {
         'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36'
@@ -27,9 +26,8 @@ def update_current_data():
         with open(UPDATE_DATES_FILE, 'a') as testfile:
             testfile.write(update_date+ "\n")
 
-
         data_dict = fetch_dashboard_data(DASHBOARD_BASE_URL)
-        ## TODO
+        # update database
         update(data_dict)
     else:
         print("Data is already up to date")
@@ -65,7 +63,6 @@ def update_historical_data():
     update(hist_dict_list)
 
 
-
 def main():
     # args and usage
     parser = argparse.ArgumentParser()