Merge pull request #168 from rustprooflabs/file-handling

File handling  improvements

docker/pgosm_flex.py

@@ -121,7 +121,12 @@ def run_pgosm_flex(layerset, ram, region, subregion, srid, pgosm_date, language,
     run_post_processing(layerset=layerset, paths=paths,
                         skip_nested=skip_nested)
 
-    export_filename = get_export_filename(region, subregion, layerset)
+    remove_latest_files(region, subregion, paths)
+
+    export_filename = get_export_filename(region,
+                                          subregion,
+                                          layerset,
+                                          pgosm_date)
 
     if schema_name != 'osm':
         db.rename_schema(schema_name)
@@ -269,14 +274,15 @@ def get_region_filename(region, subregion):
     return filename
 
 
-def get_export_filename(region, subregion, layerset):
+def get_export_filename(region, subregion, layerset, pgosm_date):
     """Returns the .sql filename to use from pg_dump.
 
     Parameters
     ----------------------
     region : str
     subregion : str
     layerset : str
+    pgosm_date : str
 
     Returns
     ----------------------
@@ -285,9 +291,9 @@ def get_export_filename(region, subregion, layerset):
     region = region.replace('/', '-')
     subregion = subregion.replace('/', '-')
     if subregion is None:
-        filename = f'pgosm-flex-{region}-{layerset}.sql'
+        filename = f'pgosm-flex-{region}-{layerset}-{pgosm_date}.sql'
     else:
-        filename = f'pgosm-flex-{region}-{subregion}-{layerset}.sql'
+        filename = f'pgosm-flex-{region}-{subregion}-{layerset}-{pgosm_date}.sql'
 
     return filename
 
@@ -533,6 +539,26 @@ def unarchive_data(pbf_file, md5_file, pbf_file_with_date, md5_file_with_date):
     shutil.copy2(md5_file_with_date, md5_file)
 
 
+def remove_latest_files(region, subregion, paths):
+    """Removes the PBF and MD5 file with -latest in the name.
+
+    Files are archived via prepare_data() before processing starts
+
+    Parameters
+    -------------------------
+    region : str
+    subregion : str
+    paths : dict
+    """
+    pbf_filename = get_region_filename(region, subregion)
+
+    pbf_file = os.path.join(paths['out_path'], pbf_filename)
+    md5_file = f'{pbf_file}.md5'
+    logging.info(f'Done with {pbf_file}, removing.')
+    os.remove(pbf_file)
+    logging.info(f'Done with {md5_file}, removing.')
+    os.remove(md5_file)
+
 
 def get_osm2pgsql_command(region, subregion, ram, layerset, paths):
     """Returns recommended osm2pgsql command.