Update to handle error when data is too stale to update via replication. Cleanup bad var name

Author: rustprooflabs

docker/geofabrik.py

@@ -88,7 +88,7 @@ def set_date_from_metadata(pbf_file: str):
     returncode = helpers.run_command_via_subprocess(cmd=osmium_cmd.split(),
                                                     cwd=None,
                                                     output_lines=output,
-                                                    print=False)
+                                                    print_to_log=False)
     if returncode != 0:
         logger.error(f'osmium fileinfo failed.  Output: {output}')
 

docker/helpers.py

@@ -27,8 +27,11 @@ def get_today() -> str:
     return today
 
 
-def run_command_via_subprocess(cmd: list, cwd: str, output_lines: list=[],
-                               print: bool=False) -> int:
+def run_command_via_subprocess(cmd: list,
+                               cwd: str,
+                               output_lines: list=[],
+                               print_to_log: bool=False
+                               ) -> int:
     """Wraps around subprocess.Popen() to run commands outside of Python. Prints
     output as it goes, returns the status code from the command.
 
@@ -40,7 +43,7 @@ def run_command_via_subprocess(cmd: list, cwd: str, output_lines: list=[],
         Set the working directory, or to None.
     output_lines : list
         Pass in a list to return the output details.
-    print : bool
+    print_to_log : bool
         Default False.  Set to true to also print to logger
 
     Returns
@@ -60,12 +63,23 @@ def run_command_via_subprocess(cmd: list, cwd: str, output_lines: list=[],
             if output:
                 ln = output.strip().decode('utf-8')
                 output_lines.append(ln)
-                if print:
+                if print_to_log:
                     logger.info(ln)
+
+                # Detects issue reported in https://github.com/rustprooflabs/pgosm-flex/issues/391
+                # Status code is incorrectly returned is 0, cannot detect
+                # problem using that method so forcing failure with custom
+                # status code.
+                if 'Error during diff download. Bailing out.' in ln:
+                    logger.error('Data in database is too far behind replication service.')
+                    return 999
+
             else:
                 # Only sleep when there wasn't output
                 sleep(1)
+
         status = process.poll()
+
     return status
 
 

docker/pgosm_flex.py

@@ -273,7 +273,7 @@ def run_replication_update(skip_nested, flex_path):
     update_cmd = update_cmd.replace('-d $PGOSM_CONN', f'-d {conn_string}')
     returncode = helpers.run_command_via_subprocess(cmd=update_cmd.split(),
                                                     cwd=flex_path,
-                                                    print=True)
+                                                    print_to_log=True)
 
     if returncode != 0:
         err_msg = f'Failure. Return code: {returncode}'
@@ -424,7 +424,7 @@ def run_osm2pgsql(osm2pgsql_command, flex_path, debug):
 
     returncode = helpers.run_command_via_subprocess(cmd=osm2pgsql_command.split(),
                                                     cwd=flex_path,
-                                                    print=True)
+                                                    print_to_log=True)
 
     if returncode != 0:
         err_msg = f'Failed to run osm2pgsql. Return code: {returncode}'
@@ -586,7 +586,7 @@ def check_replication_exists():
     return True
 
 
-def run_osm2pgsql_replication_init(pbf_path, pbf_filename):
+def run_osm2pgsql_replication_init(pbf_path: str, pbf_filename: str):
     """Runs osm2pgsql-replication init to support replication mode.
 
     Parameters
@@ -604,7 +604,7 @@ def run_osm2pgsql_replication_init(pbf_path, pbf_filename):
 
     returncode = helpers.run_command_via_subprocess(cmd=init_cmd.split(),
                                                     cwd=None,
-                                                    print=True)
+                                                    print_to_log=True)
 
     if returncode != 0:
         err_msg = f'Failed to run osm2pgsql-replication. Return code: {returncode}'