From 77b057102954adc00a1b9f9f511c34dcff9ade91 Mon Sep 17 00:00:00 2001 From: Ryan Lambert Date: Wed, 22 Nov 2023 15:04:01 -0700 Subject: [PATCH 1/3] Check the current PgOSM Flex version from git against the latest import tracked in pgosm_flex table. --- .gitignore | 1 + docker/db.py | 3 ++- docker/helpers.py | 16 +++++++++++++--- docker/import_mode.py | 16 ++++++++++++++++ requirements.txt | 1 + 5 files changed, 33 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 8a2616ca..4d7d0d86 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ output/ **/__pycache__ pgosm-data/* docs/book/* +.vscode/* \ No newline at end of file diff --git a/docker/db.py b/docker/db.py index dced8a3d..97d8d00c 100644 --- a/docker/db.py +++ b/docker/db.py @@ -656,7 +656,8 @@ def get_prior_import(schema_name: str) -> dict: SELECT id, osm_date, region, layerset, import_status, import_mode ->> 'replication' AS replication, import_mode ->> 'update' AS use_update, - import_mode + import_mode, + split_part(pgosm_flex_version, '-', 1) AS pgosm_flex_version_no_hash FROM {schema_name}.pgosm_flex ORDER BY imported DESC LIMIT 1 diff --git a/docker/helpers.py b/docker/helpers.py index 85dd6c2b..409d41e4 100644 --- a/docker/helpers.py +++ b/docker/helpers.py @@ -166,11 +166,16 @@ def get_region_combined(region: str, subregion: str) -> str: return pgosm_region -def get_git_info() -> str: +def get_git_info(tag_only: bool=False) -> str: """Provides git info in the form of the latest tag and most recent short sha Sends info to logger and returns string. + Parameters + ---------------------- + tag_only : bool + When true, omits the short sha portion, only returning the tag. + Returns ---------------------- git_info : str @@ -181,12 +186,17 @@ def get_git_info() -> str: sha = repo.head.object.hexsha short_sha = repo.git.rev_parse(sha, short=True) latest_tag = repo.git.describe('--abbrev=0', tags=True) - git_info = f'{latest_tag}-{short_sha}' except ValueError: git_info = 'Git info unavailable' logger.error('Unable to get git information.') - logger.info(f'PgOSM Flex version: {git_info}') + if tag_only: + git_info = latest_tag + else: + git_info = f'{latest_tag}-{short_sha}' + # Logging only this full version, not the tag_only run + logger.info(f'PgOSM Flex version: {git_info}') + return git_info diff --git a/docker/import_mode.py b/docker/import_mode.py index 9307736a..344f4385 100644 --- a/docker/import_mode.py +++ b/docker/import_mode.py @@ -2,6 +2,9 @@ """ import logging import json +from packaging.version import parse as parse_version + +import helpers class ImportMode(): @@ -83,6 +86,19 @@ def okay_to_run(self, prior_import: dict) -> bool: prior_replication = prior_import['replication'] + # Check git version against latest. + # If current version is lower than prior version from latest import, stop. + prior_import_version = prior_import['pgosm_flex_version_no_hash'] + git_tag = helpers.get_git_info(tag_only=True) + if parse_version(git_tag) < parse_version(prior_import_version): + msg = f'PgOSM Flex version ({git_tag}) is lower than latest import' + msg += f' tracked in the pgosm_flex table ({prior_import_version}).' + msg += f' Use PgOSM Flex version {git_tag} or newer' + self.logger.error(msg) + return False + else: + self.logger.info(f'Prior import used PgOSM Flex: {prior_import_version}') + if self.replication: if not prior_replication: self.logger.error('Running w/ replication but prior import did not. Requires --force to proceed.') diff --git a/requirements.txt b/requirements.txt index dfb9f502..37441f80 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ coverage>=6.4.1 GitPython>=3.1.31 osm2pgsql-tuner==0.0.6 osmium>=3.4.1 +packaging>=23.0 psycopg>=3.1 psycopg-binary>=3.1 sh>=1.14.2 From b64e63f1bb5e96ab268b43728c28bda17488ac6e Mon Sep 17 00:00:00 2001 From: Ryan Lambert Date: Wed, 22 Nov 2023 15:12:15 -0700 Subject: [PATCH 2/3] Fix version included in suggestion --- docker/import_mode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/import_mode.py b/docker/import_mode.py index 344f4385..8dbfb5a7 100644 --- a/docker/import_mode.py +++ b/docker/import_mode.py @@ -93,7 +93,7 @@ def okay_to_run(self, prior_import: dict) -> bool: if parse_version(git_tag) < parse_version(prior_import_version): msg = f'PgOSM Flex version ({git_tag}) is lower than latest import' msg += f' tracked in the pgosm_flex table ({prior_import_version}).' - msg += f' Use PgOSM Flex version {git_tag} or newer' + msg += f' Use PgOSM Flex version {prior_import_version} or newer' self.logger.error(msg) return False else: From b4a709beab59371c4a3447e332d36da9d84c9fa6 Mon Sep 17 00:00:00 2001 From: Ryan Lambert Date: Wed, 22 Nov 2023 15:20:33 -0700 Subject: [PATCH 3/3] Fix unit tests, handle missing git version details when running make --- docker/helpers.py | 10 +++++++++- docker/import_mode.py | 7 ++++++- docker/tests/test_import_mode.py | 12 +++++++++--- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/docker/helpers.py b/docker/helpers.py index 409d41e4..a8e6dcfd 100644 --- a/docker/helpers.py +++ b/docker/helpers.py @@ -181,7 +181,14 @@ def get_git_info(tag_only: bool=False) -> str: git_info : str """ logger = logging.getLogger('pgosm-flex') - repo = git.Repo() + + try: + repo = git.Repo() + except git.exc.InvalidGitRepositoryError: + # This error happens when running via make for some reason... + # This appears to fix it. + repo = git.Repo('../') + try: sha = repo.head.object.hexsha short_sha = repo.git.rev_parse(sha, short=True) @@ -189,6 +196,7 @@ def get_git_info(tag_only: bool=False) -> str: except ValueError: git_info = 'Git info unavailable' logger.error('Unable to get git information.') + return '-- (version unknown) --' if tag_only: git_info = latest_tag diff --git a/docker/import_mode.py b/docker/import_mode.py index 8dbfb5a7..d56662c9 100644 --- a/docker/import_mode.py +++ b/docker/import_mode.py @@ -90,7 +90,12 @@ def okay_to_run(self, prior_import: dict) -> bool: # If current version is lower than prior version from latest import, stop. prior_import_version = prior_import['pgosm_flex_version_no_hash'] git_tag = helpers.get_git_info(tag_only=True) - if parse_version(git_tag) < parse_version(prior_import_version): + + if git_tag == '-- (version unknown) --': + msg = 'Unable to detect PgOSM Flex version from Git.' + msg += ' Not enforcing version check against prior version.' + self.logger.warning(msg) + elif parse_version(git_tag) < parse_version(prior_import_version): msg = f'PgOSM Flex version ({git_tag}) is lower than latest import' msg += f' tracked in the pgosm_flex table ({prior_import_version}).' msg += f' Use PgOSM Flex version {prior_import_version} or newer' diff --git a/docker/tests/test_import_mode.py b/docker/tests/test_import_mode.py index d908d4b6..13fc3a40 100644 --- a/docker/tests/test_import_mode.py +++ b/docker/tests/test_import_mode.py @@ -146,7 +146,9 @@ def test_import_mode_okay_to_run_returns_false_when_prior_record_not_replication This should return False to avoid overwriting data. """ replication = True - prior_import = {'replication': False} + prior_import = {'replication': False, + 'pgosm_flex_version_no_hash': '99.99.99' + } replication_update = False update = None force = False @@ -170,7 +172,9 @@ def test_import_mode_okay_to_run_returns_true_when_replication_prior_record_repl This should return True to allow replication to updated """ replication = True - prior_import = {'replication': True} + prior_import = {'replication': True, + 'pgosm_flex_version_no_hash': '99.99.99' + } replication_update = False update = None force = False @@ -193,7 +197,9 @@ def test_import_mode_okay_to_run_returns_false_when_prior_import(self): This should return False to protect the data. """ replication = False - prior_import = {'replication': False} + prior_import = {'replication': False, + 'pgosm_flex_version_no_hash': '99.99.99' + } replication_update = False update = None force = False