Merge pull request #190 from rustprooflabs/dev

Prep for 0.4.0 rc1

Author: rustprooflabs

Dockerfile

@@ -10,13 +10,18 @@ RUN apt-get update \
         libboost-filesystem-dev libexpat1-dev zlib1g-dev \
         libbz2-dev libpq-dev libproj-dev lua5.2 liblua5.2-dev \
         python3 python3-distutils python3-psycopg2 \
-        curl \
+        postgresql-server-dev-14 \
+        curl luarocks \
     && rm -rf /var/lib/apt/lists/*
 
 RUN curl -o /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py \
     && python3 /tmp/get-pip.py \
     && rm /tmp/get-pip.py
 
+RUN luarocks install inifile
+RUN luarocks install luasql-postgres PGSQL_INCDIR=/usr/include/postgresql/
+
+
 WORKDIR /tmp
 RUN git clone git://github.com/openstreetmap/osm2pgsql.git \
     && mkdir osm2pgsql/build \
@@ -27,15 +32,15 @@ RUN git clone git://github.com/openstreetmap/osm2pgsql.git \
     && apt remove -y \
         make cmake g++ \
         libexpat1-dev zlib1g-dev \
-        libbz2-dev libpq-dev libproj-dev \
+        libbz2-dev libproj-dev \
         curl \
     && apt autoremove -y \
     && cd /tmp && rm -r /tmp/osm2pgsql
 
+
 COPY ./sqitch.conf /etc/sqitch/sqitch.conf
 
 WORKDIR /app
 COPY . ./
 
-# --pre added to switch to psycopg3 during beta, remove after inital official release
-RUN pip install --pre -r requirements.txt
+RUN pip install -r requirements.txt

Makefile

@@ -49,7 +49,7 @@ build-run-docker: ## Builds and runs PgOSM Flex with D.C. test file
 		-e POSTGRES_USER=postgres \
 		-u $(CURRENT_UID):$(CURRENT_GID) \
 		pgosm python3 docker/pgosm_flex.py  \
-		--layerset=run-all \
+		--layerset=default \
 		--ram=1 \
 		--region=north-america/us \
 		--subregion=district-of-columbia \

README.md

@@ -56,6 +56,8 @@ mkdir ~/pgosm-data
 ```
 
 Set environment variables for the temporary Postgres connection in Docker.
+These are required for the Docker container to run.
+
 
 ```bash
 export POSTGRES_USER=postgres
@@ -74,11 +76,10 @@ docker run --name pgosm -d --rm \
 ```
 
 Run the processing for the Washington D.C.  The `docker/pgosm_flex.py` script
-requires three (3) parameters, typical use will use four (4) to include
+requires two (2) parameters, typical use will use three (3) to include
 the `--subregion`.
 
 
-* PgOSM-Flex layer set (`run-all`)
 * Total RAM for osm2pgsql, Postgres and OS (`8`)
 * Region (`north-america/us`)
 * Sub-region (`district-of-columbia`) (Optional)
@@ -87,10 +88,7 @@ the `--subregion`.
 
 ```bash
 docker exec -it \
-    -e POSTGRES_PASSWORD=$POSTGRES_PASSWORD \
-    -e POSTGRES_USER=$POSTGRES_USER \
     pgosm python3 docker/pgosm_flex.py \
-    --layerset=run-all \
     --ram=8 \
     --region=north-america/us \
     --subregion=district-of-columbia
@@ -413,9 +411,12 @@ Current `JSONB` columns:
 
 ## Additional resources
 
+See the listing of known
+[projects using PgOSM Flex](docs/PROJECTS.md).
+
+
 Blog posts covering various details and background information.
 
 * [Better OpenStreetMap places in PostGIS](https://blog.rustprooflabs.com/2021/01/pgosm-flex-improved-openstreetmap-places-postgis)
 * [Improved OpenStreetMap data structure in PostGIS](https://blog.rustprooflabs.com/2021/01/postgis-openstreetmap-flex-structure) 
 * [Hands on with osm2pgsql's new Flex output](https://blog.rustprooflabs.com/2020/12/osm2gpsql-flex-output-to-postgis).
-

docker/db.py

@@ -302,18 +302,20 @@ def get_db_conn(db_name):
     return conn
 
 
-def pgosm_after_import(layerset, paths):
-    """Runs post-processing SQL via psql.
+def pgosm_after_import(paths):
+    """Runs post-processing SQL via Lua script.
+
+    Layerset logic is established via environment variable, must happen
+    before this step.
 
     Parameters
     ---------------------
-    layerset : str
-
     paths : dict
     """
     LOGGER.info('Running post-processing...')
-    conn_string = connection_string(db_name='pgosm')
-    cmds = ['psql', '-d', conn_string, '-f', f'{layerset}.sql']
+
+    cmds = ['lua', 'run-sql.lua']
+
     output = subprocess.run(cmds,
                             text=True,
                             capture_output=True,
@@ -336,10 +338,18 @@ def pgosm_nested_admin_polygons(paths):
     LOGGER.info('Building nested polygons... (this can take a while)')
     output = subprocess.run(cmds,
                             text=True,
-                            capture_output=True,
                             cwd=paths['flex_path'],
-                            check=True)
-    LOGGER.info(f'Nested polygon output: \n {output.stderr}')
+                            check=False,
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
+    LOGGER.info(f'Nested polygon output: \n {output.stdout}')
+
+
+    if output.returncode != 0:
+        err_msg = f'Failed to build nested polygons. Return code: {output.returncode}'
+        LOGGER.error(err_msg)
+        sys.exit(f'{err_msg} - Check the log output for details.')
+
 
 
 def rename_schema(schema_name):

docker/osm2pgsql_recommendation.py

@@ -10,7 +10,7 @@
 LOGGER = logging.getLogger('pgosm-flex')
 
 
-def osm2pgsql_recommendation(ram, layerset, pbf_filename, out_path):
+def osm2pgsql_recommendation(ram, pbf_filename, out_path):
     """Returns recommended osm2pgsql command.
 
     Recommendation from API at https://osm2pgsql-tuner.com
@@ -29,7 +29,8 @@ def osm2pgsql_recommendation(ram, layerset, pbf_filename, out_path):
     osm2pgsql_cmd : str
     """
     system_ram_gb = ram
-    pgosm_layer_set = layerset
+    # The layerset is now set via env var.  This is used to set filename for osm2pgsql command
+    pgosm_layer_set = 'run'
 
     pbf_file = os.path.join(out_path, pbf_filename)
     osm_pbf_gb = os.path.getsize(pbf_file) / 1024 / 1024 / 1024