Fix aeroway in infrastructure. Add DB functions to help with append mode

README.md

@@ -32,7 +32,7 @@ Minimum versions supported:
 
 * Postgres 12
 * PostGIS 3.0
-* osm2pgsql 1.5.0
+* osm2pgsql 1.6.0
 
 ## Minimum Hardware
 
@@ -162,7 +162,6 @@ psql -h localhost -p 5433 -d pgosm -U postgres -c "SELECT COUNT(*) FROM osm.road
 
 
 
-
 See [more in docs/DOCKER-RUN.md](docs/DOCKER-RUN.md) about other ways to customize
 how PgOSM Flex runs.
 

docs/APPEND-MODE.md

@@ -0,0 +1,84 @@
+# Running osm2pgsql in append mode
+
+----
+
+> WARNING: The process described here is experimental and details will probably change!
+
+----
+
+## Using manual steps
+
+This page documents differences from [MANUAL-STEPS-RUN.md](MANUAL-STEPS-RUN.md)
+
+
+Setup - Need to use Python venv, osmium is a requirement.  Something like...
+
+```bash
+python -m venv venv && source venv/bin/activate
+cd ~/git/pgosm-flex && pip install -r requirements.txt
+```
+
+
+Run osm2pgsql. Must use `--slim` mode without drop.
+
+
+```bash
+cd pgosm-flex/flex-config
+
+osm2pgsql --output=flex --style=./run.lua \
+    --slim \
+    -d $PGOSM_CONN \
+    ~/pgosm-data/district-of-columbia-latest.osm.pbf
+```
+
+Run the normal post-processing as you normally would.
+
+
+`osm2pgsql-replication` is bundled with osm2pgsql install.
+
+https://osm2pgsql.org/doc/manual.html#keeping-the-database-up-to-date-with-osm2pgsql-replication
+
+
+Initialize replication.
+
+
+```bash
+osm2pgsql-replication init -d $PGOSM_CONN \
+    --osm-file ~/pgosm-data/district-of-columbia-latest.osm.pbf
+```
+
+
+
+Refresh the data.  First clear out data that might violate foreign keys. Packaged
+in convenient procedure.
+
+
+```sql
+CALL osm.append_data_start();
+```
+
+Update the data.
+
+
+```bash
+osm2pgsql-replication update -d $PGOSM_CONN \
+    -- \
+    --output=flex --style=./run.lua \
+    --slim \
+    -d $PGOSM_CONN
+```
+
+Refresh Mat views, rebuilds nested place polygon data.
+
+
+```sql
+CALL osm.append_data_finish();
+```
+
+
+Skip nested:
+
+```sql
+CALL osm.append_data_finish(skip_nested := True);
+```
+

docs/MANUAL-STEPS-RUN.md

@@ -214,6 +214,7 @@ The post-processing SQL scripts create a procedure to calculate the nested place
 
 
 ```bash
+psql -d $PGOSM_CONN -c "CALL osm.populate_place_polygon_nested();"
 psql -d $PGOSM_CONN -c "CALL osm.build_nested_admin_polygons();"
 ```
 

flex-config/sql/pgosm-meta.sql

@@ -11,3 +11,47 @@ COMMENT ON COLUMN osm.pgosm_flex.osm2pgsql_version IS 'Version of osm2pgsql used
 COMMENT ON COLUMN osm.pgosm_flex.region IS 'Region specified at run time via env var PGOSM_REGION.';
 COMMENT ON COLUMN osm.pgosm_flex.language IS 'Preferred language specified at run time via env var PGOSM_LANGUAGE.  Empty string when not defined.';
 COMMENT ON COLUMN osm.pgosm_flex.osm2pgsql_mode IS 'Indicates which osm2pgsql mode was used, create or append.';
+
+
+
+-- Helper Procedures for allowing updates via osm2pgsql-replication or similar
+
+
+CREATE OR REPLACE PROCEDURE osm.append_data_start()
+ LANGUAGE plpgsql
+ AS $$
+
+ BEGIN
+
+    RAISE NOTICE 'Truncating table osm.place_polygon_nested;';
+    TRUNCATE TABLE osm.place_polygon_nested;
+
+END $$;
+
+
+
+
+CREATE OR REPLACE PROCEDURE osm.append_data_finish(skip_nested BOOLEAN = False)
+ LANGUAGE plpgsql
+ AS $$
+ BEGIN
+
+    REFRESH MATERIALIZED VIEW osm.vplace_polygon;
+    REFRESH MATERIALIZED VIEW osm.vplace_polygon_subdivide;
+    REFRESH MATERIALIZED VIEW osm.vpoi_all;
+
+    IF $1 = False THEN
+        RAISE NOTICE 'Populating nested place table';
+        CALL osm.populate_place_polygon_nested();
+        RAISE NOTICE 'Calculating nesting of place polygons';
+        CALL osm.build_nested_admin_polygons();
+
+    END IF;
+
+
+END $$;
+
+
+COMMENT ON PROCEDURE osm.append_data_start() IS 'Prepares PgOSM Flex database for running osm2pgsql in append mode.  Removes records from place_polygon_nested if they existed.';
+COMMENT ON PROCEDURE osm.append_data_finish() IS 'Finalizes PgOSM Flex after osm2pgsql-replication.  Refreshes materialized view and (optionally) processes the place_polygon_nested data.';
+

flex-config/sql/place.sql

@@ -146,17 +146,31 @@ COMMENT ON COLUMN osm.place_polygon_nested.osm_type IS 'Values from place if a p
 COMMENT ON COLUMN osm.place_polygon_nested.geom IS 'Geometry loaded by osm2pgsql.';
 
 
-INSERT INTO osm.place_polygon_nested (osm_id, name, osm_type, admin_level, geom)
-SELECT p.osm_id, p.name, p.osm_type,
-        COALESCE(p.admin_level::INT, 99) AS admin_level,
-        geom
-    FROM osm.vplace_polygon p
-    WHERE (p.boundary = 'administrative'
-            OR p.osm_type IN   ('neighborhood', 'city', 'suburb', 'town', 'admin_level', 'locality')
-       )
-        AND p.name IS NOT NULL
-;
+CREATE OR REPLACE PROCEDURE osm.populate_place_polygon_nested()
+LANGUAGE sql
+AS $$
+
+
+    INSERT INTO osm.place_polygon_nested (osm_id, name, osm_type, admin_level, geom)
+    SELECT p.osm_id, p.name, p.osm_type,
+            COALESCE(p.admin_level::INT, 99) AS admin_level,
+            geom
+        FROM osm.vplace_polygon p
+        WHERE (p.boundary = 'administrative'
+                OR p.osm_type IN   ('neighborhood', 'city', 'suburb', 'town', 'admin_level', 'locality')
+           )
+            AND p.name IS NOT NULL
+            AND NOT EXISTS (
+                SELECT osm_id
+                    FROM osm.place_polygon_nested n
+                    WHERE n.osm_id = p.osm_id
+                )
+    ;
+
+$$;
+
 
+CALL osm.populate_place_polygon_nested();
 
 
 CREATE OR REPLACE PROCEDURE osm.build_nested_admin_polygons(

flex-config/style/infrastructure.lua

@@ -101,6 +101,9 @@ local function get_osm_type_subtype(object)
     elseif object.tags.utility then
         osm_type_table['osm_type'] = 'utility'
         osm_type_table['osm_subtype'] = nil
+    elseif object.tags.aeroway then
+        osm_type_table['osm_type'] = 'aeroway'
+        osm_type_table['osm_subtype'] = object.tags.aeroway
     else
         osm_type_table['osm_type'] = 'unknown'
         osm_type_table['osm_subtype'] = nil
@@ -210,4 +213,4 @@ else
         nested(object)
         infrastructure_process_way(object_copy)
     end
-end
+end

requirements.txt

@@ -1,6 +1,7 @@
 click>=8.0.1
 coverage>=6.1.2
 osm2pgsql-tuner==0.0.4
+osmium==3.2.0
 psycopg>=3.0.3
 psycopg-binary>=3.0.3
 sh>=1.14.2

tests/expected/infrastructure_line_osm_type_subtype_count.out

@@ -1,3 +1,5 @@
+aeroway|runway|2
+aeroway|taxiway|1
 power|cable|6
 power|line|11
 power|minor_line|2

tests/expected/infrastructure_point_osm_type_subtype_count.out

@@ -1,3 +1,5 @@
+aeroway|helipad|9
+aeroway|navigationaid|1
 communications_tower||1
 emergency_access||1
 emergency_phone||16

tests/expected/infrastructure_polygon_osm_type_subtype_count.out

@@ -1,3 +1,9 @@
+aeroway|aerodrome|2
+aeroway|apron|1
+aeroway|hangar|2
+aeroway|helipad|12
+aeroway|heliport|2
+aeroway|runway|3
 power|generator|9
 power|plant|1
 power|substation|15