From d7be15456c35a8cbeff41ada45213ec93e5d0ed7 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sun, 26 Apr 2020 17:11:43 -0700 Subject: [PATCH 01/54] Switch to the new osm2pgsql flex backend The flex backend allows the style transformation to configure much more, including features that were not possible with the pgsql backend. --- INSTALL.md | 4 +- openstreetmap-carto.lua | 397 +++++++++++++++++++++++++++----------- scripts/lua/test.lua | 409 ++++++++++++++++++++++++++-------------- 3 files changed, 550 insertions(+), 260 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 83d68b4376..ebcae736fb 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -3,7 +3,7 @@ This document describes how to manually configure your system for running OpenStreetMap Carto. If you prefer quick, platform independent setup for a development environment, without the need to install and configure tools by hand, follow a Docker installation guide in [DOCKER.md](https://github.com/gravitystorm/openstreetmap-carto/blob/master/DOCKER.md). ## OpenStreetMap data -You need OpenStreetMap data loaded into a PostGIS database (see below for [dependencies](#dependencies)). These stylesheets expect a database generated with osm2pgsql using the pgsql backend (table names of `planet_osm_point`, etc), the default database name (`gis`), and the [lua transforms](https://github.com/openstreetmap/osm2pgsql/blob/master/docs/lua.md) documented in the instructions below. +You need OpenStreetMap data loaded into a PostGIS database (see below for [dependencies](#dependencies)). These stylesheets expect a database generated with osm2pgsql using the flex backend and the default database name (`gis`). This requires osm2pgsql built from source. Start by creating a database @@ -21,7 +21,7 @@ psql -d gis -c 'CREATE EXTENSION postgis; CREATE EXTENSION hstore;' then grab some OSM data. It's probably easiest to grab an PBF of OSM data from [Geofabrik](https://download.geofabrik.de/). Once you've done that, import with osm2pgsql: ``` -osm2pgsql -G --hstore --style openstreetmap-carto.style --tag-transform-script openstreetmap-carto.lua -d gis ~/path/to/data.osm.pbf +osm2pgsql --output flex --style openstreetmap-carto.lua -d gis ~/path/to/data.osm.pbf ``` You can find a more detailed guide to setting up a database and loading data with osm2pgsql at [switch2osm.org](https://switch2osm.org/manually-building-a-tile-server-16-04-2-lts/). diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 8dbca62a64..93a2f7eee4 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -1,6 +1,167 @@ -- For documentation of Lua tag transformations, see: -- https://github.com/openstreetmap/osm2pgsql/blob/master/docs/lua.md +local tables = {} + +-- A list of columns per table, replacing the osm2pgsql .style file +-- These need to be ordered, so that means a list +local point_columns = { + 'access', + 'addr:housename', + 'addr:housenumber', + 'admin_level', + 'aerialway', + 'aeroway', + 'amenity', + 'barrier', + 'boundary', + 'building', + 'highway', + 'historic', + 'junction', + 'landuse', + 'leisure', + 'lock', + 'man_made', + 'military', + 'name', + 'natural', + 'oneway', + 'place', + 'power', + 'railway', + 'ref', + 'religion', + 'shop', + 'tourism', + 'water', + 'waterway' } + +local line_columns = { + 'access', + 'addr:housename', + 'addr:housenumber', + 'addr:interpolation', + 'admin_level', + 'aerialway', + 'aeroway', + 'amenity', + 'barrier', + 'bicycle', + 'bridge', + 'boundary', + 'building', + 'construction', + 'covered', + 'foot', + 'highway', + 'historic', + 'horse', + 'junction', + 'landuse', + 'leisure', + 'lock', + 'man_made', + 'military', + 'name', + 'natural', + 'oneway', + 'place', + 'power', + 'railway', + 'ref', + 'religion', + 'route', + 'service', + 'shop', + 'surface', + 'tourism', + 'tracktype', + 'tunnel', + 'water', + 'waterway'} + +-- These are the same for now +local polygon_columns = line_columns +local road_columns = line_columns + +-- Process the columns above +local point_col_def = { + { column = 'way', type = 'point' }, + { column = 'tags', type = 'hstore' }, + { column = 'layer', type = 'int4' } +} + +local point_columns_map = {} +for _, key in ipairs(point_columns) do + table.insert(point_col_def, {column = key, type = "text"}) + point_columns_map[key] = true +end + +tables.point = osm2pgsql.define_table{ + name = 'planet_osm_point', + ids = { type = 'node', id_column = 'osm_id' }, + columns = point_col_def +} + +local line_col_def = { + { column = 'way', type = 'linestring' }, + { column = 'tags', type = 'hstore' }, + { column = 'layer', type = 'int4' }, + { column = 'z_order', type = 'int4' } +} + +local line_columns_map = {} +for _, key in ipairs(line_columns) do + table.insert(line_col_def, {column = key, type = "text"}) + line_columns_map[key] = true +end + +tables.line = osm2pgsql.define_table{ + name = 'planet_osm_line', + ids = { type = 'way', id_column = 'osm_id' }, + columns = line_col_def +} + +local roads_col_def = { + { column = 'way', type = 'linestring' }, + { column = 'tags', type = 'hstore' }, + { column = 'layer', type = 'int4' }, + { column = 'z_order', type = 'int4' } +} + +local roads_columns_map = {} +for _, key in ipairs(road_columns) do + table.insert(roads_col_def, {column = key, type = "text"}) + roads_columns_map[key] = true +end + +tables.roads = osm2pgsql.define_table{ + name = 'planet_osm_roads', + ids = { type = 'way', id_column = 'osm_id' }, + columns = roads_col_def +} + +local polygon_col_def = { + { column = 'way', type = 'geometry' }, + { column = 'tags', type = 'hstore' }, + { column = 'layer', type = 'int4' }, + { column = 'z_order', type = 'int4' }, + { column = 'way_area', type = 'area' } +} + +local polygon_columns_map = {} +for _, key in ipairs(polygon_columns) do + table.insert(polygon_col_def, {column = key, type = "text"}) + polygon_columns_map[key] = true +end + +tables.polygon = osm2pgsql.define_table{ + name = 'planet_osm_polygon', + ids = { type = 'way', id_column = 'osm_id' }, + columns = polygon_col_def +} + -- Objects with any of the following keys will be treated as polygon local polygon_keys = { 'abandoned:aeroway', @@ -265,34 +426,73 @@ end --- Gets the roads table status for a set of tags -- @param tags OSM tags --- @return 1 if it belongs in the roads table, 0 otherwise +-- @return true if it belongs in the roads table, false otherwise function roads(tags) for k, v in pairs(tags) do if roads_info[k] and roads_info[k][v] and roads_info[k][v].roads then if not (k ~= 'railway' or tags.service) then - return 1 + return true elseif not excluded_railway_service[tags.service] then - return 1 + return true end end end - return 0 + return false end ---- Generic filtering of OSM tags --- @param tags Raw OSM tags --- @return Filtered OSM tags -function filter_tags_generic(tags) +--- Check if an object with given tags should be treated as polygon +-- @param tags OSM tags +-- @return 1 if area, 0 if linear +function isarea (tags) + -- Treat objects tagged as area=yes polygon, other area as no + if tags["area"] then + return tags["area"] == "yes" and true or false + end + + -- Search through object's tags + for k, v in pairs(tags) do + -- Check if it has a polygon key and not a linestring override, or a polygon k=v + for _, ptag in ipairs(polygon_keys) do + if k == ptag and v ~= "no" and not (linestring_values[k] and linestring_values[k][v]) then + return true + end + end + + if (polygon_values[k] and polygon_values[k][v]) then + return true + end + end + return false +end + +function is_in (needle, haystack) + for index, value in ipairs (haystack) do + if value == needle then + return true + end + end + return false +end + +--- Normalizes layer tags +-- @param v The layer tag value +-- @return An integer for the layer tag +function layer (v) + return v and string.find(v, "^-?%d+$") and tonumber(v) < 100 and tonumber(v) > -100 and v or nil +end + +--- Clean tags of deleted tags +-- @return True if no tags are left after cleaning +function clean_tags(tags) -- Short-circuit for untagged objects if next(tags) == nil then - return 1, {} + return true end -- Delete tags listed in delete_tags for _, d in ipairs(delete_tags) do tags[d] = nil end - -- By using a second loop for wildcards we avoid checking already deleted tags for tag, _ in pairs (tags) do for _, d in ipairs(delete_prefixes) do @@ -303,134 +503,103 @@ function filter_tags_generic(tags) end end - -- Filter out objects that have no tags after deleting - if next(tags) == nil then - return 1, {} - end - - -- Convert layer to an integer - tags['layer'] = layer(tags['layer']) - return 0, tags + return next(tags) == nil end --- Filtering on nodes -function filter_tags_node (keyvalues, numberofkeys) - return filter_tags_generic(keyvalues) +--- Splits a tag into tags and hstore tags +-- @return columns, hstore tags +function split_tags(tags, tag_map) + local cols = {tags = {}} + for key, value in pairs(tags) do + if tag_map[key] then + cols[key] = value + else + cols.tags[key] = value + end + end + return cols end --- Filtering on relations -function filter_basic_tags_rel (keyvalues, numberofkeys) - -- Filter out objects that are filtered out by filter_tags_generic - local filter, keyvalues = filter_tags_generic(keyvalues) - if filter == 1 then - return 1, keyvalues - end +function add_point(tags) + local cols = split_tags(tags, point_columns_map) + cols['layer'] = layer(tags['layer']) + tables.point:add_row(cols) +end - -- Filter out all relations except route, multipolygon and boundary relations - if ((keyvalues["type"] ~= "route") and (keyvalues["type"] ~= "multipolygon") and (keyvalues["type"] ~= "boundary")) then - return 1, keyvalues - end +function add_line(tags) + local cols = split_tags(tags, line_columns_map) + cols['layer'] = layer(tags['layer']) + cols['z_order'] = z_order(tags) + cols.way = { create = 'line', split_at = 100000 } + tables.line:add_row(cols) +end - return 0, keyvalues +function add_roads(tags) + local cols = split_tags(tags, roads_columns_map) + cols['layer'] = layer(tags['layer']) + cols['z_order'] = z_order(tags) + cols.way = { create = 'line', split_at = 100000 } + tables.roads:add_row(cols) end --- Filtering on ways -function filter_tags_way (keyvalues, numberofkeys) - local filter = 0 -- Will object be filtered out? - local polygon = 0 -- Will object be treated as polygon? +function add_polygon(tags) + local cols = split_tags(tags, polygon_columns_map) + cols['layer'] = layer(tags['layer']) + cols['z_order'] = z_order(tags) + cols.way = { create = 'area', multi = true } + tables.polygon:add_row(cols) +end - -- Filter out objects that are filtered out by filter_tags_generic - filter, keyvalues = filter_tags_generic(keyvalues) - if filter == 1 then - return filter, keyvalues, polygon, roads +function osm2pgsql.process_node(object) + if clean_tags(object.tags) then + return end - polygon = isarea(keyvalues) - - -- Add z_order column - keyvalues["z_order"] = z_order(keyvalues) - - return filter, keyvalues, polygon, roads(keyvalues) + add_point(object.tags) end ---- Handling for relation members and multipolygon generation --- @param keyvalues OSM tags, after processing by relation transform --- @param keyvaluemembers OSM tags of relation members, after processing by way transform --- @param roles OSM roles of relation members --- @param membercount number of members --- @return filter, cols, member_superseded, boundary, polygon, roads -function filter_tags_relation_member (keyvalues, keyvaluemembers, roles, membercount) - local members_superseded = {} - - -- Start by assuming that this not an old-style MP - for i = 1, membercount do - members_superseded[i] = 0 +function osm2pgsql.process_way(object) + if clean_tags(object.tags) then + return end - local type = keyvalues["type"] - - -- Remove type key - keyvalues["type"] = nil + local area_tags = isarea(object.tags) + if object.is_closed and area_tags then + add_polygon(object.tags) + else + add_line(object.tags) - -- Filter out relations with just a type tag or no tags - if next(keyvalues) == nil then - return 1, keyvalues, members_superseded, 0, 0, 0 - end - - if type == "boundary" or (type == "multipolygon" and keyvalues["boundary"]) then - keyvalues.z_order = z_order(keyvalues) - return 0, keyvalues, members_superseded, 1, 0, roads(keyvalues) - -- For multipolygons... - elseif (type == "multipolygon") then - -- Multipolygons by definition are polygons, so we know roads = linestring = 0, polygon = 1 - keyvalues.z_order = z_order(keyvalues) - return 0, keyvalues, members_superseded, 0, 1, 0 - elseif type == "route" then - keyvalues.z_order = z_order(keyvalues) - return 0, keyvalues, members_superseded, 1, 0, roads(keyvalues) + if roads(object.tags) then + add_roads(object.tags) + end end - - -- Unknown type of relation or no type tag - return 1, keyvalues, members_superseded, 0, 0, 0 end ---- Check if an object with given tags should be treated as polygon --- @param tags OSM tags --- @return 1 if area, 0 if linear -function isarea (tags) - -- Treat objects tagged as area=yes polygon, other area as no - if tags["area"] then - return tags["area"] == "yes" and 1 or 0 +function osm2pgsql.process_relation(object) + -- grab the type tag before filtering tags + local type = object.tags.type + object.tags.type = nil + + if clean_tags(object.tags) then + return end + if type == "boundary" or (type == "multipolygon" and object.tags["boundary"]) then + add_line(object.tags) - -- Search through object's tags - for k, v in pairs(tags) do - -- Check if it has a polygon key and not a linestring override, or a polygon k=v - for _, ptag in ipairs(polygon_keys) do - if k == ptag and v ~= "no" and not (linestring_values[k] and linestring_values[k][v]) then - return 1 - end + if roads(object.tags) then + add_roads(object.tags) end - if (polygon_values[k] and polygon_values[k][v]) then - return 1 - end - end - return 0 -end + add_polygon(object.tags) -function is_in (needle, haystack) - for index, value in ipairs (haystack) do - if value == needle then - return true + elseif type == "multipolygon" then + add_polygon(object.tags) + elseif type == "route" then + add_line(object.tags) + + -- TODO: Remove this, roads tags don't belong on route relations + if roads(object.tags) then + add_roads(object.tags) end end - return false -end - ---- Normalizes layer tags --- @param v The layer tag value --- @return An integer for the layer tag -function layer (v) - return v and string.find(v, "^-?%d+$") and tonumber(v) < 100 and tonumber(v) > -100 and v or nil end diff --git a/scripts/lua/test.lua b/scripts/lua/test.lua index 9f7962fc48..516b205463 100644 --- a/scripts/lua/test.lua +++ b/scripts/lua/test.lua @@ -4,22 +4,71 @@ This file is part of OpenStreetMap Carto and used for validating the Lua tag tra Run it with lua test.lua ]] -require ("openstreetmap-carto") +--- Utility function to do a deep compare +-- (C) Anonymous on snippets.luacode.org, MIT license +function deepcompare(t1,t2) + local ty1 = type(t1) + local ty2 = type(t2) + if ty1 ~= ty2 then return false end + -- non-table types can be directly compared + if ty1 ~= 'table' and ty2 ~= 'table' then return t1 == t2 end ---- compare two tables. --- @param t1 A table --- @param t2 A table --- @return true or false -function equaltables (t1,t2) - for k, v in pairs(t1) do - if t2[k] ~= v then return false end + for k1,v1 in pairs(t1) do + local v2 = t2[k1] + if v2 == nil or not deepcompare(v1,v2) then return false end end - for k, v in pairs(t2) do - if t1[k] ~= v then return false end + for k2,v2 in pairs(t2) do + local v1 = t1[k2] + if v1 == nil or not deepcompare(v1,v2) then return false end end return true end +-- Before testing we need to mock the supplied osm2pgsql object +osm2pgsql = { srid = 3857 } + +-- +local table_definitions = {} +local table_contents = {} + +function osm2pgsql.define_table(definition) + table_definitions[definition.name] = definition + table_contents[definition.name] = {} + return {add_row = function(self, obj) table.insert(table_contents[definition.name], obj) end} +end + +require ("openstreetmap-carto") + +print("TESTING: define_table") + +assert(deepcompare(table_definitions.planet_osm_point.ids, { type = 'node', id_column = 'osm_id' }), "planet_osm_point id column") +assert(deepcompare(table_definitions.planet_osm_point.columns[1], { column = 'way', type = 'point' }), "planet_osm_point way column") +assert(deepcompare(table_definitions.planet_osm_point.columns[2], { column = 'tags', type = 'hstore' }), "planet_osm_point tags column") +assert(deepcompare(table_definitions.planet_osm_point.columns[3], { column = 'layer', type = 'int4' }), "planet_osm_point layer column") +assert(deepcompare(table_definitions.planet_osm_point.columns[4], { column = 'access', type = 'text' }), "planet_osm_point access column") + +assert(deepcompare(table_definitions.planet_osm_line.ids, { type = 'way', id_column = 'osm_id' }), "planet_osm_line id column") +assert(deepcompare(table_definitions.planet_osm_line.columns[1], { column = 'way', type = 'linestring' }), "planet_osm_line way column") +assert(deepcompare(table_definitions.planet_osm_line.columns[2], { column = 'tags', type = 'hstore' }), "planet_osm_line tags column") +assert(deepcompare(table_definitions.planet_osm_line.columns[3], { column = 'layer', type = 'int4' }), "planet_osm_line layer column") +assert(deepcompare(table_definitions.planet_osm_line.columns[4], { column = 'z_order', type = 'int4' }), "planet_osm_line z_order column") +assert(deepcompare(table_definitions.planet_osm_line.columns[5], { column = 'access', type = 'text' }), "planet_osm_line access column") + +assert(deepcompare(table_definitions.planet_osm_roads.ids, { type = 'way', id_column = 'osm_id' }), "planet_osm_roads id column") +assert(deepcompare(table_definitions.planet_osm_roads.columns[1], { column = 'way', type = 'linestring' }), "planet_osm_roads way column") +assert(deepcompare(table_definitions.planet_osm_roads.columns[2], { column = 'tags', type = 'hstore' }), "planet_osm_roads tags column") +assert(deepcompare(table_definitions.planet_osm_roads.columns[3], { column = 'layer', type = 'int4' }), "planet_osm_roads layer column") +assert(deepcompare(table_definitions.planet_osm_roads.columns[4], { column = 'z_order', type = 'int4' }), "planet_osm_roads z_order column") +assert(deepcompare(table_definitions.planet_osm_roads.columns[5], { column = 'access', type = 'text' }), "planet_osm_roads access column") + +assert(deepcompare(table_definitions.planet_osm_polygon.ids, { type = 'way', id_column = 'osm_id' }), "planet_osm_polygon id column") +assert(deepcompare(table_definitions.planet_osm_polygon.columns[1], { column = 'way', type = 'geometry' }), "planet_osm_polygon way column") +assert(deepcompare(table_definitions.planet_osm_polygon.columns[2], { column = 'tags', type = 'hstore' }), "planet_osm_polygon tags column") +assert(deepcompare(table_definitions.planet_osm_polygon.columns[3], { column = 'layer', type = 'int4' }), "planet_osm_polygon layer column") +assert(deepcompare(table_definitions.planet_osm_polygon.columns[4], { column = 'z_order', type = 'int4' }), "planet_osm_polygon z_order column") +assert(deepcompare(table_definitions.planet_osm_polygon.columns[5], { column = 'way_area', type = 'area' }), "planet_osm_polygon way_area column") +assert(deepcompare(table_definitions.planet_osm_polygon.columns[6], { column = 'access', type = 'text' }), "planet_osm_polygon access column") + print("TESTING: z_order") assert(z_order({}) == nil, "test failed: no tags") @@ -47,142 +96,214 @@ assert(z_order({highway="construction", construction="foo"}) == 33 , "test faile assert(z_order({highway="motorway", construction="service"}) == 380 , "test failed: highway=construction + construction=service") print("TESTING: roads") -assert(roads({}) == 0, "test failed: no tags") -assert(roads({foo="bar"}) == 0, "test failed: other tags") -assert(roads({highway="motorway"}) == 1, "test failed: motorway") -assert(roads({railway="rail"}) == 1, "test failed: rail") -assert(roads({highway="residential", railway="rail"}) == 1, "test failed: rail+residential") -assert(roads({railway="turntable"}) == 0, "test failed: rail=turntable") -assert(roads({railway="rail", service="spur"}) == 0, "test failed: rail SSY") -assert(roads({railway="rail", service="main"}) == 1, "test failed: rail non-SSY") -assert(roads({boundary="administrative"}) == 1, "test failed: boundary administrative") +assert(not roads({}), "test failed: no tags") +assert(not roads({foo="bar"}), "test failed: other tags") +assert(roads({highway="motorway"}), "test failed: motorway") +assert(roads({railway="rail"}), "test failed: rail") +assert(roads({highway="residential", railway="rail"}), "test failed: rail+residential") +assert(not roads({railway="turntable"}), "test failed: rail=turntable") +assert(not roads({railway="rail", service="spur"}), "test failed: rail SSY") +assert(roads({railway="rail", service="main"}), "test failed: rail non-SSY") +assert(roads({boundary="administrative"}), "test failed: boundary administrative") print("TESTING: isarea") -assert(isarea({}) == 0, "test failed: no tags") -assert(isarea({foo = "bar"}) == 0, "test failed: random tag") -assert(isarea({area = "yes"}) == 1, "test failed: explicit area") -assert(isarea({area = "no"}) == 0, "test failed: explicit not area") -assert(isarea({area = "no", landuse = "forest"}) == 0, "test failed: explicit not area with polygon tag") -assert(isarea({leisure = "track"}) == 0, "test failed: leisure=track") -assert(isarea({area = "yes", leisure = "track"}) == 1, "test failed: leisure=track with area tag") -assert(isarea({waterway = "river"}) == 0, "test failed: river") -assert(isarea({waterway = "riverbank"}) == 1, "test failed: river") -assert(isarea({highway = "services"}) == 1, "test failed: river") -assert(isarea({natural="cliff"}) == 0, "test failed: cliff") -- issue #3084 -assert(isarea({building = "no"}) == 0, "test failed: building=no") -assert(isarea({building = "no", area = "yes"}) == 1, "test failed: building=no with area tag") -assert(isarea({building = "no", landuse = "forest"}) == 1, "test failed: building=no with other area tag") - -print("TESTING: filter_tags_generic") -assert(({filter_tags_generic({})})[1] == 1, "Untagged filter") -assert(equaltables(({filter_tags_generic({})})[2], {}), "Untagged tags") -assert(({filter_tags_generic({note="foo"})})[1] == 1, "deleted filter") -assert(equaltables(({filter_tags_generic({note="foo"})})[2], {}), "deleted tags") -assert(({filter_tags_generic({foo="bar"})})[1] == 0, "single tag filter") -assert(equaltables(({filter_tags_generic({foo="bar"})})[2], {foo="bar"}), "single tag tags") -assert(({filter_tags_generic({foo="bar", note="baz"})})[1] == 0, "tag + deleted tag filter") -assert(equaltables(({filter_tags_generic({foo="bar", note="baz"})})[2], {foo="bar"}), "tag + deleted tags") -assert(({filter_tags_generic({["note:xx"]="foo"})})[1] == 1, "wildcard deleted filter") -assert(equaltables(({filter_tags_generic({["note:xx"]="foo"})})[2], {}), "wildcard deleted tags") -assert(({filter_tags_generic({["note:xx"]="foo", foo="bar"})})[1] == 0, "wildcard deleted + tag filter") -assert(equaltables(({filter_tags_generic({["note:xx"]="foo", foo="bar"})})[2], {foo="bar"}), "wildcard deleted + tag tags") - -assert(({filter_tags_generic({["foo:note:xx"]="foo"})})[1] == 0, "prefix later in tag filter") -assert(equaltables(({filter_tags_generic({["foo:note:xx"]="foo"})})[2], {["foo:note:xx"]="foo"}), "prefix later in tag tags") - -print("TESTING: filter_tags_relation_member") - ---- Tests filter_tags_relation_member against expected values --- @param keyvalues OSM tags, after processing by relation transform --- @param keyvaluemembers OSM tags of relation members, after processing by way transform --- @param filter expected filter result --- @param cols expected cols result --- @param member_superseded expected member_superseded result --- @param boundary expected boundary result --- @param polygon expected polygon result --- @param roads expected roads result -local function check_rel_member(keyvalues, keyvaluemembers, filter, cols, member_superseded, boundary, polygon, roads) - - local i = 0 - for _ in pairs(keyvaluemembers) do - i = i + 1 - end +assert(not isarea({}), "test failed: no tags") +assert(not isarea({foo = "bar"}), "test failed: random tag") +assert(isarea({area = "yes"}), "test failed: explicit area") +assert(not isarea({area = "no"}), "test failed: explicit not area") +assert(not isarea({area = "no", landuse = "forest"}), "test failed: explicit not area with polygon tag") +assert(not isarea({leisure = "track"}), "test failed: leisure=track") +assert(isarea({area = "yes", leisure = "track"}), "test failed: leisure=track with area tag") +assert(not isarea({waterway = "river"}), "test failed: river") +assert(isarea({waterway = "riverbank"}), "test failed: river") +assert(isarea({highway = "services"}), "test failed: river") +assert(not isarea({natural="cliff"}), "test failed: cliff") -- issue #3084 +assert(not isarea({building = "no"}), "test failed: building=no") +assert(isarea({building = "no", area = "yes"}), "test failed: building=no with area tag") +assert(isarea({building = "no", landuse = "forest"}), "test failed: building=no with other area tag") - local actual_filter, actual_cols, actual_member_superseded, actual_boundary, actual_polygon, actual_roads - = filter_tags_relation_member(keyvalues, keyvaluemembers, nil, i) +print("TESTING: clean_tags") +assert(clean_tags({}), "Untagged") - if actual_filter ~= filter then - print("filter mismatch") - return false - end - if not equaltables(actual_cols, cols) then - print("cols mismatch") - return false - end - if not equaltables(actual_member_superseded, member_superseded) then - print("member_superseded mismatch, actual table was") - for i, v in ipairs(actual_member_superseded) do - print(i, v) - end - return false - end - if actual_boundary ~= boundary then - print("boundary mismatch") - return false - end - if actual_polygon ~= polygon then - print("polygon mismatch") - return false - end - if actual_roads ~= roads then - print("roads mismatch") - return false - end - return true -end +tags={odbl = "yes"} +assert(clean_tags(tags), "delete tag return") +assert(deepcompare(tags, {}), "delete tags") + +tags={['source:note'] = "yes"} +assert(clean_tags(tags), "delete wildcard tag return") +assert(deepcompare(tags, {}), "delete wildcard tags") + +tags={natural = "tree"} +assert(not clean_tags(tags), "no delete tags return") +assert(deepcompare(tags, {natural = "tree"}), "no delete tags") + +tags={natural = "tree", odbl = "yes"} +assert(not clean_tags(tags), "mixed tags return") +assert(deepcompare(tags, {natural = "tree"}), "mixed tags") + +print("TESTING: add_point") +table_contents.planet_osm_point = {} +add_point({natural = "tree"}) +assert(deepcompare(table_contents.planet_osm_point[1], {natural = "tree", tags = {}}), "Tag with column") + +table_contents.planet_osm_point = {} +add_point({natural = "tree", foo = "bar"}) +assert(deepcompare(table_contents.planet_osm_point[1], {natural = "tree", tags = {foo = "bar"}}), "Tag with column + hstore") + +table_contents.planet_osm_point = {} +add_point({foo = "bar"}) +assert(deepcompare(table_contents.planet_osm_point[1], {tags = {foo = "bar"}}), "hstore only") + +print("TESTING: add_line") +table_contents.planet_osm_line = {} +add_line({highway = "road"}) +assert(deepcompare(table_contents.planet_osm_line[1], {highway = "road", z_order = 330, tags = {}, way = { create = 'line', split_at = 100000 }}), "Tag with column") + +table_contents.planet_osm_line = {} +add_line({highway = "road", foo = "bar"}) +assert(deepcompare(table_contents.planet_osm_line[1], {highway = "road", z_order = 330, tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "Tag with column + hstore") + +table_contents.planet_osm_line = {} +add_line({foo = "bar"}) +assert(deepcompare(table_contents.planet_osm_line[1], {tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "hstore only") + +print("TESTING: add_roads") +table_contents.planet_osm_roads = {} +add_roads({highway = "road"}) +assert(deepcompare(table_contents.planet_osm_roads[1], {highway = "road", z_order = 330, tags = {}, way = { create = 'line', split_at = 100000 }}), "Tag with column") + +table_contents.planet_osm_roads = {} +add_roads({highway = "road", foo = "bar"}) +assert(deepcompare(table_contents.planet_osm_roads[1], {highway = "road", z_order = 330, tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "Tag with column + hstore") + +table_contents.planet_osm_roads = {} +add_roads({foo = "bar"}) +assert(deepcompare(table_contents.planet_osm_roads[1], {tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "hstore only") + +print("TESTING: add_polygon") +table_contents.planet_osm_polygon = {} +add_polygon({natural = "wood"}) +assert(deepcompare(table_contents.planet_osm_polygon[1], {natural = "wood", tags = {}, way = { create = 'area', multi=true }}), "Tag with column") + +table_contents.planet_osm_polygon = {} +add_polygon({natural = "wood", foo = "bar"}) +assert(deepcompare(table_contents.planet_osm_polygon[1], {natural = "wood", tags = {foo = "bar"}, way = { create = 'area', multi=true }}), "Tag with column + hstore") + +table_contents.planet_osm_polygon = {} +add_polygon({foo = "bar"}) +assert(deepcompare(table_contents.planet_osm_polygon[1], {tags = {foo = "bar"}, way = { create = 'area', multi=true }}), "hstore only") + +print("TESTING: osm2pgsql.process_node") +table_contents.planet_osm_point = {} +osm2pgsql.process_node({tags = {}}) +assert(deepcompare(table_contents.planet_osm_point, {}), "Untagged") + +osm2pgsql.process_node({tags = {odbl = "yes"}}) +assert(deepcompare(table_contents.planet_osm_point, {}), "Deleted tag") + +-- By running process_node then manually running add_point the first and second entry +-- in the output tables should be the same. +osm2pgsql.process_node({tags = {natural = "tree", foo = "bar"}}) +add_point({natural = "tree", foo = "bar"}) +assert(deepcompare(table_contents.planet_osm_point[1], table_contents.planet_osm_point[2]), "Accepted tag") + +print("TESTING: osm2pgsql.process_way") +table_contents.planet_osm_line = {} +table_contents.planet_osm_roads = {} +table_contents.planet_osm_polygon = {} + +osm2pgsql.process_way({tags = {}}) +assert(deepcompare(table_contents.planet_osm_line, {}), "Untagged line") +assert(deepcompare(table_contents.planet_osm_roads, {}), "Untagged roads") +assert(deepcompare(table_contents.planet_osm_polygon, {}), "Untagged polygon") + +osm2pgsql.process_way({tags = {odbl = "yes"}}) +assert(deepcompare(table_contents.planet_osm_line, {}), "Deleted tag line") +assert(deepcompare(table_contents.planet_osm_roads, {}), "Deleted tag roads") +assert(deepcompare(table_contents.planet_osm_polygon, {}), "Deleted tag polygon") + +osm2pgsql.process_way({is_closed = false, tags = { highway = "road"}}) +add_line({ highway = "road"}) +assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_line[2]), "Line tag, open way, line table") +assert(deepcompare(table_contents.planet_osm_roads, {}), "Line tag, open way, roads table") +assert(deepcompare(table_contents.planet_osm_polygon, {}), "Line tag, open way, polygon table") +table_contents.planet_osm_line = {} + +osm2pgsql.process_way({is_closed = true, tags = { highway = "motorway"}}) +add_line({highway = "motorway"}) +add_roads({highway = "motorway"}) +assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_line[2]), "Line tag, closed way, line table") +assert(deepcompare(table_contents.planet_osm_roads[1], table_contents.planet_osm_roads[2]), "Line tag, closed way, line table") +assert(deepcompare(table_contents.planet_osm_polygon, {}), "Line tag, closed way, polygon table") +table_contents.planet_osm_line = {} +table_contents.planet_osm_roads = {} + +osm2pgsql.process_way({is_closed = false, tags = { natural = "wood"}}) +add_line({natural = "wood"}) +assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_line[1]), "area tag, open way, line table") +assert(deepcompare(table_contents.planet_osm_roads, {}), "area tag, open way, roads table") +assert(deepcompare(table_contents.planet_osm_polygon, {}), "area tag, open way, polygon table") +table_contents.planet_osm_line = {} + +osm2pgsql.process_way({is_closed = true, tags = { natural = "wood"}}) +add_polygon({natural = "wood"}) +assert(deepcompare(table_contents.planet_osm_line, {}), "area tag, closed way, line table") +assert(deepcompare(table_contents.planet_osm_roads, {}), "area tag, closed way, roads table") +assert(deepcompare(table_contents.planet_osm_polygon[1], table_contents.planet_osm_polygon[2]), "area tag, closed way, polygon table") + +print("TESTING: osm2pgsql.process_relation") +table_contents.planet_osm_line = {} +table_contents.planet_osm_roads = {} +table_contents.planet_osm_polygon = {} + +osm2pgsql.process_relation({tags = {}}) +assert(deepcompare(table_contents.planet_osm_line, {}), "Untagged line") +assert(deepcompare(table_contents.planet_osm_roads, {}), "Untagged roads") +assert(deepcompare(table_contents.planet_osm_polygon, {}), "Untagged polygon") + +osm2pgsql.process_relation({tags = {odbl = "yes"}}) +assert(deepcompare(table_contents.planet_osm_line, {}), "Deleted tag line") +assert(deepcompare(table_contents.planet_osm_roads, {}), "Deleted tag roads") +assert(deepcompare(table_contents.planet_osm_polygon, {}), "Deleted tag polygon") + +osm2pgsql.process_relation({tags = {type = "multipolygon", odbl = "yes"}}) +assert(deepcompare(table_contents.planet_osm_line, {}), "Deleted tag line with type") +assert(deepcompare(table_contents.planet_osm_roads, {}), "Deleted tag roads with type") +assert(deepcompare(table_contents.planet_osm_polygon, {}), "Deleted tag polygon with type") + +osm2pgsql.process_relation({tags = {type = "boundary", boundary = "foo"}}) +add_line({boundary = "foo"}) +add_polygon({boundary = "foo"}) +assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_line[2]), "Boundary line") +assert(deepcompare(table_contents.planet_osm_roads, {}), "Boundary roads") +assert(deepcompare(table_contents.planet_osm_polygon[1], table_contents.planet_osm_polygon[2]), "Boundary polygon") +table_contents.planet_osm_line = {} +table_contents.planet_osm_polygon = {} + +osm2pgsql.process_relation({tags = {type = "boundary", boundary = "administrative"}}) +add_line({boundary = "administrative"}) +add_roads({boundary = "administrative"}) +add_polygon({boundary = "administrative"}) +assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_line[2]), "admin boundary line") +assert(deepcompare(table_contents.planet_osm_roads[1], table_contents.planet_osm_roads[2]), "admin boundary roads") +assert(deepcompare(table_contents.planet_osm_polygon[1], table_contents.planet_osm_polygon[2]), "admin boundary polygon") +table_contents.planet_osm_line = {} +table_contents.planet_osm_polygon = {} +table_contents.planet_osm_roads = {} + +osm2pgsql.process_relation({tags = {type = "multipolygon", natural = "tree"}}) +add_polygon({natural = "tree"}) +assert(deepcompare(table_contents.planet_osm_line, {}), "MP line") +assert(deepcompare(table_contents.planet_osm_roads, {}), "MP roads") +assert(deepcompare(table_contents.planet_osm_polygon[1], table_contents.planet_osm_polygon[2]), "admin boundary polygon") +table_contents.planet_osm_polygon = {} + +osm2pgsql.process_relation({tags = {type = "route", route = "road"}}) +add_line({route = "road"}) -assert(check_rel_member({}, {}, 1, {}, {}, 0, 0, 0), "test failed: untagged memberless relation") -assert(check_rel_member({}, {{}}, 1, {}, {0}, 0, 0, 0), "test failed: untagged relation") - -assert(check_rel_member({type="multipolygon"}, {{}}, 1, {}, {0}, 0, 0, 0), - "test failed: untagged MP") -assert(check_rel_member({type="multipolygon", foo="bar"}, {{}}, 0, {foo="bar"}, {0}, 0, 1, 0), - "test failed: MP with tag") - --- New-style MPs -assert(check_rel_member({type="multipolygon", foo="bar"}, {{},{}}, 0, {foo="bar"}, {0,0}, 0, 1, 0), - "test failed: MP with tag, two ways") -assert(check_rel_member({type="multipolygon", foo="bar"}, {{baz="qax"}}, 0, {foo="bar"}, {0}, 0, 1, 0), - "test failed: MP with tag, way with different tag") -assert(check_rel_member({type="multipolygon", foo="bar"}, {{baz="qax"}, {}}, 0, {foo="bar"}, {0,0}, 0, 1, 0), - "test failed: MP with tag, way with different tag + untagged way") -assert(check_rel_member({type="multipolygon", foo="bar"}, {{foo="bar"}}, 0, {foo="bar"}, {0}, 0, 1, 0), - "test failed: MP with tag, way with same tag") -assert(check_rel_member({type="multipolygon", foo="bar"}, {{foo="bar"},{}}, 0, {foo="bar"}, {0,0}, 0, 1, 0), - "test failed: MP with tag, way with same tag + untagged way") -assert(check_rel_member({type="multipolygon", foo="bar"}, {{foo="bar"}, {baz="qax"}}, 0, {foo="bar"}, {0,0}, 0, 1, 0), - "test failed: MP with tag, way with same tag") - --- Old-style MPs -assert(check_rel_member({type="multipolygon"}, {{foo="bar"}}, 1, {}, {0}, 0, 0, 0), - "test failed: MP w/o tag, way with tag") -assert(check_rel_member({type="multipolygon"}, {{foo="bar"}, {}}, 1, {}, {0,0}, 0, 0, 0), - "test failed: MP w/o tag, way with tag + untagged way") -assert(check_rel_member({type="multipolygon"}, {{foo="bar"}, {baz="qax"}}, 1, {}, {0,0}, 0, 0, 0), - "test failed: MP w/o tag, way with tag + way with other tag") - --- Boundary relations -assert(check_rel_member({type="boundary"}, {{}}, 1, {}, {0}, 0, 0, 0), - "test failed: untagged boundary") -assert(check_rel_member({type="boundary", boundary="administrative"}, {{}}, 0, {boundary="administrative"}, {0}, 1, 0, 1), - "test failed: untagged boundary") -assert(check_rel_member({type="boundary", boundary="administrative"}, {{}}, 0, {boundary="administrative"}, {0}, 1, 0, 1), - "test failed: untagged boundary") -assert(check_rel_member({type="boundary", boundary="administrative"}, {{foo="bar"}}, 0, {boundary="administrative"}, {0}, 1, 0, 1), - "test failed: untagged boundary, tagged way") - --- Route relations -assert(check_rel_member({type="route"}, {{}}, 1, {}, {0}, 0, 0, 0), - "test failed: untagged route") -assert(check_rel_member({type="route", route="road"}, {{}}, 0, {route="road"}, {0}, 1, 0, 0), - "test failed: tagged route") +assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_line[2]), "route line") +assert(deepcompare(table_contents.planet_osm_roads, {}), "route roads") +assert(deepcompare(table_contents.planet_osm_polygon, {}), "route polygon") +table_contents.planet_osm_line = {} +table_contents.planet_osm_polygon = {} +table_contents.planet_osm_roads = {} From 1e38b850376f21b754653217d79b6d10ceddebf0 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sun, 26 Apr 2020 18:12:07 -0700 Subject: [PATCH 02/54] Refactor creation of tables to reduce duplication --- openstreetmap-carto.lua | 259 +++++++++++++++++++--------------------- 1 file changed, 123 insertions(+), 136 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 93a2f7eee4..1ff0f0a887 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -5,161 +5,148 @@ local tables = {} -- A list of columns per table, replacing the osm2pgsql .style file -- These need to be ordered, so that means a list -local point_columns = { - 'access', - 'addr:housename', - 'addr:housenumber', - 'admin_level', - 'aerialway', - 'aeroway', - 'amenity', - 'barrier', - 'boundary', - 'building', - 'highway', - 'historic', - 'junction', - 'landuse', - 'leisure', - 'lock', - 'man_made', - 'military', - 'name', - 'natural', - 'oneway', - 'place', - 'power', - 'railway', - 'ref', - 'religion', - 'shop', - 'tourism', - 'water', - 'waterway' } - -local line_columns = { - 'access', - 'addr:housename', - 'addr:housenumber', - 'addr:interpolation', - 'admin_level', - 'aerialway', - 'aeroway', - 'amenity', - 'barrier', - 'bicycle', - 'bridge', - 'boundary', - 'building', - 'construction', - 'covered', - 'foot', - 'highway', - 'historic', - 'horse', - 'junction', - 'landuse', - 'leisure', - 'lock', - 'man_made', - 'military', - 'name', - 'natural', - 'oneway', - 'place', - 'power', - 'railway', - 'ref', - 'religion', - 'route', - 'service', - 'shop', - 'surface', - 'tourism', - 'tracktype', - 'tunnel', - 'water', - 'waterway'} +local pg_cols = { + point = { + 'access', + 'addr:housename', + 'addr:housenumber', + 'admin_level', + 'aerialway', + 'aeroway', + 'amenity', + 'barrier', + 'boundary', + 'building', + 'highway', + 'historic', + 'junction', + 'landuse', + 'leisure', + 'lock', + 'man_made', + 'military', + 'name', + 'natural', + 'oneway', + 'place', + 'power', + 'railway', + 'ref', + 'religion', + 'shop', + 'tourism', + 'water', + 'waterway' + }, + line = { + 'access', + 'addr:housename', + 'addr:housenumber', + 'addr:interpolation', + 'admin_level', + 'aerialway', + 'aeroway', + 'amenity', + 'barrier', + 'bicycle', + 'bridge', + 'boundary', + 'building', + 'construction', + 'covered', + 'foot', + 'highway', + 'historic', + 'horse', + 'junction', + 'landuse', + 'leisure', + 'lock', + 'man_made', + 'military', + 'name', + 'natural', + 'oneway', + 'place', + 'power', + 'railway', + 'ref', + 'religion', + 'route', + 'service', + 'shop', + 'surface', + 'tourism', + 'tracktype', + 'tunnel', + 'water', + 'waterway' + } +} --- These are the same for now -local polygon_columns = line_columns -local road_columns = line_columns +pg_cols.roads = pg_cols.line +pg_cols.polygon = pg_cols.line --- Process the columns above -local point_col_def = { - { column = 'way', type = 'point' }, - { column = 'tags', type = 'hstore' }, - { column = 'layer', type = 'int4' } +-- These columns aren't text columns +col_definitions = { + point = { + { column = 'way', type = 'point' }, + { column = 'tags', type = 'hstore' }, + { column = 'layer', type = 'int4' } + }, + line = { + { column = 'way', type = 'linestring' }, + { column = 'tags', type = 'hstore' }, + { column = 'layer', type = 'int4' }, + { column = 'z_order', type = 'int4' } + }, + roads = { + { column = 'way', type = 'linestring' }, + { column = 'tags', type = 'hstore' }, + { column = 'layer', type = 'int4' }, + { column = 'z_order', type = 'int4' } + }, + polygon = { + { column = 'way', type = 'geometry' }, + { column = 'tags', type = 'hstore' }, + { column = 'layer', type = 'int4' }, + { column = 'z_order', type = 'int4' }, + { column = 'way_area', type = 'area' } + } } -local point_columns_map = {} -for _, key in ipairs(point_columns) do - table.insert(point_col_def, {column = key, type = "text"}) - point_columns_map[key] = true +-- Combine the two sets of columns and create a map with column names. +-- The latter is needed for quick lookup to see if a tag has a column. +local columns_map = {} +for tablename, columns in pairs(pg_cols) do + columns_map[tablename] = {} + for _, key in ipairs(columns) do + table.insert(col_definitions[tablename], {column = key, type = "text"}) + columns_map[tablename][key] = true + end end tables.point = osm2pgsql.define_table{ name = 'planet_osm_point', ids = { type = 'node', id_column = 'osm_id' }, - columns = point_col_def -} - -local line_col_def = { - { column = 'way', type = 'linestring' }, - { column = 'tags', type = 'hstore' }, - { column = 'layer', type = 'int4' }, - { column = 'z_order', type = 'int4' } + columns = col_definitions.point } -local line_columns_map = {} -for _, key in ipairs(line_columns) do - table.insert(line_col_def, {column = key, type = "text"}) - line_columns_map[key] = true -end - tables.line = osm2pgsql.define_table{ name = 'planet_osm_line', ids = { type = 'way', id_column = 'osm_id' }, - columns = line_col_def -} - -local roads_col_def = { - { column = 'way', type = 'linestring' }, - { column = 'tags', type = 'hstore' }, - { column = 'layer', type = 'int4' }, - { column = 'z_order', type = 'int4' } + columns = col_definitions.line } -local roads_columns_map = {} -for _, key in ipairs(road_columns) do - table.insert(roads_col_def, {column = key, type = "text"}) - roads_columns_map[key] = true -end - tables.roads = osm2pgsql.define_table{ name = 'planet_osm_roads', ids = { type = 'way', id_column = 'osm_id' }, - columns = roads_col_def + columns = col_definitions.roads } - -local polygon_col_def = { - { column = 'way', type = 'geometry' }, - { column = 'tags', type = 'hstore' }, - { column = 'layer', type = 'int4' }, - { column = 'z_order', type = 'int4' }, - { column = 'way_area', type = 'area' } -} - -local polygon_columns_map = {} -for _, key in ipairs(polygon_columns) do - table.insert(polygon_col_def, {column = key, type = "text"}) - polygon_columns_map[key] = true -end - tables.polygon = osm2pgsql.define_table{ name = 'planet_osm_polygon', ids = { type = 'way', id_column = 'osm_id' }, - columns = polygon_col_def + columns = col_definitions.polygon } -- Objects with any of the following keys will be treated as polygon @@ -521,13 +508,13 @@ function split_tags(tags, tag_map) end function add_point(tags) - local cols = split_tags(tags, point_columns_map) + local cols = split_tags(tags, columns_map.point) cols['layer'] = layer(tags['layer']) tables.point:add_row(cols) end function add_line(tags) - local cols = split_tags(tags, line_columns_map) + local cols = split_tags(tags, columns_map.line) cols['layer'] = layer(tags['layer']) cols['z_order'] = z_order(tags) cols.way = { create = 'line', split_at = 100000 } @@ -535,7 +522,7 @@ function add_line(tags) end function add_roads(tags) - local cols = split_tags(tags, roads_columns_map) + local cols = split_tags(tags, columns_map.roads) cols['layer'] = layer(tags['layer']) cols['z_order'] = z_order(tags) cols.way = { create = 'line', split_at = 100000 } @@ -543,7 +530,7 @@ function add_roads(tags) end function add_polygon(tags) - local cols = split_tags(tags, polygon_columns_map) + local cols = split_tags(tags, columns_map.polygon) cols['layer'] = layer(tags['layer']) cols['z_order'] = z_order(tags) cols.way = { create = 'area', multi = true } From 717d5490a7958c1f014358ab2384231f341244e9 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sun, 26 Apr 2020 18:13:03 -0700 Subject: [PATCH 03/54] Remove unused lua function --- openstreetmap-carto.lua | 9 --------- 1 file changed, 9 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 1ff0f0a887..2e07d0e007 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -452,15 +452,6 @@ function isarea (tags) return false end -function is_in (needle, haystack) - for index, value in ipairs (haystack) do - if value == needle then - return true - end - end - return false -end - --- Normalizes layer tags -- @param v The layer tag value -- @return An integer for the layer tag From c0f759b4df86fd484f933bda93d9483b51d8358b Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Mon, 27 Apr 2020 20:30:05 -0700 Subject: [PATCH 04/54] Create a route table Because one way may have many parent relatiolns, a table is used so at query time the data in the line table can be joined with the route table. Also, this avoids stage 2 osm2pgsql processing. --- indexes.sql | 2 ++ indexes.yml | 4 ++++ openstreetmap-carto.lua | 32 +++++++++++++++++++++++++++++++- scripts/indexes.py | 16 +++++++++------- scripts/lua/test.lua | 29 +++++++++++++++++++++++++++-- 5 files changed, 73 insertions(+), 10 deletions(-) diff --git a/indexes.sql b/indexes.sql index f0a998bdb0..d99cc8cca3 100644 --- a/indexes.sql +++ b/indexes.sql @@ -43,3 +43,5 @@ CREATE INDEX planet_osm_roads_admin_low CREATE INDEX planet_osm_roads_roads_ref ON planet_osm_roads USING GIST (way) WHERE highway IS NOT NULL AND ref IS NOT NULL; +CREATE INDEX planet_osm_route_member_id + ON planet_osm_route USING btree (member_id); diff --git a/indexes.yml b/indexes.yml index 575cad7f22..7f2ca36f3b 100644 --- a/indexes.yml +++ b/indexes.yml @@ -40,3 +40,7 @@ roads: where: boundary = 'administrative' roads_ref: where: highway IS NOT NULL AND ref IS NOT NULL +route: + # The route table has no geospatial data, so it's index is a btree + member_id: + column: btree (member_id) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 2e07d0e007..ef9f35114e 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -81,6 +81,11 @@ local pg_cols = { 'tunnel', 'water', 'waterway' + }, + route = { + 'route', + 'ref', + 'network' } } @@ -112,6 +117,11 @@ col_definitions = { { column = 'layer', type = 'int4' }, { column = 'z_order', type = 'int4' }, { column = 'way_area', type = 'area' } + }, + route = { + { column = 'member_id', type = 'int8' }, + { column = 'member_position', type = 'int4' }, + { column = 'tags', type = 'hstore' } } } @@ -143,12 +153,19 @@ tables.roads = osm2pgsql.define_table{ ids = { type = 'way', id_column = 'osm_id' }, columns = col_definitions.roads } + tables.polygon = osm2pgsql.define_table{ name = 'planet_osm_polygon', ids = { type = 'way', id_column = 'osm_id' }, columns = col_definitions.polygon } +tables.route = osm2pgsql.define_table{ + name = 'planet_osm_route', + ids = { type = 'relation', id_column = 'osm_id' }, + columns = col_definitions.route +} + -- Objects with any of the following keys will be treated as polygon local polygon_keys = { 'abandoned:aeroway', @@ -498,6 +515,8 @@ function split_tags(tags, tag_map) return cols end + +-- TODO: Make add_* take object, not object.tags function add_point(tags) local cols = split_tags(tags, columns_map.point) cols['layer'] = layer(tags['layer']) @@ -528,6 +547,17 @@ function add_polygon(tags) tables.polygon:add_row(cols) end +function add_route(object) + for i, member in ipairs(object.members) do + if member.type == 'w' then + local cols = split_tags(object.tags, columns_map.roads) + cols.member_id = member.ref + cols.member_position = i + tables.route:add_row(cols) + end + end +end + function osm2pgsql.process_node(object) if clean_tags(object.tags) then return @@ -574,7 +604,7 @@ function osm2pgsql.process_relation(object) add_polygon(object.tags) elseif type == "route" then add_line(object.tags) - + add_route(object) -- TODO: Remove this, roads tags don't belong on route relations if roads(object.tags) then add_roads(object.tags) diff --git a/scripts/indexes.py b/scripts/indexes.py index ba95b1a43c..39fde46f0a 100755 --- a/scripts/indexes.py +++ b/scripts/indexes.py @@ -8,16 +8,17 @@ import argparse, sys, os, yaml -def index_statement(table, name, conditions=None, concurrent=False,notexist=False, fillfactor=None): +def index_statement(table, name, conditions=None, using=None, concurrent=False, notexist=False, fillfactor=None): options = ' CONCURRENTLY' if concurrent else '' options += ' IF NOT EXISTS' if notexist else '' storage = '' if fillfactor is None else '\n WITH (fillfactor={})'.format(fillfactor) where = '' if conditions is None else '\n WHERE {}'.format(conditions) + column = 'GIST (way)' if using is None else using return ('CREATE INDEX{options} {table}_{name}\n' + - ' ON {table} USING GIST (way)' + + ' ON {table} USING {column}' + '{storage}' + '{where};\n').format(table="planet_osm_"+table, name=name, - storage=storage, options=options, where=where) + storage=storage, options=options, where=where, column=column) def parse(cb): with open(os.path.join(os.path.dirname(__file__), '../indexes.yml')) as yaml_file: @@ -25,7 +26,8 @@ def parse(cb): for table, data in sorted(indexes.items()): for name, definition in sorted(data.items()): - cb(table, name, definition["where"]) + cb(table, name, definition.get("where"), + definition.get("column")) # The same as parse, but for osm2pgsql-built indexes def osm2pgsql_parse(cb): @@ -42,14 +44,14 @@ def osm2pgsql_parse(cb): parser.add_argument('--reindex', help='Rebuild existing indexes', action='store_true', default=False) args = parser.parse_args() -def cb (table, name, where): - print(index_statement(table, name, where, args.concurrent, args.notexist, args.fillfactor), end='') +def cb (table, name, where, column): + print(index_statement(table, name, where, column, args.concurrent, args.notexist, args.fillfactor), end='') def reindex_cb(table, name, where): if not args.concurrent: print('REINDEX planet_osm_{table}_{name};'.format(table=table, name=name)) else: - # Rebuilding indexes concurently requires making a new index, dropping the old one, and renaming. + # Rebuilding indexes concurrently requires making a new index, dropping the old one, and renaming. print('ALTER INDEX planet_osm_{table}_{name} RENAME TO planet_osm_{table}_{name}_old;'.format(table=table, name=name)) cb(table, name, where) print('DROP INDEX planet_osm_{table}_{name}_old;\n'.format(table=table, name=name)) diff --git a/scripts/lua/test.lua b/scripts/lua/test.lua index 516b205463..661472e8ad 100644 --- a/scripts/lua/test.lua +++ b/scripts/lua/test.lua @@ -69,6 +69,13 @@ assert(deepcompare(table_definitions.planet_osm_polygon.columns[4], { column = ' assert(deepcompare(table_definitions.planet_osm_polygon.columns[5], { column = 'way_area', type = 'area' }), "planet_osm_polygon way_area column") assert(deepcompare(table_definitions.planet_osm_polygon.columns[6], { column = 'access', type = 'text' }), "planet_osm_polygon access column") +assert(deepcompare(table_definitions.planet_osm_route.ids, { type = 'relation', id_column = 'osm_id' }), "planet_osm_route id column") +assert(deepcompare(table_definitions.planet_osm_route.columns[1], { column = 'member_id', type = 'int8' }), "planet_osm_route member_id column") +assert(deepcompare(table_definitions.planet_osm_route.columns[2], { column = 'member_position', type = 'int4' }), "planet_osm_route member_position column") +assert(deepcompare(table_definitions.planet_osm_route.columns[3], { column = 'tags', type = 'hstore' }), "planet_osm_route tags column") +assert(deepcompare(table_definitions.planet_osm_route.columns[4], { column = 'route', type = 'text' }), "planet_osm_route route column") + + print("TESTING: z_order") assert(z_order({}) == nil, "test failed: no tags") @@ -193,6 +200,22 @@ table_contents.planet_osm_polygon = {} add_polygon({foo = "bar"}) assert(deepcompare(table_contents.planet_osm_polygon[1], {tags = {foo = "bar"}, way = { create = 'area', multi=true }}), "hstore only") + +print("TESTING: add_route") +table_contents.planet_osm_route = {} +add_route({tags = {route = "unicycle"}, members = {{type = 'w', ref = 1234, role = 'foo'}}}) +assert(deepcompare(table_contents.planet_osm_route[1], {route = "unicycle", member_id = 1234, member_position = 1, tags = {}}), "Route with 1 way") + +table_contents.planet_osm_route = {} +add_route({tags = {route = "unicycle"}, members = {{type = 'w', ref = 1234, role = 'foo'}, {type = 'n', ref = 1235, role = 'foo'}, }}) +assert(deepcompare(table_contents.planet_osm_route[1], {route = "unicycle", member_id = 1234, member_position = 1, tags = {}}), "Route with 1 way + 1 node") + +table_contents.planet_osm_route = {} +add_route({tags = {route = "unicycle"}, members = {{type = 'w', ref = 1234, role = 'foo'}, {type = 'w', ref = 1235, role = 'foo'}}}) +assert(deepcompare(table_contents.planet_osm_route[1], {route = "unicycle", member_id = 1234, member_position = 1, tags = {}}), "Route with 2 ways") +assert(deepcompare(table_contents.planet_osm_route[2], {route = "unicycle", member_id = 1235, member_position = 2, tags = {}}), "Route with 2 ways") + + print("TESTING: osm2pgsql.process_node") table_contents.planet_osm_point = {} osm2pgsql.process_node({tags = {}}) @@ -255,6 +278,7 @@ print("TESTING: osm2pgsql.process_relation") table_contents.planet_osm_line = {} table_contents.planet_osm_roads = {} table_contents.planet_osm_polygon = {} +table_contents.planet_osm_route = {} osm2pgsql.process_relation({tags = {}}) assert(deepcompare(table_contents.planet_osm_line, {}), "Untagged line") @@ -298,12 +322,13 @@ assert(deepcompare(table_contents.planet_osm_roads, {}), "MP roads") assert(deepcompare(table_contents.planet_osm_polygon[1], table_contents.planet_osm_polygon[2]), "admin boundary polygon") table_contents.planet_osm_polygon = {} -osm2pgsql.process_relation({tags = {type = "route", route = "road"}}) +osm2pgsql.process_relation({tags = {type = "route", route = "road"}, members = {{type = 'w', ref = 1234, role = 'forward'}}}) add_line({route = "road"}) - +add_route({tags = { route = "road"}, members = {{type = 'w', ref = 1234, role = 'forward'}}}) assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_line[2]), "route line") assert(deepcompare(table_contents.planet_osm_roads, {}), "route roads") assert(deepcompare(table_contents.planet_osm_polygon, {}), "route polygon") +assert(deepcompare(table_contents.planet_osm_route[1], table_contents.planet_osm_route[2]), "route route") table_contents.planet_osm_line = {} table_contents.planet_osm_polygon = {} table_contents.planet_osm_roads = {} From 6a6e69d41fab58fdbaa4c161fc22de9d06f600ff Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Tue, 28 Apr 2020 19:24:16 -0700 Subject: [PATCH 05/54] Skip Travis osm2pgsql tests The Flex backend requires a more recent osm2pgsql than Travis has, so skip them for now. --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 15397f385c..b0e1455029 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,9 +39,10 @@ script: # Check the indexes and road colours files are up to date - diff -qu <(scripts/indexes.py) indexes.sql - diff -qu <(scripts/generate_road_colours.py) style/road-colors-generated.mss + # Skip the next two steps because they required a more recent version of osm2pgsql # Create the PostgreSQL tables - - osm2pgsql -G --hstore --style openstreetmap-carto.style --tag-transform-script openstreetmap-carto.lua -U postgres -d gis -r xml <(echo '') + # - osm2pgsql -G --hstore --style openstreetmap-carto.style --tag-transform-script openstreetmap-carto.lua -U postgres -d gis -r xml <(echo '') # Apply the custom indexes - - psql -1Xq -v ON_ERROR_STOP=1 -d gis -f indexes.sql + # - psql -1Xq -v ON_ERROR_STOP=1 -d gis -f indexes.sql # Test for classes in project.mml (not supported in vector tiles) - '! grep "class:" project.mml > /dev/null' From 924e71fb722e3fcb2983c14201e573d0dc49c805 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sat, 6 Feb 2021 21:38:41 -0800 Subject: [PATCH 06/54] Update README with osm2pgsql version requirements --- INSTALL.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index b7a34e4055..9c85f63696 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -3,7 +3,7 @@ This document describes how to manually configure your system for running OpenStreetMap Carto. If you prefer quick, platform independent setup for a development environment, without the need to install and configure tools by hand, follow a Docker installation guide in [DOCKER.md](https://github.com/gravitystorm/openstreetmap-carto/blob/master/DOCKER.md). ## OpenStreetMap data -You need OpenStreetMap data loaded into a PostGIS database (see below for [dependencies](#dependencies)). These stylesheets expect a database generated with osm2pgsql using the flex backend and the default database name (`gis`). This requires osm2pgsql built from source. +You need OpenStreetMap data loaded into a PostGIS database (see below for [dependencies](#dependencies)). These stylesheets expect a database generated with osm2pgsql using the flex backend and the default database name (`gis`). This requires osm2pgsql 1.4.1 or later. Start by creating a database @@ -101,7 +101,7 @@ To display any map a database containing OpenStreetMap data and some utilities a * [PostgreSQL](https://www.postgresql.org/) * [PostGIS](https://postgis.net/) -* [osm2pgsql](https://github.com/openstreetmap/osm2pgsql#installing) to [import your data](https://switch2osm.org/loading-osm-data/) into a PostGIS database +* [osm2pgsql](https://github.com/openstreetmap/osm2pgsql#installing) 1.4.1 or later to import your data into a PostGIS database * Python 3 with the psycopg2, yaml, and requests libraries (`python3-psycopg2` `python3-yaml` `python3-requests` packages on Debian-derived systems) * `ogr2ogr` for loading shapefiles into the database (`gdal-bin` on Debian-derived systems) From 4448c69c4a035f5863a208b6761257079a2a4080 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sun, 7 Feb 2021 13:48:26 -0800 Subject: [PATCH 07/54] Switch from multi to new split_at in transforms --- openstreetmap-carto.lua | 2 +- scripts/lua/test.lua | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index ef9f35114e..4a2fd1e36e 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -543,7 +543,7 @@ function add_polygon(tags) local cols = split_tags(tags, columns_map.polygon) cols['layer'] = layer(tags['layer']) cols['z_order'] = z_order(tags) - cols.way = { create = 'area', multi = true } + cols.way = { create = 'area', split_at = nil } tables.polygon:add_row(cols) end diff --git a/scripts/lua/test.lua b/scripts/lua/test.lua index 661472e8ad..3de6b20fb5 100644 --- a/scripts/lua/test.lua +++ b/scripts/lua/test.lua @@ -190,15 +190,15 @@ assert(deepcompare(table_contents.planet_osm_roads[1], {tags = {foo = "bar"}, wa print("TESTING: add_polygon") table_contents.planet_osm_polygon = {} add_polygon({natural = "wood"}) -assert(deepcompare(table_contents.planet_osm_polygon[1], {natural = "wood", tags = {}, way = { create = 'area', multi=true }}), "Tag with column") +assert(deepcompare(table_contents.planet_osm_polygon[1], {natural = "wood", tags = {}, way = { create = 'area', split_at = nil }}), "Tag with column") table_contents.planet_osm_polygon = {} add_polygon({natural = "wood", foo = "bar"}) -assert(deepcompare(table_contents.planet_osm_polygon[1], {natural = "wood", tags = {foo = "bar"}, way = { create = 'area', multi=true }}), "Tag with column + hstore") +assert(deepcompare(table_contents.planet_osm_polygon[1], {natural = "wood", tags = {foo = "bar"}, way = { create = 'area', split_at = nil }}), "Tag with column + hstore") table_contents.planet_osm_polygon = {} add_polygon({foo = "bar"}) -assert(deepcompare(table_contents.planet_osm_polygon[1], {tags = {foo = "bar"}, way = { create = 'area', multi=true }}), "hstore only") +assert(deepcompare(table_contents.planet_osm_polygon[1], {tags = {foo = "bar"}, way = { create = 'area', split_at = nil }}), "hstore only") print("TESTING: add_route") From 8c06ed9122877ea888a5093df62e918b7d21f174 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sun, 7 Feb 2021 14:04:26 -0800 Subject: [PATCH 08/54] Correctly split tags for route_member table The code was splitting based on columns for the wrong table. --- openstreetmap-carto.lua | 2 +- scripts/lua/test.lua | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 4a2fd1e36e..78dc79adc4 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -550,7 +550,7 @@ end function add_route(object) for i, member in ipairs(object.members) do if member.type == 'w' then - local cols = split_tags(object.tags, columns_map.roads) + local cols = split_tags(object.tags, columns_map.route) cols.member_id = member.ref cols.member_position = i tables.route:add_row(cols) diff --git a/scripts/lua/test.lua b/scripts/lua/test.lua index 3de6b20fb5..060738f7a7 100644 --- a/scripts/lua/test.lua +++ b/scripts/lua/test.lua @@ -215,6 +215,9 @@ add_route({tags = {route = "unicycle"}, members = {{type = 'w', ref = 1234, role assert(deepcompare(table_contents.planet_osm_route[1], {route = "unicycle", member_id = 1234, member_position = 1, tags = {}}), "Route with 2 ways") assert(deepcompare(table_contents.planet_osm_route[2], {route = "unicycle", member_id = 1235, member_position = 2, tags = {}}), "Route with 2 ways") +table_contents.planet_osm_route = {} +add_route({tags = {route = "unicycle", network = "clown"}, members = {{type = 'w', ref = 1234, role = 'foo'}}}) +assert(deepcompare(table_contents.planet_osm_route[1], {route = "unicycle", network="clown", member_id = 1234, member_position = 1, tags = {}}), "Route with network") print("TESTING: osm2pgsql.process_node") table_contents.planet_osm_point = {} From 9df706e15d08903fb6fc504328fcf8a45995f194 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sun, 7 Feb 2021 15:21:10 -0800 Subject: [PATCH 09/54] Add tests for layer function --- scripts/lua/test.lua | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/lua/test.lua b/scripts/lua/test.lua index 060738f7a7..1730758de8 100644 --- a/scripts/lua/test.lua +++ b/scripts/lua/test.lua @@ -129,6 +129,14 @@ assert(not isarea({building = "no"}), "test failed: building=no") assert(isarea({building = "no", area = "yes"}), "test failed: building=no with area tag") assert(isarea({building = "no", landuse = "forest"}), "test failed: building=no with other area tag") +print("TESTING: layer") +assert(layer("foo") == nil, "non-numeric layer") +assert(layer("0") == "0", "0 layer") +assert(layer("3") == "3", "3 layer") +assert(layer("-3") == "-3", "-3 layer") +assert(layer("300") == nil, "large layer") +assert(layer("3.5") == nil, "non-integer layer") + print("TESTING: clean_tags") assert(clean_tags({}), "Untagged") From 25421a4c8c29db8a8b92a162bade0d1fcfe20ae8 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sat, 26 Jun 2021 09:34:20 -0700 Subject: [PATCH 10/54] Add an admin table This is an osm2pgsql phase 2 table, so we need to keep around the IDs of all ways that are members of admin relations, and then in phase 2, add those ways to the admin table. --- openstreetmap-carto.lua | 66 +++++++++++++++++---- scripts/lua/test.lua | 126 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 180 insertions(+), 12 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 78dc79adc4..4c7bdfb9d0 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -122,6 +122,11 @@ col_definitions = { { column = 'member_id', type = 'int8' }, { column = 'member_position', type = 'int4' }, { column = 'tags', type = 'hstore' } + }, + admin = { + { column = 'way', type = 'linestring' }, + { column = 'admin_level', type = 'int2' }, + { column = 'multiple_relations', type = 'boolean' } } } @@ -166,6 +171,12 @@ tables.route = osm2pgsql.define_table{ columns = col_definitions.route } +tables.admin = osm2pgsql.define_table{ + name = 'planet_osm_admin', + ids = { type = 'way', id_column = 'osm_id' }, + columns = col_definitions.admin +} + -- Objects with any of the following keys will be treated as polygon local polygon_keys = { 'abandoned:aeroway', @@ -476,6 +487,10 @@ function layer (v) return v and string.find(v, "^-?%d+$") and tonumber(v) < 100 and tonumber(v) > -100 and v or nil end +function admin_level (v) + return v and string.find(v, "^-?%d+$") and tonumber(v) < 100 and tonumber(v) > 0 and v or nil +end + --- Clean tags of deleted tags -- @return True if no tags are left after cleaning function clean_tags(tags) @@ -515,6 +530,7 @@ function split_tags(tags, tag_map) return cols end +phase2_admin_ways = {} -- TODO: Make add_* take object, not object.tags function add_point(tags) @@ -567,18 +583,26 @@ function osm2pgsql.process_node(object) end function osm2pgsql.process_way(object) - if clean_tags(object.tags) then - return - end + if osm2pgsql.stage == 1 then + if clean_tags(object.tags) then + return + end - local area_tags = isarea(object.tags) - if object.is_closed and area_tags then - add_polygon(object.tags) - else - add_line(object.tags) + local area_tags = isarea(object.tags) + if object.is_closed and area_tags then + add_polygon(object.tags) + else + add_line(object.tags) - if roads(object.tags) then - add_roads(object.tags) + if roads(object.tags) then + add_roads(object.tags) + end + end + elseif osm2pgsql.stage == 2 then + -- Stage two processing is called on ways that are part of admin boundary relations + local props = phase2_admin_ways[object.id] + if props ~= nil then + tables.admin:add_row({admin_level = props.level, multiple_relations = (props.parents > 1), geom = { create = 'line' }}) end end end @@ -611,3 +635,25 @@ function osm2pgsql.process_relation(object) end end end + +function osm2pgsql.select_relation_members(relation) + if relation.tags.type == 'boundary' + and relation.tags.boundary == 'administrative' then + local admin = admin_level(relation.tags.admin_level) + if admin ~= nil then + for _, ref in ipairs(osm2pgsql.way_member_ids(relation)) do + -- Store the lowest admin_level, and how many relations it used in + if phase2_admin_ways[ref] == nil then + phase2_admin_ways[ref] = {level = admin, parents = 1} + else + if phase2_admin_ways[ref].level == admin then + phase2_admin_ways[ref].parents = phase2_admin_ways[ref].parents + 1 + elseif admin < phase2_admin_ways[ref].level then + phase2_admin_ways[ref] = {level = admin, parents = 1} + end + end + end + return { ways = osm2pgsql.way_member_ids(relation) } + end + end +end diff --git a/scripts/lua/test.lua b/scripts/lua/test.lua index 1730758de8..ea3f1fc1ec 100644 --- a/scripts/lua/test.lua +++ b/scripts/lua/test.lua @@ -24,9 +24,25 @@ function deepcompare(t1,t2) return true end --- Before testing we need to mock the supplied osm2pgsql object -osm2pgsql = { srid = 3857 } +--- See http://lua-users.org/wiki/CopyTable +function deepcopy(orig) + local orig_type = type(orig) + local copy + if orig_type == 'table' then + copy = {} + for orig_key, orig_value in next, orig, nil do + copy[deepcopy(orig_key)] = deepcopy(orig_value) + end + setmetatable(copy, deepcopy(getmetatable(orig))) + else -- number, string, boolean, etc + copy = orig + end + return copy +end + +-- Before testing we need to mock the supplied osm2pgsql object +osm2pgsql = { srid = 3857, stage = 1 } -- local table_definitions = {} local table_contents = {} @@ -37,6 +53,17 @@ function osm2pgsql.define_table(definition) return {add_row = function(self, obj) table.insert(table_contents[definition.name], obj) end} end +function osm2pgsql.way_member_ids(relation) + local members = {} + if relation.members ~= nil then + for _, member in ipairs(relation.members) do + if member.type == 'w' then + table.insert(members, member.ref) + end + end + end + return members +end require ("openstreetmap-carto") print("TESTING: define_table") @@ -75,6 +102,10 @@ assert(deepcompare(table_definitions.planet_osm_route.columns[2], { column = 'me assert(deepcompare(table_definitions.planet_osm_route.columns[3], { column = 'tags', type = 'hstore' }), "planet_osm_route tags column") assert(deepcompare(table_definitions.planet_osm_route.columns[4], { column = 'route', type = 'text' }), "planet_osm_route route column") +assert(deepcompare(table_definitions.planet_osm_admin.ids, { type = 'way', id_column = 'osm_id' }), "planet_osm_admin id column") +assert(deepcompare(table_definitions.planet_osm_admin.columns[1], { column = 'way', type = 'linestring' }), "planet_osm_admin way column") +assert(deepcompare(table_definitions.planet_osm_admin.columns[2], { column = 'admin_level', type = 'int2' }), "planet_osm_admin admin_level column") +assert(deepcompare(table_definitions.planet_osm_admin.columns[3], { column = 'multiple_relations', type = 'boolean' }), "planet_osm_admin multiple_relations column") print("TESTING: z_order") @@ -137,6 +168,14 @@ assert(layer("-3") == "-3", "-3 layer") assert(layer("300") == nil, "large layer") assert(layer("3.5") == nil, "non-integer layer") +print("TESTING: admin_level") +assert(admin_level("foo") == nil, "non-numeric admin_level") +assert(admin_level("0") == nil, "0 admin_level") +assert(admin_level("3") == "3", "3 admin_level") +assert(admin_level("-3") == nil, "-3 admin_level") +assert(admin_level("300") == nil, "large admin_level") +assert(admin_level("3.5") == nil, "non-integer admin_level") + print("TESTING: clean_tags") assert(clean_tags({}), "Untagged") @@ -343,3 +382,86 @@ assert(deepcompare(table_contents.planet_osm_route[1], table_contents.planet_osm table_contents.planet_osm_line = {} table_contents.planet_osm_polygon = {} table_contents.planet_osm_roads = {} + +-- Testing of phase 2 + +--[[ + This sets up an + 1. admin_level=2 relation with ways 1, 2, 3, 5, 7, 8 + 2. admin_level=2 relation with ways 1, 2, 6, 7, 8 + 3. admin_level=4 relation with ways 1, 3, 4 + + The ways have correct tags except for 7 and 8 +]] + +-- Currently in phase 1 + +local test_ways = { + { id = 1, tags = { boundary = "administrative", admin_level = "2"} }, + { id = 2, tags = { boundary = "administrative", admin_level = "2"} }, + { id = 3, tags = { boundary = "administrative", admin_level = "2"} }, + { id = 4, tags = { boundary = "administrative", admin_level = "4"} }, + { id = 5, tags = { boundary = "administrative", admin_level = "2"} }, + { id = 6, tags = { boundary = "administrative", admin_level = "2"} }, + { id = 7, tags = { boundary = "administrative", admin_level = "3"} }, -- incorrect tags + { id = 8, tags = { } } -- incorrect tags +} +-- add another way that isn't part of a relation +local test_relations = { + { id = 1, + tags = {type = "boundary", boundary = "administrative", admin_level = "2"}, + members = { {type = 'w', ref = 1, role = "outer"}, + {type = 'w', ref = 2, role = "outer"}, + {type = 'w', ref = 3, role = "outer"}, + {type = 'w', ref = 5, role = "outer"}, + {type = 'w', ref = 7, role = "outer"}, + {type = 'w', ref = 8, role = "outer"} } }, + { id = 2, tags = {type = "boundary", boundary = "administrative", admin_level = "2"}, + members = { {type = 'w', ref = 1, role = "outer"}, + {type = 'w', ref = 2, role = "outer"}, + {type = 'w', ref = 6, role = "outer"}, + {type = 'w', ref = 7, role = "outer"}, + {type = 'w', ref = 8, role = "outer"} } }, + { id = 3, tags = {type = "boundary", boundary = "administrative", admin_level = "4"}, + members = { {type = 'w', ref = 1, role = "outer"}, + {type = 'w', ref = 3, role = "outer"}, + {type = 'w', ref = 4, role = "outer"} } } +} +for _, way in ipairs(test_ways) do + osm2pgsql.process_way(way) +end + +local pending_ways = {} + +function table.clone(org) + return {table.unpack(org)} +end + +for _, relation in ipairs(test_relations) do + osm2pgsql.process_relation(deepcopy(relation)) + local ret = osm2pgsql.select_relation_members(relation) + if ret ~= nil then + for _, ref in ipairs(ret.ways) do + pending_ways[ref] = true + end + end +end + +osm2pgsql.stage = 2 + +for _, way in ipairs(test_ways) do + if pending_ways[way.id] then + osm2pgsql.process_way(way) + end +end + +-- Because everything is done in a fixed order we can use that to figure out which row is which. This is important because the add_row method doesn't take in the osm_id, and osm2pgsql tracks it separately + +assert(deepcompare(table_contents.planet_osm_admin[1], {admin_level = "2", multiple_relations = true, geom = {create = "line" } }), "row 1") +assert(deepcompare(table_contents.planet_osm_admin[2], {admin_level = "2", multiple_relations = true, geom = {create = "line" } }), "row 2") +assert(deepcompare(table_contents.planet_osm_admin[3], {admin_level = "2", multiple_relations = false, geom = {create = "line" } }), "row 3") +assert(deepcompare(table_contents.planet_osm_admin[4], {admin_level = "4", multiple_relations = false, geom = {create = "line" } }), "row 4") +assert(deepcompare(table_contents.planet_osm_admin[5], {admin_level = "2", multiple_relations = false, geom = {create = "line" } }), "row 5") +assert(deepcompare(table_contents.planet_osm_admin[6], {admin_level = "2", multiple_relations = false, geom = {create = "line" } }), "row 6") +assert(deepcompare(table_contents.planet_osm_admin[7], {admin_level = "2", multiple_relations = true, geom = {create = "line" } }), "row 7") +assert(deepcompare(table_contents.planet_osm_admin[8], {admin_level = "2", multiple_relations = true, geom = {create = "line" } }), "row 8") From 05005ee576e1b54fc2da7744e44cc7f7a2c391b3 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sat, 26 Jun 2021 16:12:21 -0700 Subject: [PATCH 11/54] Compare admin levels as numbers This ifxes a bug where admin level 10 would be treated as more important than admin level 4. --- openstreetmap-carto.lua | 2 +- scripts/lua/test.lua | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 4c7bdfb9d0..9fef3a4c6e 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -639,7 +639,7 @@ end function osm2pgsql.select_relation_members(relation) if relation.tags.type == 'boundary' and relation.tags.boundary == 'administrative' then - local admin = admin_level(relation.tags.admin_level) + local admin = tonumber(admin_level(relation.tags.admin_level)) if admin ~= nil then for _, ref in ipairs(osm2pgsql.way_member_ids(relation)) do -- Store the lowest admin_level, and how many relations it used in diff --git a/scripts/lua/test.lua b/scripts/lua/test.lua index ea3f1fc1ec..e0eaac8fd8 100644 --- a/scripts/lua/test.lua +++ b/scripts/lua/test.lua @@ -457,11 +457,11 @@ end -- Because everything is done in a fixed order we can use that to figure out which row is which. This is important because the add_row method doesn't take in the osm_id, and osm2pgsql tracks it separately -assert(deepcompare(table_contents.planet_osm_admin[1], {admin_level = "2", multiple_relations = true, geom = {create = "line" } }), "row 1") -assert(deepcompare(table_contents.planet_osm_admin[2], {admin_level = "2", multiple_relations = true, geom = {create = "line" } }), "row 2") -assert(deepcompare(table_contents.planet_osm_admin[3], {admin_level = "2", multiple_relations = false, geom = {create = "line" } }), "row 3") -assert(deepcompare(table_contents.planet_osm_admin[4], {admin_level = "4", multiple_relations = false, geom = {create = "line" } }), "row 4") -assert(deepcompare(table_contents.planet_osm_admin[5], {admin_level = "2", multiple_relations = false, geom = {create = "line" } }), "row 5") -assert(deepcompare(table_contents.planet_osm_admin[6], {admin_level = "2", multiple_relations = false, geom = {create = "line" } }), "row 6") -assert(deepcompare(table_contents.planet_osm_admin[7], {admin_level = "2", multiple_relations = true, geom = {create = "line" } }), "row 7") -assert(deepcompare(table_contents.planet_osm_admin[8], {admin_level = "2", multiple_relations = true, geom = {create = "line" } }), "row 8") +assert(deepcompare(table_contents.planet_osm_admin[1], {admin_level = 2, multiple_relations = true, geom = {create = "line" } }), "row 1") +assert(deepcompare(table_contents.planet_osm_admin[2], {admin_level = 2, multiple_relations = true, geom = {create = "line" } }), "row 2") +assert(deepcompare(table_contents.planet_osm_admin[3], {admin_level = 2, multiple_relations = false, geom = {create = "line" } }), "row 3") +assert(deepcompare(table_contents.planet_osm_admin[4], {admin_level = 4, multiple_relations = false, geom = {create = "line" } }), "row 4") +assert(deepcompare(table_contents.planet_osm_admin[5], {admin_level = 2, multiple_relations = false, geom = {create = "line" } }), "row 5") +assert(deepcompare(table_contents.planet_osm_admin[6], {admin_level = 2, multiple_relations = false, geom = {create = "line" } }), "row 6") +assert(deepcompare(table_contents.planet_osm_admin[7], {admin_level = 2, multiple_relations = true, geom = {create = "line" } }), "row 7") +assert(deepcompare(table_contents.planet_osm_admin[8], {admin_level = 2, multiple_relations = true, geom = {create = "line" } }), "row 8") From e491c0acfb0ab50295a8802fc807afee2abe2a42 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sun, 27 Jun 2021 20:33:43 -0700 Subject: [PATCH 12/54] Add a transport_line table This table contains linear transportation features, which will allow z_order and layer to be removed from the line table. --- openstreetmap-carto.lua | 44 ++++++++++++++++++++++++++++++++++++++++- scripts/lua/test.lua | 42 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 9fef3a4c6e..dfa0b19bfa 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -82,6 +82,24 @@ local pg_cols = { 'water', 'waterway' }, + transport_line = { + 'access', + 'bicycle', + 'bridge', + 'construction', + 'covered', + 'foot', + 'highway', + 'horse', + 'name', + 'oneway', + 'railway', + 'ref', + 'service', + 'surface', + 'tracktype', + 'tunnel' + }, route = { 'route', 'ref', @@ -105,6 +123,12 @@ col_definitions = { { column = 'layer', type = 'int4' }, { column = 'z_order', type = 'int4' } }, + transport_line = { + { column = 'way', type = 'linestring' }, + { column = 'tags', type = 'hstore' }, + { column = 'layer', type = 'int4' }, + { column = 'z_order', type = 'int4' } + }, roads = { { column = 'way', type = 'linestring' }, { column = 'tags', type = 'hstore' }, @@ -153,6 +177,12 @@ tables.line = osm2pgsql.define_table{ columns = col_definitions.line } +tables.transport_line = osm2pgsql.define_table{ + name = 'planet_osm_transport_line', + ids = { type = 'way', id_column = 'osm_id' }, + columns = col_definitions.transport_line +} + tables.roads = osm2pgsql.define_table{ name = 'planet_osm_roads', ids = { type = 'way', id_column = 'osm_id' }, @@ -457,7 +487,7 @@ end --- Check if an object with given tags should be treated as polygon -- @param tags OSM tags --- @return 1 if area, 0 if linear +-- @return true if area, false if linear function isarea (tags) -- Treat objects tagged as area=yes polygon, other area as no if tags["area"] then @@ -547,6 +577,14 @@ function add_line(tags) tables.line:add_row(cols) end +function add_transport_line(tags) + local cols = split_tags(tags, columns_map.line) + cols['layer'] = layer(tags['layer']) + cols['z_order'] = z_order(tags) + cols.way = { create = 'line', split_at = 100000 } + tables.transport_line:add_row(cols) +end + function add_roads(tags) local cols = split_tags(tags, columns_map.roads) cols['layer'] = layer(tags['layer']) @@ -594,6 +632,10 @@ function osm2pgsql.process_way(object) else add_line(object.tags) + if z_order(object.tags) ~= nil then + add_transport_line(object.tags) + end + if roads(object.tags) then add_roads(object.tags) end diff --git a/scripts/lua/test.lua b/scripts/lua/test.lua index e0eaac8fd8..bdf725d353 100644 --- a/scripts/lua/test.lua +++ b/scripts/lua/test.lua @@ -81,6 +81,13 @@ assert(deepcompare(table_definitions.planet_osm_line.columns[3], { column = 'lay assert(deepcompare(table_definitions.planet_osm_line.columns[4], { column = 'z_order', type = 'int4' }), "planet_osm_line z_order column") assert(deepcompare(table_definitions.planet_osm_line.columns[5], { column = 'access', type = 'text' }), "planet_osm_line access column") +assert(deepcompare(table_definitions.planet_osm_transport_line.ids, { type = 'way', id_column = 'osm_id' }), "planet_osm_transport_line id column") +assert(deepcompare(table_definitions.planet_osm_transport_line.columns[1], { column = 'way', type = 'linestring' }), "planet_osm_transport_line way column") +assert(deepcompare(table_definitions.planet_osm_transport_line.columns[2], { column = 'tags', type = 'hstore' }), "planet_osm_transport_line tags column") +assert(deepcompare(table_definitions.planet_osm_transport_line.columns[3], { column = 'layer', type = 'int4' }), "planet_osm_transport_line layer column") +assert(deepcompare(table_definitions.planet_osm_transport_line.columns[4], { column = 'z_order', type = 'int4' }), "planet_osm_transport_line z_order column") +assert(deepcompare(table_definitions.planet_osm_transport_line.columns[5], { column = 'access', type = 'text' }), "planet_osm_transport_line access column") + assert(deepcompare(table_definitions.planet_osm_roads.ids, { type = 'way', id_column = 'osm_id' }), "planet_osm_roads id column") assert(deepcompare(table_definitions.planet_osm_roads.columns[1], { column = 'way', type = 'linestring' }), "planet_osm_roads way column") assert(deepcompare(table_definitions.planet_osm_roads.columns[2], { column = 'tags', type = 'hstore' }), "planet_osm_roads tags column") @@ -221,6 +228,20 @@ table_contents.planet_osm_line = {} add_line({foo = "bar"}) assert(deepcompare(table_contents.planet_osm_line[1], {tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "hstore only") +print("TESTING: add_transport_line") +table_contents.planet_osm_transport_line = {} +add_transport_line({highway = "road"}) +assert(deepcompare(table_contents.planet_osm_transport_line[1], {highway = "road", z_order = 330, tags = {}, way = { create = 'line', split_at = 100000 }}), "Tag with column") + +table_contents.planet_osm_transport_line = {} +add_transport_line({highway = "road", foo = "bar"}) +assert(deepcompare(table_contents.planet_osm_transport_line[1], {highway = "road", z_order = 330, tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "Tag with column + hstore") + +-- The z_order check where add_transport_line is called prevents this case from being hit in practice +table_contents.planet_osm_transport_line = {} +add_transport_line({foo = "bar"}) +assert(deepcompare(table_contents.planet_osm_transport_line[1], {tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "hstore only") + print("TESTING: add_roads") table_contents.planet_osm_roads = {} add_roads({highway = "road"}) @@ -282,38 +303,49 @@ assert(deepcompare(table_contents.planet_osm_point[1], table_contents.planet_osm print("TESTING: osm2pgsql.process_way") table_contents.planet_osm_line = {} +table_contents.planet_osm_transport_line = {} table_contents.planet_osm_roads = {} table_contents.planet_osm_polygon = {} osm2pgsql.process_way({tags = {}}) assert(deepcompare(table_contents.planet_osm_line, {}), "Untagged line") +assert(deepcompare(table_contents.planet_osm_transport_line, {}), "Untagged transport_line") assert(deepcompare(table_contents.planet_osm_roads, {}), "Untagged roads") assert(deepcompare(table_contents.planet_osm_polygon, {}), "Untagged polygon") osm2pgsql.process_way({tags = {odbl = "yes"}}) assert(deepcompare(table_contents.planet_osm_line, {}), "Deleted tag line") +assert(deepcompare(table_contents.planet_osm_transport_line, {}), "Deleted tag transport_line") assert(deepcompare(table_contents.planet_osm_roads, {}), "Deleted tag roads") assert(deepcompare(table_contents.planet_osm_polygon, {}), "Deleted tag polygon") osm2pgsql.process_way({is_closed = false, tags = { highway = "road"}}) +-- Add something to compare against. We know add_line and add_transport_line are okay since we tested them above add_line({ highway = "road"}) +add_transport_line({ highway = "road"}) assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_line[2]), "Line tag, open way, line table") +assert(deepcompare(table_contents.planet_osm_transport_line[1], table_contents.planet_osm_transport_line[2]), "Line tag, open way, transport_line table") assert(deepcompare(table_contents.planet_osm_roads, {}), "Line tag, open way, roads table") assert(deepcompare(table_contents.planet_osm_polygon, {}), "Line tag, open way, polygon table") table_contents.planet_osm_line = {} +table_contents.planet_osm_transport_line = {} osm2pgsql.process_way({is_closed = true, tags = { highway = "motorway"}}) add_line({highway = "motorway"}) +add_transport_line({ highway = "motorway"}) add_roads({highway = "motorway"}) assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_line[2]), "Line tag, closed way, line table") -assert(deepcompare(table_contents.planet_osm_roads[1], table_contents.planet_osm_roads[2]), "Line tag, closed way, line table") +assert(deepcompare(table_contents.planet_osm_transport_line[1], table_contents.planet_osm_transport_line[2]), "Line tag, closed way, transport_line table") +assert(deepcompare(table_contents.planet_osm_roads[1], table_contents.planet_osm_roads[2]), "Line tag, closed way, roads table") assert(deepcompare(table_contents.planet_osm_polygon, {}), "Line tag, closed way, polygon table") table_contents.planet_osm_line = {} +table_contents.planet_osm_transport_line = {} table_contents.planet_osm_roads = {} osm2pgsql.process_way({is_closed = false, tags = { natural = "wood"}}) add_line({natural = "wood"}) assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_line[1]), "area tag, open way, line table") +assert(deepcompare(table_contents.planet_osm_transport_line, {}), "area tag, open way, transport_line table") assert(deepcompare(table_contents.planet_osm_roads, {}), "area tag, open way, roads table") assert(deepcompare(table_contents.planet_osm_polygon, {}), "area tag, open way, polygon table") table_contents.planet_osm_line = {} @@ -321,6 +353,7 @@ table_contents.planet_osm_line = {} osm2pgsql.process_way({is_closed = true, tags = { natural = "wood"}}) add_polygon({natural = "wood"}) assert(deepcompare(table_contents.planet_osm_line, {}), "area tag, closed way, line table") +assert(deepcompare(table_contents.planet_osm_transport_line, {}), "area tag, closed way, transport_line table") assert(deepcompare(table_contents.planet_osm_roads, {}), "area tag, closed way, roads table") assert(deepcompare(table_contents.planet_osm_polygon[1], table_contents.planet_osm_polygon[2]), "area tag, closed way, polygon table") @@ -332,16 +365,19 @@ table_contents.planet_osm_route = {} osm2pgsql.process_relation({tags = {}}) assert(deepcompare(table_contents.planet_osm_line, {}), "Untagged line") +assert(deepcompare(table_contents.planet_osm_transport_line, {}), "Untagged transport_line") assert(deepcompare(table_contents.planet_osm_roads, {}), "Untagged roads") assert(deepcompare(table_contents.planet_osm_polygon, {}), "Untagged polygon") osm2pgsql.process_relation({tags = {odbl = "yes"}}) assert(deepcompare(table_contents.planet_osm_line, {}), "Deleted tag line") +assert(deepcompare(table_contents.planet_osm_transport_line, {}), "Deleted tag transport_line") assert(deepcompare(table_contents.planet_osm_roads, {}), "Deleted tag roads") assert(deepcompare(table_contents.planet_osm_polygon, {}), "Deleted tag polygon") osm2pgsql.process_relation({tags = {type = "multipolygon", odbl = "yes"}}) assert(deepcompare(table_contents.planet_osm_line, {}), "Deleted tag line with type") +assert(deepcompare(table_contents.planet_osm_transport_line, {}), "Deleted tag transport_line with type") assert(deepcompare(table_contents.planet_osm_roads, {}), "Deleted tag roads with type") assert(deepcompare(table_contents.planet_osm_polygon, {}), "Deleted tag polygon with type") @@ -349,6 +385,7 @@ osm2pgsql.process_relation({tags = {type = "boundary", boundary = "foo"}}) add_line({boundary = "foo"}) add_polygon({boundary = "foo"}) assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_line[2]), "Boundary line") +assert(deepcompare(table_contents.planet_osm_transport_line, {}), "Boundary transport_line") assert(deepcompare(table_contents.planet_osm_roads, {}), "Boundary roads") assert(deepcompare(table_contents.planet_osm_polygon[1], table_contents.planet_osm_polygon[2]), "Boundary polygon") table_contents.planet_osm_line = {} @@ -359,6 +396,7 @@ add_line({boundary = "administrative"}) add_roads({boundary = "administrative"}) add_polygon({boundary = "administrative"}) assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_line[2]), "admin boundary line") +assert(deepcompare(table_contents.planet_osm_transport_line, {}), "admin boundary transport_line") assert(deepcompare(table_contents.planet_osm_roads[1], table_contents.planet_osm_roads[2]), "admin boundary roads") assert(deepcompare(table_contents.planet_osm_polygon[1], table_contents.planet_osm_polygon[2]), "admin boundary polygon") table_contents.planet_osm_line = {} @@ -368,6 +406,7 @@ table_contents.planet_osm_roads = {} osm2pgsql.process_relation({tags = {type = "multipolygon", natural = "tree"}}) add_polygon({natural = "tree"}) assert(deepcompare(table_contents.planet_osm_line, {}), "MP line") +assert(deepcompare(table_contents.planet_osm_transport_line, {}), "MP transport_line") assert(deepcompare(table_contents.planet_osm_roads, {}), "MP roads") assert(deepcompare(table_contents.planet_osm_polygon[1], table_contents.planet_osm_polygon[2]), "admin boundary polygon") table_contents.planet_osm_polygon = {} @@ -376,6 +415,7 @@ osm2pgsql.process_relation({tags = {type = "route", route = "road"}, members = { add_line({route = "road"}) add_route({tags = { route = "road"}, members = {{type = 'w', ref = 1234, role = 'forward'}}}) assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_line[2]), "route line") +assert(deepcompare(table_contents.planet_osm_transport_line, {}), "route transport_line") assert(deepcompare(table_contents.planet_osm_roads, {}), "route roads") assert(deepcompare(table_contents.planet_osm_polygon, {}), "route polygon") assert(deepcompare(table_contents.planet_osm_route[1], table_contents.planet_osm_route[2]), "route route") From 2e47c05d74c59fb28cee0bf5bdb77310a56d5eec Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Mon, 28 Jun 2021 02:43:56 -0700 Subject: [PATCH 13/54] Add a transport_polygon table This is a table for transport features (i.e. those with z_order) which will allow the removal of z_order from the general polygon table --- openstreetmap-carto.lua | 47 +++++++++++++++++++++++++++++++ scripts/lua/test.lua | 62 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 107 insertions(+), 2 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index dfa0b19bfa..ce8b013d20 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -100,6 +100,24 @@ local pg_cols = { 'tracktype', 'tunnel' }, + transport_polygon = { + 'access', + 'bicycle', + 'bridge', + 'construction', + 'covered', + 'foot', + 'highway', + 'horse', + 'name', + 'oneway', + 'railway', + 'ref', + 'service', + 'surface', + 'tracktype', + 'tunnel' + }, route = { 'route', 'ref', @@ -129,6 +147,13 @@ col_definitions = { { column = 'layer', type = 'int4' }, { column = 'z_order', type = 'int4' } }, + transport_polygon = { + { column = 'way', type = 'geometry' }, + { column = 'tags', type = 'hstore' }, + { column = 'layer', type = 'int4' }, + { column = 'z_order', type = 'int4' }, + { column = 'way_area', type = 'area' } + }, roads = { { column = 'way', type = 'linestring' }, { column = 'tags', type = 'hstore' }, @@ -195,6 +220,12 @@ tables.polygon = osm2pgsql.define_table{ columns = col_definitions.polygon } +tables.transport_polygon = osm2pgsql.define_table{ + name = 'planet_osm_transport_polygon', + ids = { type = 'way', id_column = 'osm_id' }, + columns = col_definitions.transport_polygon +} + tables.route = osm2pgsql.define_table{ name = 'planet_osm_route', ids = { type = 'relation', id_column = 'osm_id' }, @@ -601,6 +632,14 @@ function add_polygon(tags) tables.polygon:add_row(cols) end +function add_transport_polygon(tags) + local cols = split_tags(tags, columns_map.polygon) + cols['layer'] = layer(tags['layer']) + cols['z_order'] = z_order(tags) + cols.way = { create = 'area', split_at = nil } + tables.transport_polygon:add_row(cols) +end + function add_route(object) for i, member in ipairs(object.members) do if member.type == 'w' then @@ -629,6 +668,10 @@ function osm2pgsql.process_way(object) local area_tags = isarea(object.tags) if object.is_closed and area_tags then add_polygon(object.tags) + + if z_order(object.tags) ~= nil then + add_transport_polygon(object.tags) + end else add_line(object.tags) @@ -668,6 +711,10 @@ function osm2pgsql.process_relation(object) elseif type == "multipolygon" then add_polygon(object.tags) + + if z_order(object.tags) ~= nil then + add_transport_polygon(object.tags) + end elseif type == "route" then add_line(object.tags) add_route(object) diff --git a/scripts/lua/test.lua b/scripts/lua/test.lua index bdf725d353..9c5efee2dd 100644 --- a/scripts/lua/test.lua +++ b/scripts/lua/test.lua @@ -103,6 +103,14 @@ assert(deepcompare(table_definitions.planet_osm_polygon.columns[4], { column = ' assert(deepcompare(table_definitions.planet_osm_polygon.columns[5], { column = 'way_area', type = 'area' }), "planet_osm_polygon way_area column") assert(deepcompare(table_definitions.planet_osm_polygon.columns[6], { column = 'access', type = 'text' }), "planet_osm_polygon access column") +assert(deepcompare(table_definitions.planet_osm_transport_polygon.ids, { type = 'way', id_column = 'osm_id' }), "planet_osm_transport_polygon id column") +assert(deepcompare(table_definitions.planet_osm_transport_polygon.columns[1], { column = 'way', type = 'geometry' }), "planet_osm_transport_polygon way column") +assert(deepcompare(table_definitions.planet_osm_transport_polygon.columns[2], { column = 'tags', type = 'hstore' }), "planet_osm_transport_polygon tags column") +assert(deepcompare(table_definitions.planet_osm_transport_polygon.columns[3], { column = 'layer', type = 'int4' }), "planet_osm_transport_polygon layer column") +assert(deepcompare(table_definitions.planet_osm_transport_polygon.columns[4], { column = 'z_order', type = 'int4' }), "planet_osm_transport_polygon z_order column") +assert(deepcompare(table_definitions.planet_osm_transport_polygon.columns[5], { column = 'way_area', type = 'area' }), "planet_osm_transport_polygon way_area column") +assert(deepcompare(table_definitions.planet_osm_transport_polygon.columns[6], { column = 'access', type = 'text' }), "planet_osm_transport_polygon access column") + assert(deepcompare(table_definitions.planet_osm_route.ids, { type = 'relation', id_column = 'osm_id' }), "planet_osm_route id column") assert(deepcompare(table_definitions.planet_osm_route.columns[1], { column = 'member_id', type = 'int8' }), "planet_osm_route member_id column") assert(deepcompare(table_definitions.planet_osm_route.columns[2], { column = 'member_position', type = 'int4' }), "planet_osm_route member_position column") @@ -268,6 +276,18 @@ table_contents.planet_osm_polygon = {} add_polygon({foo = "bar"}) assert(deepcompare(table_contents.planet_osm_polygon[1], {tags = {foo = "bar"}, way = { create = 'area', split_at = nil }}), "hstore only") +print("TESTING: add_transport_polygon") +table_contents.planet_osm_transport_polygon = {} +add_transport_polygon({natural = "wood"}) +assert(deepcompare(table_contents.planet_osm_transport_polygon[1], {natural = "wood", tags = {}, way = { create = 'area', split_at = nil }}), "Tag with column") + +table_contents.planet_osm_transport_polygon = {} +add_transport_polygon({natural = "wood", foo = "bar"}) +assert(deepcompare(table_contents.planet_osm_transport_polygon[1], {natural = "wood", tags = {foo = "bar"}, way = { create = 'area', split_at = nil }}), "Tag with column + hstore") + +table_contents.planet_osm_transport_polygon = {} +add_transport_polygon({foo = "bar"}) +assert(deepcompare(table_contents.planet_osm_transport_polygon[1], {tags = {foo = "bar"}, way = { create = 'area', split_at = nil }}), "hstore only") print("TESTING: add_route") table_contents.planet_osm_route = {} @@ -296,7 +316,8 @@ osm2pgsql.process_node({tags = {odbl = "yes"}}) assert(deepcompare(table_contents.planet_osm_point, {}), "Deleted tag") -- By running process_node then manually running add_point the first and second entry --- in the output tables should be the same. +-- in the output tables should be the same. A cleaner way would be to mock the add_* functions +-- and then check if they are called. osm2pgsql.process_node({tags = {natural = "tree", foo = "bar"}}) add_point({natural = "tree", foo = "bar"}) assert(deepcompare(table_contents.planet_osm_point[1], table_contents.planet_osm_point[2]), "Accepted tag") @@ -306,18 +327,21 @@ table_contents.planet_osm_line = {} table_contents.planet_osm_transport_line = {} table_contents.planet_osm_roads = {} table_contents.planet_osm_polygon = {} +table_contents.planet_osm_transport_polygon = {} osm2pgsql.process_way({tags = {}}) assert(deepcompare(table_contents.planet_osm_line, {}), "Untagged line") assert(deepcompare(table_contents.planet_osm_transport_line, {}), "Untagged transport_line") assert(deepcompare(table_contents.planet_osm_roads, {}), "Untagged roads") assert(deepcompare(table_contents.planet_osm_polygon, {}), "Untagged polygon") +assert(deepcompare(table_contents.planet_osm_transport_polygon, {}), "Untagged transport_polygon") osm2pgsql.process_way({tags = {odbl = "yes"}}) assert(deepcompare(table_contents.planet_osm_line, {}), "Deleted tag line") assert(deepcompare(table_contents.planet_osm_transport_line, {}), "Deleted tag transport_line") assert(deepcompare(table_contents.planet_osm_roads, {}), "Deleted tag roads") assert(deepcompare(table_contents.planet_osm_polygon, {}), "Deleted tag polygon") +assert(deepcompare(table_contents.planet_osm_transport_polygon, {}), "Deleted tag transport_polygon") osm2pgsql.process_way({is_closed = false, tags = { highway = "road"}}) -- Add something to compare against. We know add_line and add_transport_line are okay since we tested them above @@ -327,6 +351,7 @@ assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_ assert(deepcompare(table_contents.planet_osm_transport_line[1], table_contents.planet_osm_transport_line[2]), "Line tag, open way, transport_line table") assert(deepcompare(table_contents.planet_osm_roads, {}), "Line tag, open way, roads table") assert(deepcompare(table_contents.planet_osm_polygon, {}), "Line tag, open way, polygon table") +assert(deepcompare(table_contents.planet_osm_transport_polygon, {}), "Line tag, open way, transport_polygon table") table_contents.planet_osm_line = {} table_contents.planet_osm_transport_line = {} @@ -338,6 +363,7 @@ assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_ assert(deepcompare(table_contents.planet_osm_transport_line[1], table_contents.planet_osm_transport_line[2]), "Line tag, closed way, transport_line table") assert(deepcompare(table_contents.planet_osm_roads[1], table_contents.planet_osm_roads[2]), "Line tag, closed way, roads table") assert(deepcompare(table_contents.planet_osm_polygon, {}), "Line tag, closed way, polygon table") +assert(deepcompare(table_contents.planet_osm_transport_polygon, {}), "Line tag, closed way, transport_polygon table") table_contents.planet_osm_line = {} table_contents.planet_osm_transport_line = {} table_contents.planet_osm_roads = {} @@ -348,6 +374,7 @@ assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_ assert(deepcompare(table_contents.planet_osm_transport_line, {}), "area tag, open way, transport_line table") assert(deepcompare(table_contents.planet_osm_roads, {}), "area tag, open way, roads table") assert(deepcompare(table_contents.planet_osm_polygon, {}), "area tag, open way, polygon table") +assert(deepcompare(table_contents.planet_osm_transport_polygon, {}), "area tag, open way, transport_polygon table") table_contents.planet_osm_line = {} osm2pgsql.process_way({is_closed = true, tags = { natural = "wood"}}) @@ -356,11 +383,24 @@ assert(deepcompare(table_contents.planet_osm_line, {}), "area tag, closed way, l assert(deepcompare(table_contents.planet_osm_transport_line, {}), "area tag, closed way, transport_line table") assert(deepcompare(table_contents.planet_osm_roads, {}), "area tag, closed way, roads table") assert(deepcompare(table_contents.planet_osm_polygon[1], table_contents.planet_osm_polygon[2]), "area tag, closed way, polygon table") +assert(deepcompare(table_contents.planet_osm_transport_polygon, {}), "area tag, closed way, transport_polygon table") +table_contents.planet_osm_polygon = {} + +osm2pgsql.process_way({is_closed = true, tags = { railway = "platform", area = "yes"}}) +add_polygon({railway = "platform", area = "yes"}) +add_transport_polygon({railway = "platform", area = "yes"}) +assert(deepcompare(table_contents.planet_osm_line, {}), "area tag, closed way, line table") +assert(deepcompare(table_contents.planet_osm_transport_line, {}), "area tag, closed way, transport_line table") +assert(deepcompare(table_contents.planet_osm_roads, {}), "area tag, closed way, roads table") +assert(deepcompare(table_contents.planet_osm_polygon[1], table_contents.planet_osm_polygon[2]), "area tag, closed way, polygon table") +assert(deepcompare(table_contents.planet_osm_transport_polygon[1], table_contents.planet_osm_transport_polygon[2]), "area tag, closed way, transport_polygon table") print("TESTING: osm2pgsql.process_relation") table_contents.planet_osm_line = {} +table_contents.planet_osm_transport_line = {} table_contents.planet_osm_roads = {} table_contents.planet_osm_polygon = {} +table_contents.planet_osm_transport_polygon = {} table_contents.planet_osm_route = {} osm2pgsql.process_relation({tags = {}}) @@ -368,18 +408,21 @@ assert(deepcompare(table_contents.planet_osm_line, {}), "Untagged line") assert(deepcompare(table_contents.planet_osm_transport_line, {}), "Untagged transport_line") assert(deepcompare(table_contents.planet_osm_roads, {}), "Untagged roads") assert(deepcompare(table_contents.planet_osm_polygon, {}), "Untagged polygon") +assert(deepcompare(table_contents.planet_osm_transport_polygon, {}), "Untagged transport_polygon") osm2pgsql.process_relation({tags = {odbl = "yes"}}) assert(deepcompare(table_contents.planet_osm_line, {}), "Deleted tag line") assert(deepcompare(table_contents.planet_osm_transport_line, {}), "Deleted tag transport_line") assert(deepcompare(table_contents.planet_osm_roads, {}), "Deleted tag roads") assert(deepcompare(table_contents.planet_osm_polygon, {}), "Deleted tag polygon") +assert(deepcompare(table_contents.planet_osm_transport_polygon, {}), "Deleted tag transport_polygon") osm2pgsql.process_relation({tags = {type = "multipolygon", odbl = "yes"}}) assert(deepcompare(table_contents.planet_osm_line, {}), "Deleted tag line with type") assert(deepcompare(table_contents.planet_osm_transport_line, {}), "Deleted tag transport_line with type") assert(deepcompare(table_contents.planet_osm_roads, {}), "Deleted tag roads with type") assert(deepcompare(table_contents.planet_osm_polygon, {}), "Deleted tag polygon with type") +assert(deepcompare(table_contents.planet_osm_transport_polygon, {}), "Deleted tag transport_polygon with type") osm2pgsql.process_relation({tags = {type = "boundary", boundary = "foo"}}) add_line({boundary = "foo"}) @@ -388,6 +431,7 @@ assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_ assert(deepcompare(table_contents.planet_osm_transport_line, {}), "Boundary transport_line") assert(deepcompare(table_contents.planet_osm_roads, {}), "Boundary roads") assert(deepcompare(table_contents.planet_osm_polygon[1], table_contents.planet_osm_polygon[2]), "Boundary polygon") +assert(deepcompare(table_contents.planet_osm_transport_polygon, {}), "Boundary transport_polygon") table_contents.planet_osm_line = {} table_contents.planet_osm_polygon = {} @@ -399,6 +443,7 @@ assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_ assert(deepcompare(table_contents.planet_osm_transport_line, {}), "admin boundary transport_line") assert(deepcompare(table_contents.planet_osm_roads[1], table_contents.planet_osm_roads[2]), "admin boundary roads") assert(deepcompare(table_contents.planet_osm_polygon[1], table_contents.planet_osm_polygon[2]), "admin boundary polygon") +assert(deepcompare(table_contents.planet_osm_transport_polygon, {}), "admin boundary transport_polygon") table_contents.planet_osm_line = {} table_contents.planet_osm_polygon = {} table_contents.planet_osm_roads = {} @@ -408,8 +453,20 @@ add_polygon({natural = "tree"}) assert(deepcompare(table_contents.planet_osm_line, {}), "MP line") assert(deepcompare(table_contents.planet_osm_transport_line, {}), "MP transport_line") assert(deepcompare(table_contents.planet_osm_roads, {}), "MP roads") -assert(deepcompare(table_contents.planet_osm_polygon[1], table_contents.planet_osm_polygon[2]), "admin boundary polygon") +assert(deepcompare(table_contents.planet_osm_polygon[1], table_contents.planet_osm_polygon[2]), "MP boundary polygon") +assert(deepcompare(table_contents.planet_osm_transport_polygon, {}), "MP transport_polygon") +table_contents.planet_osm_polygon = {} + +osm2pgsql.process_relation({tags = {type = "multipolygon", highway = "motorway"}}) +add_polygon({highway = "motorway"}) +add_transport_polygon({highway = "motorway"}) +assert(deepcompare(table_contents.planet_osm_line, {}), "MP highway line") +assert(deepcompare(table_contents.planet_osm_transport_line, {}), "MP highway transport_line") +assert(deepcompare(table_contents.planet_osm_roads, {}), "MP highway roads") +assert(deepcompare(table_contents.planet_osm_polygon[1], table_contents.planet_osm_polygon[2]), "MP highway polygon") +assert(deepcompare(table_contents.planet_osm_transport_polygon[1], table_contents.planet_osm_transport_polygon[2]), "MP highway transport_polygon") table_contents.planet_osm_polygon = {} +table_contents.planet_osm_transport_polygon = {} osm2pgsql.process_relation({tags = {type = "route", route = "road"}, members = {{type = 'w', ref = 1234, role = 'forward'}}}) add_line({route = "road"}) @@ -418,6 +475,7 @@ assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_ assert(deepcompare(table_contents.planet_osm_transport_line, {}), "route transport_line") assert(deepcompare(table_contents.planet_osm_roads, {}), "route roads") assert(deepcompare(table_contents.planet_osm_polygon, {}), "route polygon") +assert(deepcompare(table_contents.planet_osm_transport_polygon, {}), "route transport_polygon") assert(deepcompare(table_contents.planet_osm_route[1], table_contents.planet_osm_route[2]), "route route") table_contents.planet_osm_line = {} table_contents.planet_osm_polygon = {} From c5c021b42b2cb85874d5e4334841237c8fac918c Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Tue, 29 Jun 2021 01:06:09 -0700 Subject: [PATCH 14/54] Don't repeat layer tag in columns and hstore --- openstreetmap-carto.lua | 3 +++ scripts/lua/test.lua | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index ce8b013d20..2ca40d7db9 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -604,6 +604,7 @@ function add_line(tags) local cols = split_tags(tags, columns_map.line) cols['layer'] = layer(tags['layer']) cols['z_order'] = z_order(tags) + cols.tags['layer'] = nil cols.way = { create = 'line', split_at = 100000 } tables.line:add_row(cols) end @@ -612,6 +613,7 @@ function add_transport_line(tags) local cols = split_tags(tags, columns_map.line) cols['layer'] = layer(tags['layer']) cols['z_order'] = z_order(tags) + cols.tags['layer'] = nil cols.way = { create = 'line', split_at = 100000 } tables.transport_line:add_row(cols) end @@ -620,6 +622,7 @@ function add_roads(tags) local cols = split_tags(tags, columns_map.roads) cols['layer'] = layer(tags['layer']) cols['z_order'] = z_order(tags) + cols.tags['layer'] = nil cols.way = { create = 'line', split_at = 100000 } tables.roads:add_row(cols) end diff --git a/scripts/lua/test.lua b/scripts/lua/test.lua index 9c5efee2dd..d0f6661163 100644 --- a/scripts/lua/test.lua +++ b/scripts/lua/test.lua @@ -232,6 +232,10 @@ table_contents.planet_osm_line = {} add_line({highway = "road", foo = "bar"}) assert(deepcompare(table_contents.planet_osm_line[1], {highway = "road", z_order = 330, tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "Tag with column + hstore") +table_contents.planet_osm_line = {} +add_line({highway = "road", foo = "bar", layer = "5"}) +assert(deepcompare(table_contents.planet_osm_line[1], {highway = "road", z_order = 330, layer = "5", tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "Tag with column + hstore + layer") + table_contents.planet_osm_line = {} add_line({foo = "bar"}) assert(deepcompare(table_contents.planet_osm_line[1], {tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "hstore only") @@ -250,6 +254,10 @@ table_contents.planet_osm_transport_line = {} add_transport_line({foo = "bar"}) assert(deepcompare(table_contents.planet_osm_transport_line[1], {tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "hstore only") +table_contents.planet_osm_transport_line = {} +add_transport_line({highway = "road", foo = "bar", layer = "5"}) +assert(deepcompare(table_contents.planet_osm_transport_line[1], {highway = "road", z_order = 330, layer = "5", tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "Tag with column + hstore + layer") + print("TESTING: add_roads") table_contents.planet_osm_roads = {} add_roads({highway = "road"}) @@ -259,6 +267,10 @@ table_contents.planet_osm_roads = {} add_roads({highway = "road", foo = "bar"}) assert(deepcompare(table_contents.planet_osm_roads[1], {highway = "road", z_order = 330, tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "Tag with column + hstore") +table_contents.planet_osm_roads = {} +add_roads({highway = "road", layer = "5", foo = "bar"}) +assert(deepcompare(table_contents.planet_osm_roads[1], {highway = "road", z_order = 330, layer = "5", tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "Tag with column + hstore") + table_contents.planet_osm_roads = {} add_roads({foo = "bar"}) assert(deepcompare(table_contents.planet_osm_roads[1], {tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "hstore only") From d378bf6a852764a5840f93e7568a0e808fed7d22 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Tue, 29 Jun 2021 01:28:30 -0700 Subject: [PATCH 15/54] Use new planet_osm_admin table This doesn't yet take advantage of the deduplication done in the Lua transforms, so there's room for improvement, but this makes use of the new table. --- project.mml | 20 +++++++------------- style/admin.mss | 50 ++++++++++++++++++++++++------------------------- 2 files changed, 32 insertions(+), 38 deletions(-) diff --git a/project.mml b/project.mml index 22869c1979..b2d7ebcdb8 100644 --- a/project.mml +++ b/project.mml @@ -1066,10 +1066,8 @@ Layer: (SELECT way, admin_level - FROM planet_osm_roads - WHERE boundary = 'administrative' - AND admin_level IN ('0', '1', '2', '3', '4') - AND osm_id < 0 + FROM planet_osm_admin + WHERE admin_level BETWEEN 0 AND 4 ORDER BY admin_level DESC ) AS admin_low_zoom properties: @@ -1084,10 +1082,8 @@ Layer: (SELECT way, admin_level - FROM planet_osm_roads - WHERE boundary = 'administrative' - AND admin_level IN ('0', '1', '2', '3', '4', '5', '6', '7', '8') - AND osm_id < 0 + FROM planet_osm_admin + WHERE admin_level BETWEEN 0 AND 8 ORDER BY admin_level DESC ) AS admin_mid_zoom properties: @@ -1102,11 +1098,9 @@ Layer: (SELECT way, admin_level - FROM planet_osm_roads - WHERE boundary = 'administrative' - AND admin_level IN ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10') - AND osm_id < 0 - ORDER BY admin_level::integer DESC -- With 10 as a valid value, we need to do a numeric ordering, not a text ordering + FROM planet_osm_admin + WHERE admin_level BETWEEN 0 AND 10 + ORDER BY admin_level DESC ) AS admin_high_zoom properties: minzoom: 13 diff --git a/style/admin.mss b/style/admin.mss index a103e88206..e39388b0f8 100644 --- a/style/admin.mss +++ b/style/admin.mss @@ -13,7 +13,7 @@ Then all three layers are added to the rendering with comp-op: darken, so that t #admin-low-zoom[zoom < 8], #admin-mid-zoom[zoom >= 8][zoom < 13], #admin-high-zoom[zoom >= 13] { - [admin_level = '2']::firstline { + [admin_level = 2]::firstline { [zoom >= 8] { background/line-join: bevel; background/line-color: white; @@ -26,7 +26,7 @@ Then all three layers are added to the rendering with comp-op: darken, so that t [zoom >= 13] { background/line-width: 7; } [zoom >= 14] { background/line-width: 8; } } - [admin_level = '2']::wideline { + [admin_level = 2]::wideline { [zoom >= 4] { background/line-join: bevel; background/line-color: white; @@ -77,7 +77,7 @@ Then all three layers are added to the rendering with comp-op: darken, so that t line-width: 8; } } - [admin_level = '2']::narrowline { + [admin_level = 2]::narrowline { [zoom >= 8] { background/line-join: bevel; background/line-color: white; @@ -115,7 +115,7 @@ Then all three layers are added to the rendering with comp-op: darken, so that t } } - [admin_level = '3']::firstline { + [admin_level = 3]::firstline { [zoom >= 8] { background/line-join: bevel; background/line-color: white; @@ -128,7 +128,7 @@ Then all three layers are added to the rendering with comp-op: darken, so that t [zoom >= 13] { background/line-width: 5; } [zoom >= 14] { background/line-width: 5.5; } } - [admin_level = '3']::wideline { + [admin_level = 3]::wideline { [zoom >= 4] { background/line-join: bevel; background/line-color: white; @@ -179,7 +179,7 @@ Then all three layers are added to the rendering with comp-op: darken, so that t line-width: 5.5; } } - [admin_level = '3']::narrowline { + [admin_level = 3]::narrowline { [zoom >= 10] { background/line-join: bevel; background/line-color: white; @@ -209,7 +209,7 @@ Then all three layers are added to the rendering with comp-op: darken, so that t } } - [admin_level = '4']::firstline { + [admin_level = 4]::firstline { [zoom >= 8] { background/line-join: bevel; background/line-color: white; @@ -222,7 +222,7 @@ Then all three layers are added to the rendering with comp-op: darken, so that t [zoom >= 13] { background/line-width: 3.5; } [zoom >= 14] { background/line-width: 4; } } - [admin_level = '4']::wideline { + [admin_level = 4]::wideline { [zoom >= 4] { background/line-join: bevel; background/line-color: white; @@ -274,7 +274,7 @@ Then all three layers are added to the rendering with comp-op: darken, so that t line-width: 4; } } - [admin_level = '4']::narrowline { + [admin_level = 4]::narrowline { [zoom >= 10] { background/line-join: bevel; background/line-color: white; @@ -320,7 +320,7 @@ Then all three layers are added to the rendering with comp-op: darken, so that t ::wideline, ::narrowline { comp-op: darken; } - [admin_level = '5'][zoom >= 8]::firstline { + [admin_level = 5][zoom >= 8]::firstline { background/line-join: bevel; background/line-color: white; background/line-width: 0.6; @@ -354,7 +354,7 @@ Then all three layers are added to the rendering with comp-op: darken, so that t line-dasharray: 20,2,8,2; } } - [admin_level = '6'][zoom >= 10]::firstline { + [admin_level = 6][zoom >= 10]::firstline { background/line-join: bevel; background/line-color: white; background/line-width: 1; @@ -378,7 +378,7 @@ Then all three layers are added to the rendering with comp-op: darken, so that t line-dasharray: 16,2,3,2; } } - [admin_level = '7']::firstline { + [admin_level = 7]::firstline { [zoom >= 11] { background/line-join: bevel; background/line-color: white; @@ -400,7 +400,7 @@ Then all three layers are added to the rendering with comp-op: darken, so that t line-dasharray: 12,2,3,2,3,2; } } - [admin_level = '8']::firstline { + [admin_level = 8]::firstline { [zoom >= 12] { background/line-join: bevel; background/line-color: white; @@ -418,7 +418,7 @@ Then all three layers are added to the rendering with comp-op: darken, so that t } } - [admin_level = '9'][zoom >= 13]::firstline { + [admin_level = 9][zoom >= 13]::firstline { background/line-join: bevel; background/line-color: white; background/line-width: 1.2; @@ -433,7 +433,7 @@ Then all three layers are added to the rendering with comp-op: darken, so that t line-dasharray: 0,4,2,2,3,2,2,4; } } - [admin_level = '10'][zoom >= 14]::firstline { + [admin_level = 10][zoom >= 14]::firstline { background/line-join: bevel; background/line-color: white; background/line-width: 1.2; @@ -446,20 +446,20 @@ Then all three layers are added to the rendering with comp-op: darken, so that t } #admin-text[zoom >= 11][way_pixels >= 196000] { - [admin_level = '1'][way_pixels >= 360000], - [admin_level = '2'][way_pixels >= 360000], - [zoom >= 11][admin_level = '3'], - [zoom >= 11][admin_level = '4'], - [zoom >= 11][admin_level = '5'], - [zoom >= 12][admin_level = '6'], - [zoom >= 13][admin_level = '7'], - [zoom >= 14][admin_level = '8'], - [zoom >= 15][admin_level = '9'], + [admin_level = 1][way_pixels >= 360000], + [admin_level = 2][way_pixels >= 360000], + [zoom >= 11][admin_level = 3], + [zoom >= 11][admin_level = 4], + [zoom >= 11][admin_level = 5], + [zoom >= 12][admin_level = 6], + [zoom >= 13][admin_level = 7], + [zoom >= 14][admin_level = 8], + [zoom >= 15][admin_level = 9], [zoom >= 16] { text-name: "[name]"; text-face-name: @book-fonts; text-fill: @state-labels; - [admin_level = '6'] { text-fill: @county-labels; } + [admin_level = 6] { text-fill: @county-labels; } text-halo-radius: @standard-halo-radius; text-halo-fill: @standard-halo-fill; text-largest-bbox-only: false; From fe8c4b6b6c6e650d24f89e6117074d86b11e798d Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Tue, 29 Jun 2021 06:22:31 -0700 Subject: [PATCH 16/54] Use the new admin table for admin boundaries. This allows significant MSS simplification, no longer relying on comp-op and allowing more flexibility in styling. --- style/admin.mss | 359 ++++++++++++------------------------------------ 1 file changed, 86 insertions(+), 273 deletions(-) diff --git a/style/admin.mss b/style/admin.mss index e39388b0f8..0dd345cf69 100644 --- a/style/admin.mss +++ b/style/admin.mss @@ -2,443 +2,256 @@ @admin-boundaries-narrow: #845283; // Lch(42,35,327) @admin-boundaries-wide: #a37da1; // Lch(57,25,327) -/* For performance reasons, the admin border layers are split into three groups -for low, middle and high zoom levels. -Three attachments are used, with minor borders before major ones, and the thin centerline last, to handle -overlapping borders correctly and allow each type to have a different level of opacity. -Overlapping borders are hidden by a white background line, rendered before each line. -Then all three layers are added to the rendering with comp-op: darken, so that the white lines will not show -*/ - #admin-low-zoom[zoom < 8], #admin-mid-zoom[zoom >= 8][zoom < 13], #admin-high-zoom[zoom >= 13] { - [admin_level = 2]::firstline { - [zoom >= 8] { - background/line-join: bevel; - background/line-color: white; - background/line-width: 3; - } - [zoom >= 9] { background/line-width: 3.5; } - [zoom >= 10] { background/line-width: 4.5; } - [zoom >= 11] { background/line-width: 5.5; } - [zoom >= 12] { background/line-width: 6; } - [zoom >= 13] { background/line-width: 7; } - [zoom >= 14] { background/line-width: 8; } - } - [admin_level = 2]::wideline { - [zoom >= 4] { - background/line-join: bevel; - background/line-color: white; - background/line-width: 1.2; - line-join: bevel; - line-color: @admin-boundaries; - line-width: 1.2; - } + [admin_level = 2][zoom >= 4] { + wide/line-join: bevel; + wide/line-color: @admin-boundaries; + wide/line-opacity: 0.5; + wide/line-width: 1.2; [zoom >= 5] { - background/line-width: 1.5; - line-width: 1.5; + wide/line-width: 1.5; } [zoom >= 6] { - background/line-width: 1.8; - line-width: 1.8; + wide/line-width: 1.8; } [zoom >= 7] { - background/line-width: 2.2; - line-width: 2.2; + wide/line-width: 2.2; } [zoom >= 8] { - background/line-width: 3; - line-width: 3; - } - [zoom >= 9] { - background/line-width: 3.5; - line-width: 3.5; - } - [zoom >= 10] { - background/line-width: 4.5; - line-color: @admin-boundaries-wide; - line-width: 4.5; - } - [zoom >= 11] { - background/line-width: 5.5; - line-width: 5; - } - [zoom >= 12] { - background/line-width: 6; - line-width: 6; - } - [zoom >= 13] { - background/line-width: 7; - line-width: 7; - } - [zoom >= 14] { - background/line-width: 8; - line-width: 8; - } - } - [admin_level = 2]::narrowline { - [zoom >= 8] { - background/line-join: bevel; - background/line-color: white; - background/line-width: 0.6; + wide/line-width: 3; thin/line-join: bevel; thin/line-color: @admin-boundaries-narrow; - thin/line-width: 0.6; + thin/line-opacity: 0.6; + thin/line-width: 1.2; } [zoom >= 9] { - background/line-width: 0.8; + wide/line-width: 3.5; thin/line-width: 0.8; } [zoom >= 10] { - background/line-width: 1; + wide/line-color: @admin-boundaries-wide; + wide/line-width: 4.5; thin/line-width: 1; thin/line-dasharray: 18,1,4,1; + thin/line-clip: false; } [zoom >= 11] { - background/line-width: 1.2; + wide/line-width: 5; thin/line-width: 1.2; } [zoom >= 12] { - background/line-width: 1.4; + wide/line-width: 6; thin/line-width: 1.4; thin/line-dasharray: 27,1.5,6,1.5; } [zoom >= 13] { - background/line-width: 1.6; + wide/line-width: 7; thin/line-width: 1.6; } [zoom >= 14] { - background/line-width: 1.8; + wide/line-width: 8; thin/line-width: 1.8; thin/line-dasharray: 36,2,8,2; } } - [admin_level = 3]::firstline { - [zoom >= 8] { - background/line-join: bevel; - background/line-color: white; - background/line-width: 1.8; - } - [zoom >= 9] { background/line-width: 2.5; } - [zoom >= 10] { background/line-width: 3.2; } - [zoom >= 11] { background/line-width: 4; } - [zoom >= 12] { background/line-width: 4.5; } - [zoom >= 13] { background/line-width: 5; } - [zoom >= 14] { background/line-width: 5.5; } - } - [admin_level = 3]::wideline { - [zoom >= 4] { - background/line-join: bevel; - background/line-color: white; - background/line-width: 0.6; - line-join: bevel; - line-color: @admin-boundaries; - line-width: 0.6; - } + [admin_level = 3][zoom >= 4] { + wide/line-join: bevel; + wide/line-color: @admin-boundaries; + wide/line-opacity: 0.5; + wide/line-width: 0.6; [zoom >= 5] { - background/line-width: 0.8; - line-width: 0.8; + wide/line-width: 0.8; } [zoom >= 6] { - background/line-width: 1; - line-width: 1; + wide/line-width: 1.0; } [zoom >= 7] { - background/line-width: 1.2; - line-width: 1.2; + wide/line-width: 1.2; } [zoom >= 8] { - background/line-width: 1.8; - line-width: 1.8; + wide/line-width: 1.8; } [zoom >= 9] { - background/line-width: 2.5; - line-width: 2.5; + wide/line-width: 2.5; } [zoom >= 10] { - background/line-width: 3.2; - line-color: @admin-boundaries-wide; - line-width: 3.2; - } - [zoom >= 11] { - background/line-width: 4; - line-width: 4; - } - [zoom >= 12] { - background/line-width: 4.5; - line-width: 4.5; - } - [zoom >= 13] { - background/line-width: 5; - line-width: 5; - } - [zoom >= 14] { - background/line-width: 5.5; - line-width: 5.5; - } - } - [admin_level = 3]::narrowline { - [zoom >= 10] { - background/line-join: bevel; - background/line-color: white; - background/line-width: 0.8; + wide/line-color: @admin-boundaries-wide; + wide/line-width: 3.2; thin/line-join: bevel; thin/line-color: @admin-boundaries-narrow; + thin/line-opacity: 0.6; thin/line-width: 0.8; thin/line-dasharray: 12,2,1.5,2; + thin/line-clip: false; } [zoom >= 11] { - background/line-width: 1; - thin/line-width: 1; + wide/line-width: 4; + thin/line-width: 1.0; } [zoom >= 12] { - background/line-width: 1.2; + wide/line-width: 4.5; thin/line-width: 1.2; thin/line-dasharray: 17,3,2,3; } [zoom >= 13] { - background/line-width: 1.4; + wide/line-width: 5; thin/line-width: 1.4; } [zoom >= 14] { - background/line-width: 1.6; + wide/line-width: 5.5; thin/line-width: 1.6; thin/line-dasharray: 23,4,3,4; } } - [admin_level = 4]::firstline { - [zoom >= 8] { - background/line-join: bevel; - background/line-color: white; - background/line-width: 1; - } - [zoom >= 9] { background/line-width: 1.5; } - [zoom >= 10] { background/line-width: 2; } - [zoom >= 11] { background/line-width: 2.5; } - [zoom >= 12] { background/line-width: 3; } - [zoom >= 13] { background/line-width: 3.5; } - [zoom >= 14] { background/line-width: 4; } - } - [admin_level = 4]::wideline { - [zoom >= 4] { - background/line-join: bevel; - background/line-color: white; - background/line-width: 0.4; - line-color: @admin-boundaries; - line-join: bevel; - line-width: 0.4; - line-clip: false; - } + [admin_level = 4][zoom >= 4] { + wide/line-join: bevel; + wide/line-color: @admin-boundaries; + wide/line-opacity: 0.5; + wide/line-width: 0.4; [zoom >= 5] { - background/line-width: 0.5; - line-width: 0.5; + wide/line-width: 0.5; } [zoom >= 6] { - background/line-width: 0.6; - line-width: 0.6; + wide/line-width: 0.6; } [zoom >= 7] { - background/line-width: 0.8; - line-width: 0.8; + wide/line-width: 0.8; } [zoom >= 8] { - background/line-width: 1; - line-width: 1; + wide/line-width: 1.0; } [zoom >= 9] { - background/line-width: 1.5; - line-width: 1.5; - } - [zoom >= 10] { - background/line-width: 2; - line-color: @admin-boundaries-wide; - line-width: 2; + wide/line-width: 1.5; } - [zoom >= 11] { - background/line-width: 2.5; - line-width: 2.8; - } - [zoom >= 12] { - background/line-width: 3; - line-width: 3; - } - [zoom >= 13] { - background/line-width: 3.5; - line-width: 3.5; - } - [zoom >= 14] { - background/line-width: 4; - line-width: 4; - } - } - [admin_level = 4]::narrowline { [zoom >= 10] { - background/line-join: bevel; - background/line-color: white; - background/line-width: 0.6; + wide/line-color: @admin-boundaries-wide; + wide/line-width: 2.0; + thin/line-join: bevel; thin/line-color: @admin-boundaries-narrow; + thin/line-opacity: 0.6; thin/line-width: 0.6; thin/line-dasharray: 8,2,1.5,2,1.5,2; + thin/line-clip: false; } [zoom >= 11] { - background/line-width: 0.8; + wide/line-width: 2.8; thin/line-width: 0.8; } [zoom >= 12] { - background/line-width: 1; - thin/line-width: 1; + wide/line-width: 3.0; + thin/line-width: 1.0; thin/line-dasharray: 12,3,2,3,2,3; } [zoom >= 13] { - background/line-width: 1.2; + wide/line-width: 3.5; thin/line-width: 1.2; } [zoom >= 14] { - background/line-width: 1.4; + wide/line-width: 4; thin/line-width: 1.4; thin/line-dasharray: 16,4,3,4,3,4; } } - ::firstline { opacity: 0.5; } - ::wideline { opacity: 0.5; } - ::narrowline { opacity: 0.6; } - /* - The following code prevents admin boundaries from being rendered on top of - each other. Comp-op works on the entire attachment, not on the individual - border. Therefore, this code generates an attachment containing a set of - @admin-boundaries/white dashed lines (of which only the top one is visible), - and with `comp-op: darken` the white part is ignored, while the - @admin-boundaries colored part is rendered (as long as the background is not - darker than @admin-boundaries). - The SQL has `ORDER BY admin_level`, so the boundary with the lowest - admin_level is rendered on top, and therefore the only visible boundary. - */ - ::firstline, - ::wideline, - ::narrowline { comp-op: darken; } - [admin_level = 5][zoom >= 8]::firstline { - background/line-join: bevel; - background/line-color: white; - background/line-width: 0.6; + [admin_level = 5][zoom >= 8] { line-join: bevel; line-color: @admin-boundaries; + line-opacity: 0.5; line-width: 0.6; line-dasharray: 4,0.6,2,0.6; line-clip: false; [zoom >= 9] { - background/line-width: 0.8; line-width: 0.8; line-dasharray: 6,1,3,1; } [zoom >= 10] { - background/line-width: 1.2; line-width: 1.2; line-dasharray: 10,1.5,4.5,1.5; } [zoom >= 11] { - background/line-width: 1.7; line-width: 1.7; } [zoom >= 12] { - background/line-width: 2.1; line-width: 2.1; line-dasharray: 16,2,6,2; } [zoom >= 14] { - background/line-width: 2.4; line-width: 2.4; line-dasharray: 20,2,8,2; } } - [admin_level = 6][zoom >= 10]::firstline { - background/line-join: bevel; - background/line-color: white; - background/line-width: 1; + + [admin_level = 6][zoom >= 10] { line-join: bevel; line-color: @admin-boundaries; - line-width: 1; + line-opacity: 0.5; + line-width: 1.0; line-dasharray: 8,1.5,1.5,1.5; line-clip: false; [zoom >= 11] { - background/line-width: 1.4; line-width: 1.4; } [zoom >= 12] { - background/line-width: 1.8; line-width: 1.8; - line-dasharray: 12,1.5,2,1.5; + line-dasharray: 12,1.5,2,1.5; } [zoom >= 14] { - background/line-width: 2.1; line-width: 2.1; line-dasharray: 16,2,3,2; } } - [admin_level = 7]::firstline { - [zoom >= 11] { - background/line-join: bevel; - background/line-color: white; - background/line-width: 1.2; - line-join: bevel; - line-color: @admin-boundaries; - line-width: 1.2; - line-dasharray: 6,1.5,1.5,1.5,1.5,1.5; - line-clip: false; - } + + [admin_level = 7][zoom >= 11] { + line-join: bevel; + line-color: @admin-boundaries; + line-opacity: 0.5; + line-width: 1.2; + line-dasharray: 6,1.5,1.5,1.5,1.5,1.5; + line-clip: false; [zoom >= 12] { - background/line-width: 1.5; line-width: 1.5; line-dasharray: 9,2,2,2,2,2; } [zoom >= 14] { - background/line-width: 1.8; line-width: 1.8; line-dasharray: 12,2,3,2,3,2; } } - [admin_level = 8]::firstline { - [zoom >= 12] { - background/line-join: bevel; - background/line-color: white; - background/line-width: 1.4; - line-join: bevel; - line-color: @admin-boundaries; - line-width: 1.4; - line-dasharray: 8,2,2,2,2.5,2,2,2; - line-clip: false; - } + + [admin_level = 8][zoom >= 12] { + line-join: bevel; + line-color: @admin-boundaries; + line-opacity: 0.5; + line-width: 1.4; + line-dasharray: 8,2,2,2,2.5,2,2,2; + line-clip: false; [zoom >= 14] { - background/line-width: 1.6; line-width: 1.6; line-dasharray: 10,2,2,2,3,2,2,2; } } - [admin_level = 9][zoom >= 13]::firstline { - background/line-join: bevel; - background/line-color: white; - background/line-width: 1.2; + [admin_level = 9][zoom >= 13] { line-join: bevel; line-color: @admin-boundaries; + line-opacity: 0.5; line-width: 1.2; line-dasharray: 0,3,2,2,2,2,2,3; line-clip: false; [zoom >= 14] { - background/line-width: 1.4; line-width: 1.4; line-dasharray: 0,4,2,2,3,2,2,4; } } - [admin_level = 10][zoom >= 14]::firstline { - background/line-join: bevel; - background/line-color: white; - background/line-width: 1.2; + + [admin_level = 10][zoom >= 14] { line-join: bevel; line-color: @admin-boundaries; + line-opacity: 0.5; line-width: 1.2; line-dasharray: 0,3,2,2,2,3; line-clip: false; From 0f9d5af416c03a96923f0181845a9294cc3e7632 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Wed, 30 Jun 2021 20:48:40 -0700 Subject: [PATCH 17/54] Correctly use the transport cols to figure out hstore --- openstreetmap-carto.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 2ca40d7db9..5c3ed8bb02 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -610,7 +610,7 @@ function add_line(tags) end function add_transport_line(tags) - local cols = split_tags(tags, columns_map.line) + local cols = split_tags(tags, columns_map.transport_line) cols['layer'] = layer(tags['layer']) cols['z_order'] = z_order(tags) cols.tags['layer'] = nil @@ -636,7 +636,7 @@ function add_polygon(tags) end function add_transport_polygon(tags) - local cols = split_tags(tags, columns_map.polygon) + local cols = split_tags(tags, columns_map.transport_polygon) cols['layer'] = layer(tags['layer']) cols['z_order'] = z_order(tags) cols.way = { create = 'area', split_at = nil } From b82ba637d977233991fc24e40ab84d7ba87644f9 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Wed, 21 Jul 2021 23:00:34 -0700 Subject: [PATCH 18/54] Don't add route relations to line and road tables Route relations can be accessed to the new tables. --- openstreetmap-carto.lua | 6 +----- project.mml | 40 ++++++++++++++++++++-------------------- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 5c3ed8bb02..a0dbe4c242 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -84,6 +84,7 @@ local pg_cols = { }, transport_line = { 'access', + 'aeroway', 'bicycle', 'bridge', 'construction', @@ -719,12 +720,7 @@ function osm2pgsql.process_relation(object) add_transport_polygon(object.tags) end elseif type == "route" then - add_line(object.tags) add_route(object) - -- TODO: Remove this, roads tags don't belong on route relations - if roads(object.tags) then - add_roads(object.tags) - end end end diff --git a/project.mml b/project.mml index b2d7ebcdb8..fa863e818a 100644 --- a/project.mml +++ b/project.mml @@ -481,7 +481,7 @@ Layer: END AS link, COALESCE(layer,0) AS layernotnull, z_order - FROM planet_osm_line + FROM planet_osm_transport_line WHERE (tunnel = 'yes' OR tunnel = 'building_passage' OR covered = 'yes') AND highway IS NOT NULL -- end of road select UNION ALL @@ -505,7 +505,7 @@ Layer: 'no' AS link, COALESCE(layer,0) AS layernotnull, z_order - FROM planet_osm_line + FROM planet_osm_transport_line WHERE (tunnel = 'yes' OR tunnel = 'building_passage' OR covered = 'yes') AND (railway NOT IN ('platform') AND railway IS NOT NULL) -- end of rail select ) AS features @@ -725,7 +725,7 @@ Layer: END AS access, construction, CASE - WHEN service IN ('parking_aisle', 'drive-through', 'driveway') OR leisure IN ('slipway') THEN 'INT-minor'::text + WHEN service IN ('parking_aisle', 'drive-through', 'driveway') OR tags->'leisure' IN ('slipway') THEN 'INT-minor'::text ELSE 'INT-normal'::text END AS service, CASE @@ -735,7 +735,7 @@ Layer: COALESCE(layer,0) AS layernotnull, osm_id, z_order - FROM planet_osm_line + FROM planet_osm_transport_line WHERE (tunnel IS NULL OR NOT tunnel IN ('yes', 'building_passage')) AND (covered IS NULL OR NOT covered = 'yes') AND (bridge IS NULL OR NOT bridge IN ('yes', 'boardwalk', 'cantilever', 'covered', 'low_water_crossing', 'movable', 'trestle', 'viaduct')) @@ -757,12 +757,12 @@ Layer: WHEN access IN ('no', 'private') THEN 'no'::text END AS access, construction, - CASE WHEN service IN ('parking_aisle', 'drive-through', 'driveway') OR leisure IN ('slipway') THEN 'INT-minor'::text ELSE 'INT-normal'::text END AS service, + CASE WHEN service IN ('parking_aisle', 'drive-through', 'driveway') OR tags->'leisure' IN ('slipway') THEN 'INT-minor'::text ELSE 'INT-normal'::text END AS service, 'no' AS link, COALESCE(layer,0) AS layernotnull, osm_id, z_order - FROM planet_osm_line + FROM planet_osm_transport_line WHERE (tunnel IS NULL OR NOT tunnel IN ('yes', 'building_passage')) AND (covered IS NULL OR NOT covered = 'yes') AND (bridge IS NULL OR NOT bridge IN ('yes', 'boardwalk', 'cantilever', 'covered', 'low_water_crossing', 'movable', 'trestle', 'viaduct')) @@ -876,7 +876,7 @@ Layer: WHEN surface IN ('paved', 'asphalt', 'cobblestone', 'cobblestone:flattened', 'sett', 'concrete', 'concrete:lanes', 'concrete:plates', 'paving_stones', 'metal', 'wood', 'unhewn_cobblestone') THEN 'paved' END AS int_surface - FROM planet_osm_roads + FROM planet_osm_transport_line WHERE highway IS NOT NULL OR (railway IS NOT NULL AND railway != 'preserved' AND (service IS NULL OR service NOT IN ('spur', 'siding', 'yard'))) @@ -956,7 +956,7 @@ Layer: END AS link, COALESCE(layer,0) AS layernotnull, z_order - FROM planet_osm_line + FROM planet_osm_transport_line WHERE bridge IN ('yes', 'boardwalk', 'cantilever', 'covered', 'low_water_crossing', 'movable', 'trestle', 'viaduct') AND highway IS NOT NULL -- end of road select UNION ALL @@ -980,7 +980,7 @@ Layer: 'no' AS link, COALESCE(layer,0) AS layernotnull, z_order - FROM planet_osm_line + FROM planet_osm_transport_line WHERE bridge IN ('yes', 'boardwalk', 'cantilever', 'covered', 'low_water_crossing', 'movable', 'trestle', 'viaduct') AND railway IS NOT NULL -- end of rail select ) AS features @@ -1722,7 +1722,7 @@ Layer: osm_id, highway, string_to_array(ref, ';') AS refs - FROM planet_osm_roads + FROM planet_osm_transport_line WHERE highway IN ('motorway', 'trunk', 'primary', 'secondary') AND ref IS NOT NULL ) AS p) AS q @@ -1832,21 +1832,21 @@ Layer: name, CASE WHEN oneway IN ('yes', '-1') THEN oneway - WHEN junction IN ('roundabout') AND (oneway IS NULL OR NOT oneway IN ('no', 'reversible')) THEN 'yes' + WHEN tags->'junction' IN ('roundabout') AND (oneway IS NULL OR NOT oneway IN ('no', 'reversible')) THEN 'yes' END AS oneway, horse, bicycle - FROM planet_osm_line l + FROM planet_osm_transport_line WHERE highway IN ('motorway', 'motorway_link', 'trunk', 'trunk_link', 'primary', 'primary_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link', 'residential', 'unclassified', 'road', 'service', 'pedestrian', 'raceway', 'living_street', 'construction') AND (name IS NOT NULL OR oneway IN ('yes', '-1') - OR junction IN ('roundabout')) + OR tags->'junction' IN ('roundabout')) ORDER BY z_order DESC, -- put important roads first COALESCE(layer, 0), -- put top layered roads first length(name) DESC, -- Try to fit big labels in first name DESC, -- Force a consistent ordering between differently named streets - l.osm_id DESC -- Force an ordering for streets of the same name, e.g. dualized roads + osm_id DESC -- Force an ordering for streets of the same name, e.g. dualized roads ) AS roads_text_name properties: cache-features: true @@ -1864,15 +1864,15 @@ Layer: name, CASE WHEN oneway IN ('yes', '-1') THEN oneway - WHEN junction IN ('roundabout') AND (oneway IS NULL OR NOT oneway IN ('no', 'reversible')) THEN 'yes' + WHEN tags->'junction' IN ('roundabout') AND (oneway IS NULL OR NOT oneway IN ('no', 'reversible')) THEN 'yes' END AS oneway, horse, bicycle - FROM planet_osm_line + FROM planet_osm_transport_line WHERE highway IN ('bridleway', 'footway', 'cycleway', 'path', 'track', 'steps', 'construction') AND (name IS NOT NULL OR oneway IN ('yes', '-1') - OR junction IN ('roundabout')) + OR tags->'junction' IN ('roundabout')) ) AS paths_text_name properties: cache-features: true @@ -1893,7 +1893,7 @@ Layer: tags->'usage' as usage, construction, name - FROM planet_osm_line l + FROM planet_osm_transport_line WHERE railway IN ('rail', 'subway', 'narrow_gauge', 'light_rail', 'preserved', 'funicular', 'monorail', 'miniature', 'tram', 'disused', 'construction') AND (tunnel IS NULL OR NOT tunnel IN ('yes', 'building_passage')) @@ -1904,7 +1904,7 @@ Layer: COALESCE(layer, 0), -- put top layered rails first length(name) DESC, -- Try to fit big labels in first name DESC, -- Force a consistent ordering between differently named railways - l.osm_id DESC -- Force an ordering for railways of the same name, e.g. dualized rails + osm_id DESC -- Force an ordering for railways of the same name, e.g. dualized rails ) AS railways_text_name properties: minzoom: 11 @@ -1934,7 +1934,7 @@ Layer: way, CASE WHEN highway IN ('unclassified', 'residential', 'track') THEN highway END AS highway, string_to_array(ref, ';') AS refs - FROM planet_osm_line + FROM planet_osm_transport_line WHERE highway IN ('unclassified', 'residential', 'track') AND ref IS NOT NULL ) AS p) AS q From ab745c3d862e8cd810af4de0f6e2b4a46aea7c6f Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Wed, 21 Jul 2021 23:12:45 -0700 Subject: [PATCH 19/54] Fix failing tests from previous two commits --- scripts/lua/test.lua | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/scripts/lua/test.lua b/scripts/lua/test.lua index d0f6661163..2ec66a6c92 100644 --- a/scripts/lua/test.lua +++ b/scripts/lua/test.lua @@ -290,12 +290,12 @@ assert(deepcompare(table_contents.planet_osm_polygon[1], {tags = {foo = "bar"}, print("TESTING: add_transport_polygon") table_contents.planet_osm_transport_polygon = {} -add_transport_polygon({natural = "wood"}) -assert(deepcompare(table_contents.planet_osm_transport_polygon[1], {natural = "wood", tags = {}, way = { create = 'area', split_at = nil }}), "Tag with column") +add_transport_polygon({highway = "service"}) +assert(deepcompare(table_contents.planet_osm_transport_polygon[1], {highway = "service", z_order = 150, tags = {}, way = { create = 'area' }}), "Tag with column") table_contents.planet_osm_transport_polygon = {} -add_transport_polygon({natural = "wood", foo = "bar"}) -assert(deepcompare(table_contents.planet_osm_transport_polygon[1], {natural = "wood", tags = {foo = "bar"}, way = { create = 'area', split_at = nil }}), "Tag with column + hstore") +add_transport_polygon({highway = "service", foo = "bar"}) +assert(deepcompare(table_contents.planet_osm_transport_polygon[1], {highway = "service", z_order = 150, tags = {foo = "bar"}, way = { create = 'area' }}), "Tag with column + hstore") table_contents.planet_osm_transport_polygon = {} add_transport_polygon({foo = "bar"}) @@ -480,19 +480,6 @@ assert(deepcompare(table_contents.planet_osm_transport_polygon[1], table_content table_contents.planet_osm_polygon = {} table_contents.planet_osm_transport_polygon = {} -osm2pgsql.process_relation({tags = {type = "route", route = "road"}, members = {{type = 'w', ref = 1234, role = 'forward'}}}) -add_line({route = "road"}) -add_route({tags = { route = "road"}, members = {{type = 'w', ref = 1234, role = 'forward'}}}) -assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_line[2]), "route line") -assert(deepcompare(table_contents.planet_osm_transport_line, {}), "route transport_line") -assert(deepcompare(table_contents.planet_osm_roads, {}), "route roads") -assert(deepcompare(table_contents.planet_osm_polygon, {}), "route polygon") -assert(deepcompare(table_contents.planet_osm_transport_polygon, {}), "route transport_polygon") -assert(deepcompare(table_contents.planet_osm_route[1], table_contents.planet_osm_route[2]), "route route") -table_contents.planet_osm_line = {} -table_contents.planet_osm_polygon = {} -table_contents.planet_osm_roads = {} - -- Testing of phase 2 --[[ From 5b4f81b5d25246d808a769e0543f968ad04e7f8b Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sun, 29 Aug 2021 15:28:31 -0700 Subject: [PATCH 20/54] Switch default database name --- INSTALL.md | 10 +++++----- project.mml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 9c85f63696..c082b150d2 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -3,25 +3,25 @@ This document describes how to manually configure your system for running OpenStreetMap Carto. If you prefer quick, platform independent setup for a development environment, without the need to install and configure tools by hand, follow a Docker installation guide in [DOCKER.md](https://github.com/gravitystorm/openstreetmap-carto/blob/master/DOCKER.md). ## OpenStreetMap data -You need OpenStreetMap data loaded into a PostGIS database (see below for [dependencies](#dependencies)). These stylesheets expect a database generated with osm2pgsql using the flex backend and the default database name (`gis`). This requires osm2pgsql 1.4.1 or later. +You need OpenStreetMap data loaded into a PostGIS database (see below for [dependencies](#dependencies)). These stylesheets expect a database generated with osm2pgsql using the flex backend with the supplied Lua scripts. This requires osm2pgsql 1.4.1 or later. Start by creating a database ``` sudo -u postgres createuser -s $USER -createdb gis +createdb flex ``` Enable PostGIS and hstore extensions with ``` -psql -d gis -c 'CREATE EXTENSION postgis; CREATE EXTENSION hstore;' +psql -d flex -c 'CREATE EXTENSION postgis; CREATE EXTENSION hstore;' ``` then grab some OSM data. It's probably easiest to grab an PBF of OSM data from [Geofabrik](https://download.geofabrik.de/). Once you've done that, import with osm2pgsql: ``` -osm2pgsql --output flex --style openstreetmap-carto.lua -d gis ~/path/to/data.osm.pbf +osm2pgsql --output flex --style openstreetmap-carto.lua -d flex ~/path/to/data.osm.pbf ``` You can find a more detailed guide to setting up a database and loading data with osm2pgsql at [switch2osm.org](https://switch2osm.org/manually-building-a-tile-server-16-04-2-lts/). @@ -30,7 +30,7 @@ You can find a more detailed guide to setting up a database and loading data wit Custom indexes are required for rendering performance and are essential on full planet databases. ``` -psql -d gis -f indexes.sql +psql -d flex -f indexes.sql ``` ## Scripted download diff --git a/project.mml b/project.mml index fa863e818a..77f55e31c5 100644 --- a/project.mml +++ b/project.mml @@ -26,7 +26,7 @@ _parts: srs: "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over" osm2pgsql: &osm2pgsql type: "postgis" - dbname: "gis" + dbname: flex key_field: "" geometry_field: "way" extent: "-20037508,-20037508,20037508,20037508" From 7a8871e0a9ae48c69a19a07a033e956c04e246c2 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sun, 29 Aug 2021 15:56:24 -0700 Subject: [PATCH 21/54] Also switch DB name for loading external data --- .travis.yml | 6 +++--- external-data.yml | 2 +- scripts/docker-startup.sh | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index a0bc8a0c27..2ee29a03f9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ env: install: - npm install carto@$CARTO - pip3 install --user colormath - - createdb -w -E utf8 -U postgres gis && psql -Xq -d gis -U postgres -w -c "CREATE EXTENSION postgis; CREATE EXTENSION hstore;" + - createdb -w -E utf8 -U postgres flex && psql -Xq -d flex -U postgres -w -c "CREATE EXTENSION postgis; CREATE EXTENSION hstore;" script: # We're using pipes in the checks, so fail if any part fails - set -o pipefail @@ -39,8 +39,8 @@ script: - diff -qu <(scripts/generate_road_colours.py) style/road-colors-generated.mss # Skip the next two steps because they required a more recent version of osm2pgsql # Create the PostgreSQL tables - # - osm2pgsql -G --hstore --style openstreetmap-carto.style --tag-transform-script openstreetmap-carto.lua -U postgres -d gis -r xml <(echo '') + # - osm2pgsql -G --hstore --style openstreetmap-carto.style --tag-transform-script openstreetmap-carto.lua -U postgres -d flex -r xml <(echo '') # Apply the custom indexes - # - psql -1Xq -v ON_ERROR_STOP=1 -d gis -f indexes.sql + # - psql -1Xq -v ON_ERROR_STOP=1 -d flex -f indexes.sql # Test for classes in project.mml (not supported in vector tiles) - '! grep "class:" project.mml > /dev/null' diff --git a/external-data.yml b/external-data.yml index c93a865361..05fb218a69 100644 --- a/external-data.yml +++ b/external-data.yml @@ -2,7 +2,7 @@ settings: temp_schema: loading schema: public data_dir: data - database: gis + database: flex metadata_table: external_data sources: simplified_water_polygons: diff --git a/scripts/docker-startup.sh b/scripts/docker-startup.sh index c111b34145..c139687662 100644 --- a/scripts/docker-startup.sh +++ b/scripts/docker-startup.sh @@ -18,9 +18,9 @@ test $i -gt $MAXCOUNT && echo "Timeout while waiting for PostgreSQL to be runnin case "$1" in import) # Creating default database - psql -c "SELECT 1 FROM pg_database WHERE datname = 'gis';" | grep -q 1 || createdb gis && \ - psql -d gis -c 'CREATE EXTENSION IF NOT EXISTS postgis;' && \ - psql -d gis -c 'CREATE EXTENSION IF NOT EXISTS hstore;' && \ + psql -c "SELECT 1 FROM pg_database WHERE datname = 'flex';" | grep -q 1 || createdb flex && \ + psql -d flex -c 'CREATE EXTENSION IF NOT EXISTS postgis;' && \ + psql -d flex -c 'CREATE EXTENSION IF NOT EXISTS hstore;' && \ # Creating default import settings file editable by user and passing values for osm2pgsql if [ ! -e ".env" ]; then @@ -44,7 +44,7 @@ EOF --number-processes $OSM2PGSQL_NUMPROC \ --hstore \ --multi-geometry \ - --database gis \ + --database flex \ --slim \ --drop \ --style openstreetmap-carto.style \ From 7e7ac1b3272b217399a69c162f9ff139739a55a9 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Wed, 29 Dec 2021 23:42:43 -0800 Subject: [PATCH 22/54] Correctly add ways back in stage 2 flex processing When a way has has other (e.g. road) tags as well as admin tags, it needs to be re-added to the line tables because osm2pgsql deletes it. --- openstreetmap-carto.lua | 45 ++++++++++++++++++++--------------------- scripts/lua/test.lua | 13 ++++++++---- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index a0dbe4c242..55d5c3e98c 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -664,34 +664,33 @@ function osm2pgsql.process_node(object) end function osm2pgsql.process_way(object) - if osm2pgsql.stage == 1 then - if clean_tags(object.tags) then - return + if osm2pgsql.stage == 2 then + -- Stage two processing is called on ways that are part of admin boundary relations + local props = phase2_admin_ways[object.id] + if props ~= nil then + tables.admin:add_row({admin_level = props.level, multiple_relations = (props.parents > 1), geom = { create = 'line' }}) end + end + if clean_tags(object.tags) then + return + end - local area_tags = isarea(object.tags) - if object.is_closed and area_tags then - add_polygon(object.tags) - - if z_order(object.tags) ~= nil then - add_transport_polygon(object.tags) - end - else - add_line(object.tags) + local area_tags = isarea(object.tags) + if object.is_closed and area_tags then + add_polygon(object.tags) - if z_order(object.tags) ~= nil then - add_transport_line(object.tags) - end + if z_order(object.tags) ~= nil then + add_transport_polygon(object.tags) + end + else + add_line(object.tags) - if roads(object.tags) then - add_roads(object.tags) - end + if z_order(object.tags) ~= nil then + add_transport_line(object.tags) end - elseif osm2pgsql.stage == 2 then - -- Stage two processing is called on ways that are part of admin boundary relations - local props = phase2_admin_ways[object.id] - if props ~= nil then - tables.admin:add_row({admin_level = props.level, multiple_relations = (props.parents > 1), geom = { create = 'line' }}) + + if roads(object.tags) then + add_roads(object.tags) end end end diff --git a/scripts/lua/test.lua b/scripts/lua/test.lua index 2ec66a6c92..5f9289fe8a 100644 --- a/scripts/lua/test.lua +++ b/scripts/lua/test.lua @@ -484,11 +484,11 @@ table_contents.planet_osm_transport_polygon = {} --[[ This sets up an - 1. admin_level=2 relation with ways 1, 2, 3, 5, 7, 8 + 1. admin_level=2 relation with ways 1, 2, 3, 5, 7, 8, 9 2. admin_level=2 relation with ways 1, 2, 6, 7, 8 3. admin_level=4 relation with ways 1, 3, 4 - The ways have correct tags except for 7 and 8 + The ways have correct tags except for 7 and 8. Way 9 also has road tags. ]] -- Currently in phase 1 @@ -501,7 +501,8 @@ local test_ways = { { id = 5, tags = { boundary = "administrative", admin_level = "2"} }, { id = 6, tags = { boundary = "administrative", admin_level = "2"} }, { id = 7, tags = { boundary = "administrative", admin_level = "3"} }, -- incorrect tags - { id = 8, tags = { } } -- incorrect tags + { id = 8, tags = { } }, -- incorrect tags + { id = 9, tags = { boundary = "administrative", admin_level = "2", highway = "road"} } } -- add another way that isn't part of a relation local test_relations = { @@ -512,7 +513,8 @@ local test_relations = { {type = 'w', ref = 3, role = "outer"}, {type = 'w', ref = 5, role = "outer"}, {type = 'w', ref = 7, role = "outer"}, - {type = 'w', ref = 8, role = "outer"} } }, + {type = 'w', ref = 8, role = "outer"}, + {type = 'w', ref = 9, role = "outer"} } }, { id = 2, tags = {type = "boundary", boundary = "administrative", admin_level = "2"}, members = { {type = 'w', ref = 1, role = "outer"}, {type = 'w', ref = 2, role = "outer"}, @@ -546,6 +548,8 @@ end osm2pgsql.stage = 2 +-- This process should really delete from the planet_osm_line table, but there's no way to do that with the data we're collecting. +-- osm2pgsql itself has access to more data for _, way in ipairs(test_ways) do if pending_ways[way.id] then osm2pgsql.process_way(way) @@ -562,3 +566,4 @@ assert(deepcompare(table_contents.planet_osm_admin[5], {admin_level = 2, multipl assert(deepcompare(table_contents.planet_osm_admin[6], {admin_level = 2, multiple_relations = false, geom = {create = "line" } }), "row 6") assert(deepcompare(table_contents.planet_osm_admin[7], {admin_level = 2, multiple_relations = true, geom = {create = "line" } }), "row 7") assert(deepcompare(table_contents.planet_osm_admin[8], {admin_level = 2, multiple_relations = true, geom = {create = "line" } }), "row 8") +assert(deepcompare(table_contents.planet_osm_admin[9], {admin_level = 2, multiple_relations = false, geom = {create = "line" } }), "row 9") From 5adf64f930055cf014cddc5a022f533d9b3c2ba5 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Thu, 30 Dec 2021 13:11:40 -0800 Subject: [PATCH 23/54] Use ST_LineMerge on admin boundaries This avoids artifacts at the corners, at the cost of rare missed dashes on metatile boundaries. --- project.mml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/project.mml b/project.mml index b1c65d5455..08c9f7b520 100644 --- a/project.mml +++ b/project.mml @@ -1080,10 +1080,12 @@ Layer: <<: *osm2pgsql table: |- (SELECT - way, + ST_LineMerge(ST_Collect(way)) AS way, admin_level FROM planet_osm_admin WHERE admin_level BETWEEN 0 AND 4 + AND way && !BBOX! + GROUP BY admin_level ORDER BY admin_level DESC ) AS admin_low_zoom properties: @@ -1096,10 +1098,12 @@ Layer: <<: *osm2pgsql table: |- (SELECT - way, + ST_LineMerge(ST_Collect(way)) AS way, admin_level FROM planet_osm_admin WHERE admin_level BETWEEN 0 AND 8 + AND way && !BBOX! + GROUP BY admin_level ORDER BY admin_level DESC ) AS admin_mid_zoom properties: @@ -1112,10 +1116,12 @@ Layer: <<: *osm2pgsql table: |- (SELECT - way, + ST_LineMerge(ST_Collect(way)) AS way, admin_level FROM planet_osm_admin WHERE admin_level BETWEEN 0 AND 10 + AND way && !BBOX! + GROUP BY admin_level ORDER BY admin_level DESC ) AS admin_high_zoom properties: From 0068392b20de1a1366699a84576b7d52bc140c76 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Thu, 30 Dec 2021 13:13:43 -0800 Subject: [PATCH 24/54] Add a z_order for busways Even though they're not rendered right now, we may want to support them in the future. --- openstreetmap-carto.lua | 1 + scripts/lua/test.lua | 3 +++ 2 files changed, 4 insertions(+) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 55d5c3e98c..bf472773b4 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -441,6 +441,7 @@ local roads_info = { primary_link = {z = 220, roads = true}, secondary_link = {z = 210, roads = true}, tertiary_link = {z = 200, roads = false}, + busway = {z = 170, roads = false}, service = {z = 150, roads = false}, track = {z = 110, roads = false}, path = {z = 100, roads = false}, diff --git a/scripts/lua/test.lua b/scripts/lua/test.lua index 5f9289fe8a..280ab29c8f 100644 --- a/scripts/lua/test.lua +++ b/scripts/lua/test.lua @@ -140,6 +140,9 @@ assert(z_order({highway="trunk"}) > z_order({highway="primary"}) , "test failed: assert(z_order({highway="primary"}) > z_order({highway="secondary"}) , "test failed: primary > secondary") assert(z_order({highway="secondary"}) > z_order({highway="tertiary"}) , "test failed: secondary > tertiary") +assert(z_order({highway="tertiary_link"}) > z_order({highway="busway"}), "test failed: tertiary_link > busway") +assert(z_order({highway="busway"}) > z_order({highway="service"}), "test failed: busway > service") + assert(z_order({highway="construction"}) == 33 , "test failed: highway=construction") assert(z_order({highway="construction", construction="motorway"}) == 38 , "test failed: highway=construction construction=motorway") assert(z_order({highway="construction", construction="motorway", railway="rail"}) == 440, "test failed: construction motorway + rail") From e7af9bd90799103955f1d3996201ce0904be1665 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Fri, 31 Dec 2021 02:12:31 -0800 Subject: [PATCH 25/54] Use correct caps on bbox token --- project.mml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/project.mml b/project.mml index 08c9f7b520..e2b76d1a14 100644 --- a/project.mml +++ b/project.mml @@ -1084,7 +1084,7 @@ Layer: admin_level FROM planet_osm_admin WHERE admin_level BETWEEN 0 AND 4 - AND way && !BBOX! + AND way && !bbox! GROUP BY admin_level ORDER BY admin_level DESC ) AS admin_low_zoom @@ -1102,7 +1102,7 @@ Layer: admin_level FROM planet_osm_admin WHERE admin_level BETWEEN 0 AND 8 - AND way && !BBOX! + AND way && !bbox! GROUP BY admin_level ORDER BY admin_level DESC ) AS admin_mid_zoom @@ -1120,7 +1120,7 @@ Layer: admin_level FROM planet_osm_admin WHERE admin_level BETWEEN 0 AND 10 - AND way && !BBOX! + AND way && !bbox! GROUP BY admin_level ORDER BY admin_level DESC ) AS admin_high_zoom From 9741ef042d34746611977c761c2c8ce3c8e5826c Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sun, 3 Jul 2022 21:34:13 -0700 Subject: [PATCH 26/54] Remove unused .style file --- openstreetmap-carto.style | 55 --------------------------------------- 1 file changed, 55 deletions(-) delete mode 100644 openstreetmap-carto.style diff --git a/openstreetmap-carto.style b/openstreetmap-carto.style deleted file mode 100644 index 77134cb705..0000000000 --- a/openstreetmap-carto.style +++ /dev/null @@ -1,55 +0,0 @@ -# This is the osm2pgsql .style file for openstreetmap-carto. -# It is inteded to be used with openstreetmap-carto.lua and osm2pgsql Lua -# transforms. Full usage details are in INSTALL.md -# Among things, this means that the linear vs polygon distinction in this file -# doesn't matter, because that is set in the Lua and this file is only used for -# column names and types. - -# OsmType Tag DataType Flags -node,way access text linear -node,way addr:housename text linear -node,way addr:housenumber text linear -way addr:interpolation text linear -node,way admin_level text linear -node,way aerialway text linear -node,way aeroway text polygon -node,way amenity text polygon -node,way barrier text linear -way bicycle text linear -way bridge text linear -node,way boundary text linear -node,way building text polygon -way construction text linear -way covered text linear -way foot text linear -node,way highway text linear -node,way historic text polygon -way horse text linear -node,way junction text linear -node,way landuse text polygon -node,way layer int4 linear -node,way leisure text polygon -node,way lock text linear -node,way man_made text polygon -node,way military text polygon -node,way name text linear -node,way natural text polygon -node,way oneway text linear -node,way place text polygon -node,way power text polygon -node,way railway text linear -node,way ref text linear -node,way religion text linear -way route text linear -way service text linear -node,way shop text polygon -way surface text linear -node,way tourism text polygon -way tracktype text linear -way tunnel text linear -node,way water text polygon -node,way waterway text polygon -way way_area real linear # This is calculated during import - -# Columns defined in openstreetmap-carto.lua file -way z_order int4 linear From b2b253c8fc93cb69178a87b746890b8fff1ff60a Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sun, 3 Jul 2022 21:34:47 -0700 Subject: [PATCH 27/54] Refactor to take objects instead of object tags in remainder of add_* --- openstreetmap-carto.lua | 68 +++++++++++++++++------------------ scripts/lua/test.lua | 78 ++++++++++++++++++++--------------------- 2 files changed, 73 insertions(+), 73 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index bf472773b4..b3b46f299f 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -596,51 +596,51 @@ end phase2_admin_ways = {} -- TODO: Make add_* take object, not object.tags -function add_point(tags) - local cols = split_tags(tags, columns_map.point) - cols['layer'] = layer(tags['layer']) +function add_point(object) + local cols = split_tags(object.tags, columns_map.point) + cols['layer'] = layer(object.tags['layer']) tables.point:add_row(cols) end -function add_line(tags) - local cols = split_tags(tags, columns_map.line) - cols['layer'] = layer(tags['layer']) - cols['z_order'] = z_order(tags) +function add_line(object) + local cols = split_tags(object.tags, columns_map.line) + cols['layer'] = layer(object.tags['layer']) + cols['z_order'] = z_order(object.tags) cols.tags['layer'] = nil cols.way = { create = 'line', split_at = 100000 } tables.line:add_row(cols) end -function add_transport_line(tags) - local cols = split_tags(tags, columns_map.transport_line) - cols['layer'] = layer(tags['layer']) - cols['z_order'] = z_order(tags) +function add_transport_line(object) + local cols = split_tags(object.tags, columns_map.transport_line) + cols['layer'] = layer(object.tags['layer']) + cols['z_order'] = z_order(object.tags) cols.tags['layer'] = nil cols.way = { create = 'line', split_at = 100000 } tables.transport_line:add_row(cols) end -function add_roads(tags) - local cols = split_tags(tags, columns_map.roads) - cols['layer'] = layer(tags['layer']) - cols['z_order'] = z_order(tags) +function add_roads(object) + local cols = split_tags(object.tags, columns_map.roads) + cols['layer'] = layer(object.tags['layer']) + cols['z_order'] = z_order(object.tags) cols.tags['layer'] = nil cols.way = { create = 'line', split_at = 100000 } tables.roads:add_row(cols) end -function add_polygon(tags) - local cols = split_tags(tags, columns_map.polygon) - cols['layer'] = layer(tags['layer']) - cols['z_order'] = z_order(tags) +function add_polygon(object) + local cols = split_tags(object.tags, columns_map.polygon) + cols['layer'] = layer(object.tags['layer']) + cols['z_order'] = z_order(object.tags) cols.way = { create = 'area', split_at = nil } tables.polygon:add_row(cols) end -function add_transport_polygon(tags) - local cols = split_tags(tags, columns_map.transport_polygon) - cols['layer'] = layer(tags['layer']) - cols['z_order'] = z_order(tags) +function add_transport_polygon(object) + local cols = split_tags(object.tags, columns_map.transport_polygon) + cols['layer'] = layer(object.tags['layer']) + cols['z_order'] = z_order(object.tags) cols.way = { create = 'area', split_at = nil } tables.transport_polygon:add_row(cols) end @@ -661,7 +661,7 @@ function osm2pgsql.process_node(object) return end - add_point(object.tags) + add_point(object) end function osm2pgsql.process_way(object) @@ -678,20 +678,20 @@ function osm2pgsql.process_way(object) local area_tags = isarea(object.tags) if object.is_closed and area_tags then - add_polygon(object.tags) + add_polygon(object) if z_order(object.tags) ~= nil then - add_transport_polygon(object.tags) + add_transport_polygon(object) end else - add_line(object.tags) + add_line(object) if z_order(object.tags) ~= nil then - add_transport_line(object.tags) + add_transport_line(object) end if roads(object.tags) then - add_roads(object.tags) + add_roads(object) end end end @@ -705,19 +705,19 @@ function osm2pgsql.process_relation(object) return end if type == "boundary" or (type == "multipolygon" and object.tags["boundary"]) then - add_line(object.tags) + add_line(object) if roads(object.tags) then - add_roads(object.tags) + add_roads(object) end - add_polygon(object.tags) + add_polygon(object) elseif type == "multipolygon" then - add_polygon(object.tags) + add_polygon(object) if z_order(object.tags) ~= nil then - add_transport_polygon(object.tags) + add_transport_polygon(object) end elseif type == "route" then add_route(object) diff --git a/scripts/lua/test.lua b/scripts/lua/test.lua index 280ab29c8f..b8be191084 100644 --- a/scripts/lua/test.lua +++ b/scripts/lua/test.lua @@ -215,93 +215,93 @@ assert(deepcompare(tags, {natural = "tree"}), "mixed tags") print("TESTING: add_point") table_contents.planet_osm_point = {} -add_point({natural = "tree"}) +add_point({tags = {natural = "tree"}}) assert(deepcompare(table_contents.planet_osm_point[1], {natural = "tree", tags = {}}), "Tag with column") table_contents.planet_osm_point = {} -add_point({natural = "tree", foo = "bar"}) +add_point({tags = {natural = "tree", foo = "bar"}}) assert(deepcompare(table_contents.planet_osm_point[1], {natural = "tree", tags = {foo = "bar"}}), "Tag with column + hstore") table_contents.planet_osm_point = {} -add_point({foo = "bar"}) +add_point({tags = {foo = "bar"}}) assert(deepcompare(table_contents.planet_osm_point[1], {tags = {foo = "bar"}}), "hstore only") print("TESTING: add_line") table_contents.planet_osm_line = {} -add_line({highway = "road"}) +add_line({tags = {highway = "road"}}) assert(deepcompare(table_contents.planet_osm_line[1], {highway = "road", z_order = 330, tags = {}, way = { create = 'line', split_at = 100000 }}), "Tag with column") table_contents.planet_osm_line = {} -add_line({highway = "road", foo = "bar"}) +add_line({tags = {highway = "road", foo = "bar"}}) assert(deepcompare(table_contents.planet_osm_line[1], {highway = "road", z_order = 330, tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "Tag with column + hstore") table_contents.planet_osm_line = {} -add_line({highway = "road", foo = "bar", layer = "5"}) +add_line({tags = {highway = "road", foo = "bar", layer = "5"}}) assert(deepcompare(table_contents.planet_osm_line[1], {highway = "road", z_order = 330, layer = "5", tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "Tag with column + hstore + layer") table_contents.planet_osm_line = {} -add_line({foo = "bar"}) +add_line({tags = {foo = "bar"}}) assert(deepcompare(table_contents.planet_osm_line[1], {tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "hstore only") print("TESTING: add_transport_line") table_contents.planet_osm_transport_line = {} -add_transport_line({highway = "road"}) +add_transport_line({tags = {highway = "road"}}) assert(deepcompare(table_contents.planet_osm_transport_line[1], {highway = "road", z_order = 330, tags = {}, way = { create = 'line', split_at = 100000 }}), "Tag with column") table_contents.planet_osm_transport_line = {} -add_transport_line({highway = "road", foo = "bar"}) +add_transport_line({tags = {highway = "road", foo = "bar"}}) assert(deepcompare(table_contents.planet_osm_transport_line[1], {highway = "road", z_order = 330, tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "Tag with column + hstore") -- The z_order check where add_transport_line is called prevents this case from being hit in practice table_contents.planet_osm_transport_line = {} -add_transport_line({foo = "bar"}) +add_transport_line({tags = {foo = "bar"}}) assert(deepcompare(table_contents.planet_osm_transport_line[1], {tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "hstore only") table_contents.planet_osm_transport_line = {} -add_transport_line({highway = "road", foo = "bar", layer = "5"}) +add_transport_line({tags = {highway = "road", foo = "bar", layer = "5"}}) assert(deepcompare(table_contents.planet_osm_transport_line[1], {highway = "road", z_order = 330, layer = "5", tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "Tag with column + hstore + layer") print("TESTING: add_roads") table_contents.planet_osm_roads = {} -add_roads({highway = "road"}) +add_roads({tags = {highway = "road"}}) assert(deepcompare(table_contents.planet_osm_roads[1], {highway = "road", z_order = 330, tags = {}, way = { create = 'line', split_at = 100000 }}), "Tag with column") table_contents.planet_osm_roads = {} -add_roads({highway = "road", foo = "bar"}) +add_roads({tags = {highway = "road", foo = "bar"}}) assert(deepcompare(table_contents.planet_osm_roads[1], {highway = "road", z_order = 330, tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "Tag with column + hstore") table_contents.planet_osm_roads = {} -add_roads({highway = "road", layer = "5", foo = "bar"}) +add_roads({tags = {highway = "road", layer = "5", foo = "bar"}}) assert(deepcompare(table_contents.planet_osm_roads[1], {highway = "road", z_order = 330, layer = "5", tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "Tag with column + hstore") table_contents.planet_osm_roads = {} -add_roads({foo = "bar"}) +add_roads({tags = {foo = "bar"}}) assert(deepcompare(table_contents.planet_osm_roads[1], {tags = {foo = "bar"}, way = { create = 'line', split_at = 100000 }}), "hstore only") print("TESTING: add_polygon") table_contents.planet_osm_polygon = {} -add_polygon({natural = "wood"}) +add_polygon({tags = {natural = "wood"}}) assert(deepcompare(table_contents.planet_osm_polygon[1], {natural = "wood", tags = {}, way = { create = 'area', split_at = nil }}), "Tag with column") table_contents.planet_osm_polygon = {} -add_polygon({natural = "wood", foo = "bar"}) +add_polygon({tags = {natural = "wood", foo = "bar"}}) assert(deepcompare(table_contents.planet_osm_polygon[1], {natural = "wood", tags = {foo = "bar"}, way = { create = 'area', split_at = nil }}), "Tag with column + hstore") table_contents.planet_osm_polygon = {} -add_polygon({foo = "bar"}) +add_polygon({tags = {foo = "bar"}}) assert(deepcompare(table_contents.planet_osm_polygon[1], {tags = {foo = "bar"}, way = { create = 'area', split_at = nil }}), "hstore only") print("TESTING: add_transport_polygon") table_contents.planet_osm_transport_polygon = {} -add_transport_polygon({highway = "service"}) +add_transport_polygon({tags = {highway = "service"}}) assert(deepcompare(table_contents.planet_osm_transport_polygon[1], {highway = "service", z_order = 150, tags = {}, way = { create = 'area' }}), "Tag with column") table_contents.planet_osm_transport_polygon = {} -add_transport_polygon({highway = "service", foo = "bar"}) +add_transport_polygon({tags = {highway = "service", foo = "bar"}}) assert(deepcompare(table_contents.planet_osm_transport_polygon[1], {highway = "service", z_order = 150, tags = {foo = "bar"}, way = { create = 'area' }}), "Tag with column + hstore") table_contents.planet_osm_transport_polygon = {} -add_transport_polygon({foo = "bar"}) +add_transport_polygon({tags = {foo = "bar"}}) assert(deepcompare(table_contents.planet_osm_transport_polygon[1], {tags = {foo = "bar"}, way = { create = 'area', split_at = nil }}), "hstore only") print("TESTING: add_route") @@ -334,7 +334,7 @@ assert(deepcompare(table_contents.planet_osm_point, {}), "Deleted tag") -- in the output tables should be the same. A cleaner way would be to mock the add_* functions -- and then check if they are called. osm2pgsql.process_node({tags = {natural = "tree", foo = "bar"}}) -add_point({natural = "tree", foo = "bar"}) +add_point({tags = {natural = "tree", foo = "bar"}}) assert(deepcompare(table_contents.planet_osm_point[1], table_contents.planet_osm_point[2]), "Accepted tag") print("TESTING: osm2pgsql.process_way") @@ -360,8 +360,8 @@ assert(deepcompare(table_contents.planet_osm_transport_polygon, {}), "Deleted ta osm2pgsql.process_way({is_closed = false, tags = { highway = "road"}}) -- Add something to compare against. We know add_line and add_transport_line are okay since we tested them above -add_line({ highway = "road"}) -add_transport_line({ highway = "road"}) +add_line({tags = { highway = "road"}}) +add_transport_line({tags = { highway = "road"}}) assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_line[2]), "Line tag, open way, line table") assert(deepcompare(table_contents.planet_osm_transport_line[1], table_contents.planet_osm_transport_line[2]), "Line tag, open way, transport_line table") assert(deepcompare(table_contents.planet_osm_roads, {}), "Line tag, open way, roads table") @@ -371,9 +371,9 @@ table_contents.planet_osm_line = {} table_contents.planet_osm_transport_line = {} osm2pgsql.process_way({is_closed = true, tags = { highway = "motorway"}}) -add_line({highway = "motorway"}) -add_transport_line({ highway = "motorway"}) -add_roads({highway = "motorway"}) +add_line({tags = {highway = "motorway"}}) +add_transport_line({tags = { highway = "motorway"}}) +add_roads({tags = {highway = "motorway"}}) assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_line[2]), "Line tag, closed way, line table") assert(deepcompare(table_contents.planet_osm_transport_line[1], table_contents.planet_osm_transport_line[2]), "Line tag, closed way, transport_line table") assert(deepcompare(table_contents.planet_osm_roads[1], table_contents.planet_osm_roads[2]), "Line tag, closed way, roads table") @@ -384,7 +384,7 @@ table_contents.planet_osm_transport_line = {} table_contents.planet_osm_roads = {} osm2pgsql.process_way({is_closed = false, tags = { natural = "wood"}}) -add_line({natural = "wood"}) +add_line({tags = {natural = "wood"}}) assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_line[1]), "area tag, open way, line table") assert(deepcompare(table_contents.planet_osm_transport_line, {}), "area tag, open way, transport_line table") assert(deepcompare(table_contents.planet_osm_roads, {}), "area tag, open way, roads table") @@ -393,7 +393,7 @@ assert(deepcompare(table_contents.planet_osm_transport_polygon, {}), "area tag, table_contents.planet_osm_line = {} osm2pgsql.process_way({is_closed = true, tags = { natural = "wood"}}) -add_polygon({natural = "wood"}) +add_polygon({tags = {natural = "wood"}}) assert(deepcompare(table_contents.planet_osm_line, {}), "area tag, closed way, line table") assert(deepcompare(table_contents.planet_osm_transport_line, {}), "area tag, closed way, transport_line table") assert(deepcompare(table_contents.planet_osm_roads, {}), "area tag, closed way, roads table") @@ -402,8 +402,8 @@ assert(deepcompare(table_contents.planet_osm_transport_polygon, {}), "area tag, table_contents.planet_osm_polygon = {} osm2pgsql.process_way({is_closed = true, tags = { railway = "platform", area = "yes"}}) -add_polygon({railway = "platform", area = "yes"}) -add_transport_polygon({railway = "platform", area = "yes"}) +add_polygon({tags = {railway = "platform", area = "yes"}}) +add_transport_polygon({tags = {railway = "platform", area = "yes"}}) assert(deepcompare(table_contents.planet_osm_line, {}), "area tag, closed way, line table") assert(deepcompare(table_contents.planet_osm_transport_line, {}), "area tag, closed way, transport_line table") assert(deepcompare(table_contents.planet_osm_roads, {}), "area tag, closed way, roads table") @@ -440,8 +440,8 @@ assert(deepcompare(table_contents.planet_osm_polygon, {}), "Deleted tag polygon assert(deepcompare(table_contents.planet_osm_transport_polygon, {}), "Deleted tag transport_polygon with type") osm2pgsql.process_relation({tags = {type = "boundary", boundary = "foo"}}) -add_line({boundary = "foo"}) -add_polygon({boundary = "foo"}) +add_line({tags = {boundary = "foo"}}) +add_polygon({tags = {boundary = "foo"}}) assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_line[2]), "Boundary line") assert(deepcompare(table_contents.planet_osm_transport_line, {}), "Boundary transport_line") assert(deepcompare(table_contents.planet_osm_roads, {}), "Boundary roads") @@ -451,9 +451,9 @@ table_contents.planet_osm_line = {} table_contents.planet_osm_polygon = {} osm2pgsql.process_relation({tags = {type = "boundary", boundary = "administrative"}}) -add_line({boundary = "administrative"}) -add_roads({boundary = "administrative"}) -add_polygon({boundary = "administrative"}) +add_line({tags = {boundary = "administrative"}}) +add_roads({tags = {boundary = "administrative"}}) +add_polygon({tags = {boundary = "administrative"}}) assert(deepcompare(table_contents.planet_osm_line[1], table_contents.planet_osm_line[2]), "admin boundary line") assert(deepcompare(table_contents.planet_osm_transport_line, {}), "admin boundary transport_line") assert(deepcompare(table_contents.planet_osm_roads[1], table_contents.planet_osm_roads[2]), "admin boundary roads") @@ -464,7 +464,7 @@ table_contents.planet_osm_polygon = {} table_contents.planet_osm_roads = {} osm2pgsql.process_relation({tags = {type = "multipolygon", natural = "tree"}}) -add_polygon({natural = "tree"}) +add_polygon({tags = {natural = "tree"}}) assert(deepcompare(table_contents.planet_osm_line, {}), "MP line") assert(deepcompare(table_contents.planet_osm_transport_line, {}), "MP transport_line") assert(deepcompare(table_contents.planet_osm_roads, {}), "MP roads") @@ -473,8 +473,8 @@ assert(deepcompare(table_contents.planet_osm_transport_polygon, {}), "MP transpo table_contents.planet_osm_polygon = {} osm2pgsql.process_relation({tags = {type = "multipolygon", highway = "motorway"}}) -add_polygon({highway = "motorway"}) -add_transport_polygon({highway = "motorway"}) +add_polygon({tags = {highway = "motorway"}}) +add_transport_polygon({tags = {highway = "motorway"}}) assert(deepcompare(table_contents.planet_osm_line, {}), "MP highway line") assert(deepcompare(table_contents.planet_osm_transport_line, {}), "MP highway transport_line") assert(deepcompare(table_contents.planet_osm_roads, {}), "MP highway roads") From 25a5a3746231aaf2c750d44b2a8748da28088025 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sun, 3 Jul 2022 23:18:33 -0700 Subject: [PATCH 28/54] Move all layer handling to split_tags --- openstreetmap-carto.lua | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index b3b46f299f..4798d76da6 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -583,6 +583,7 @@ end -- @return columns, hstore tags function split_tags(tags, tag_map) local cols = {tags = {}} + for key, value in pairs(tags) do if tag_map[key] then cols[key] = value @@ -590,6 +591,11 @@ function split_tags(tags, tag_map) cols.tags[key] = value end end + + -- layer is a special tag. After setting the columns, we have to go back and set it after making sure it's an integer, and remove it from tags + cols['layer'] = layer(tags['layer']) + cols.tags['layer'] = nil + return cols end @@ -598,40 +604,32 @@ phase2_admin_ways = {} -- TODO: Make add_* take object, not object.tags function add_point(object) local cols = split_tags(object.tags, columns_map.point) - cols['layer'] = layer(object.tags['layer']) tables.point:add_row(cols) end function add_line(object) local cols = split_tags(object.tags, columns_map.line) - cols['layer'] = layer(object.tags['layer']) cols['z_order'] = z_order(object.tags) - cols.tags['layer'] = nil cols.way = { create = 'line', split_at = 100000 } tables.line:add_row(cols) end function add_transport_line(object) local cols = split_tags(object.tags, columns_map.transport_line) - cols['layer'] = layer(object.tags['layer']) cols['z_order'] = z_order(object.tags) - cols.tags['layer'] = nil cols.way = { create = 'line', split_at = 100000 } tables.transport_line:add_row(cols) end function add_roads(object) local cols = split_tags(object.tags, columns_map.roads) - cols['layer'] = layer(object.tags['layer']) cols['z_order'] = z_order(object.tags) - cols.tags['layer'] = nil cols.way = { create = 'line', split_at = 100000 } tables.roads:add_row(cols) end function add_polygon(object) local cols = split_tags(object.tags, columns_map.polygon) - cols['layer'] = layer(object.tags['layer']) cols['z_order'] = z_order(object.tags) cols.way = { create = 'area', split_at = nil } tables.polygon:add_row(cols) @@ -639,7 +637,6 @@ end function add_transport_polygon(object) local cols = split_tags(object.tags, columns_map.transport_polygon) - cols['layer'] = layer(object.tags['layer']) cols['z_order'] = z_order(object.tags) cols.way = { create = 'area', split_at = nil } tables.transport_polygon:add_row(cols) From 1744b84da15dd8aea78f7c608f35250ac734130c Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sun, 3 Jul 2022 23:55:02 -0700 Subject: [PATCH 29/54] Avoid hard-coding construction z_order --- openstreetmap-carto.lua | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 4798d76da6..4ca4d28592 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -422,7 +422,7 @@ delete_prefixes = { } -- Big table for z_order and roads status for certain tags. z=0 is turned into --- nil by the z_order function +-- nil by the z_order function. Road values are divided by 10 for construction, so must be multiples of 10 local roads_info = { highway = { motorway = {z = 380, roads = true}, @@ -495,7 +495,8 @@ function z_order(tags) if tags["construction"] and roads_info["highway"][tags["construction"]] then z = math.max(z, roads_info["highway"][tags["construction"]].z/10) else - z = math.max(z, 33) + -- For unknown roads, assume highway=road + z = math.max(z, roads_info["highway"]["road"].z/10) end end From 26f234cc9b41fb223abdbfe8cf92553dec9c6d33 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sat, 9 Jul 2022 16:46:53 -0700 Subject: [PATCH 30/54] Code style, comment fixes --- CONTRIBUTING.md | 7 +++++++ openstreetmap-carto.lua | 26 ++++++++++++++++++-------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 725ccf7c3e..89b90cd9d4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -134,6 +134,13 @@ Because SQL within JSON or YAML will not generally be syntax highlighted, indent * Hstore queries tested for NULL should be enclosed in parentheses, e.g. `(tags->'foo') IS NULL`. * To check if a tag is in the tags hstore, use `tags @> 'foo=>bar'`, relying on automatic conversion from `text` to `hstore`. +## Lua style guidelines + +* Four space indents, no tabs +* No external dependencies +* Use `a["name"]` where name is a tag key, and `a.name` otherwise. + + ## Map icon guidelines * All new icons must be SVG format only. The SVG must be saved as standards compliant SVG without any proprietary tags. In Inkscape software, you will need to "Save As..." and choose the format Optimized SVG (preferable) or Plain SVG. diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 4ca4d28592..038c20458d 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -1,10 +1,11 @@ -- For documentation of Lua tag transformations, see: --- https://github.com/openstreetmap/osm2pgsql/blob/master/docs/lua.md +-- https://osm2pgsql.org/doc/manual.html#the-flex-output local tables = {} --- A list of columns per table, replacing the osm2pgsql .style file --- These need to be ordered, so that means a list +-- A list of text columns per table, replacing the osm2pgsql .style file +-- These are inserted into the table list of columns in the same order as the lists here +-- Non-text columns are defined in col_definitions. Because the local pg_cols = { point = { 'access', @@ -123,9 +124,11 @@ local pg_cols = { 'route', 'ref', 'network' - } + }, + admin = {} } +-- The roads and polygon columns are the same as the line columns pg_cols.roads = pg_cols.line pg_cols.polygon = pg_cols.line @@ -544,15 +547,22 @@ function isarea (tags) return false end ---- Normalizes layer tags +--- Normalizes layer tags to integers -- @param v The layer tag value --- @return An integer for the layer tag +-- @return The input value if it is an integer between -100 and 100, or nil otherwise function layer (v) - return v and string.find(v, "^-?%d+$") and tonumber(v) < 100 and tonumber(v) > -100 and v or nil + return v and -- Check that layer is non-nil + string.find(v, "^-?%d+$") and tonumber(v) < 100 and tonumber(v) > -100 and -- Enforce numeric and range + v or nil -- return v, or nil if fails to match end +--- Normalizes admin_level tags +-- @param v The admin_level tag value +-- @return The input value if it is an integer between 0 and 100, or nil otherwise function admin_level (v) - return v and string.find(v, "^-?%d+$") and tonumber(v) < 100 and tonumber(v) > 0 and v or nil + return v and -- Check that admin_level is non-nil + string.find(v, "^%d+$") and tonumber(v) < 100 and tonumber(v) > 0 and -- Enforce numeric range + v or nil -- return v, or nil if fails to match end --- Clean tags of deleted tags From efb05f6abd6b352f5af6968db313be96ccff830c Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sat, 9 Jul 2022 16:47:16 -0700 Subject: [PATCH 31/54] Don't handle boundary MPs --- openstreetmap-carto.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 038c20458d..2e5893b07e 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -712,7 +712,7 @@ function osm2pgsql.process_relation(object) if clean_tags(object.tags) then return end - if type == "boundary" or (type == "multipolygon" and object.tags["boundary"]) then + if type == "boundary" then add_line(object) if roads(object.tags) then From fc0d624851e3ac3a8799ba68582a67b46da5533a Mon Sep 17 00:00:00 2001 From: mboeringa Date: Sun, 10 Jul 2022 10:34:45 +0200 Subject: [PATCH 32/54] Suggested change to code comment Relates to the column definitions. --- openstreetmap-carto.lua | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 8c7c6b0341..5c17c6c93a 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -3,9 +3,10 @@ local tables = {} --- A list of text columns per table, replacing the osm2pgsql .style file --- These are inserted into the table list of columns in the same order as the lists here --- Non-text columns are defined in col_definitions. Because the +-- A list of text columns per table, replacing the legacy osm2pgsql 'pgsql output' .style file. +-- These are inserted into the table list of columns in the same order as lists here, +-- and thus determine column order in the final tables. +-- Non-text columns are defined in col_definitions. local pg_cols = { point = { 'access', From dba7c3d4c5d321943a8eeacf5d79510b35eeb886 Mon Sep 17 00:00:00 2001 From: mboeringa Date: Sun, 10 Jul 2022 10:48:50 +0200 Subject: [PATCH 33/54] Suggested change to code comment (2) --- openstreetmap-carto.lua | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 8c7c6b0341..c3ecccd2a6 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -424,8 +424,12 @@ delete_prefixes = { 'mvdgis:' } --- Big table for z_order and roads status for certain tags. z=0 is turned into --- nil by the z_order function. Road values are divided by 10 for construction, so must be multiples of 10 +-- Big table for z_order and roads status for certain tags. +-- The road status (true/false) determines whether or not the feature will be +-- included in the legacy 'roads' table. +-- z=0 is turned into nil by the z_order function. +-- Road z values are divided by 10 for objects additionally tagged as construction, +-- so must be multiples of 10. local roads_info = { highway = { motorway = {z = 380, roads = true}, From 684797e17c7cb97efc5e2f7255ef5cd1b19dfe22 Mon Sep 17 00:00:00 2001 From: mboeringa Date: Sun, 10 Jul 2022 11:33:50 +0200 Subject: [PATCH 34/54] Fix missing parameter definitions in code comments, fix case of boolean --- openstreetmap-carto.lua | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 5c17c6c93a..cd56b51c4f 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -567,7 +567,8 @@ function admin_level (v) end --- Clean tags of deleted tags --- @return True if no tags are left after cleaning +-- @param tags OSM tags +-- @return true if no tags are left after cleaning function clean_tags(tags) -- Short-circuit for untagged objects if next(tags) == nil then @@ -592,6 +593,8 @@ function clean_tags(tags) end --- Splits a tag into tags and hstore tags +-- @param tags OSM tags +-- @param tag_map Lua table that contains the OSM tags that will get a dedicated column -- @return columns, hstore tags function split_tags(tags, tag_map) local cols = {tags = {}} From fa2f014d46c4807f1b92c698b5e37944fac04bc0 Mon Sep 17 00:00:00 2001 From: mboeringa Date: Sun, 10 Jul 2022 14:53:48 +0200 Subject: [PATCH 35/54] Fix to comment based on feedback --- openstreetmap-carto.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index c3ecccd2a6..8b1350fa0e 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -428,8 +428,8 @@ delete_prefixes = { -- The road status (true/false) determines whether or not the feature will be -- included in the legacy 'roads' table. -- z=0 is turned into nil by the z_order function. --- Road z values are divided by 10 for objects additionally tagged as construction, --- so must be multiples of 10. +-- Road z values are divided by 10 for objects tagged as highway=construction, +-- construction=[HIGHWAY_CLASS], so must be multiples of 10. local roads_info = { highway = { motorway = {z = 380, roads = true}, From 41545d88eb61b2ea1b2f70ace3c8d869a5321c63 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sun, 10 Jul 2022 20:58:23 -0700 Subject: [PATCH 36/54] Remove missed merge conflict --- INSTALL.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 92e9ab6e7a..7ddaa9b41c 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -101,16 +101,8 @@ To display any map a database containing OpenStreetMap data and some utilities a * [PostgreSQL](https://www.postgresql.org/) * [PostGIS](https://postgis.net/) -<<<<<<< HEAD * [osm2pgsql](https://github.com/openstreetmap/osm2pgsql#installing) 1.4.1 or later to import your data into a PostGIS database * Python 3 with the psycopg2, yaml, and requests libraries (`python3-psycopg2` `python3-yaml` `python3-requests` packages on Debian-derived systems) -||||||| db9abe26 -* [osm2pgsql](https://github.com/openstreetmap/osm2pgsql#installing) to [import your data](https://switch2osm.org/loading-osm-data/) into a PostGIS database -* Python 3 with the psycopg2, yaml, and requests libraries (`python3-psycopg2` `python3-yaml` `python3-requests` packages on Debian-derived systems) -======= -* [osm2pgsql](https://github.com/openstreetmap/osm2pgsql#installing) to [import your data](https://switch2osm.org/serving-tiles/updating-as-people-edit/) into a PostGIS database -* Python 3 with the psycopg2, yaml, and requests libraries (`python3-psycopg2`, `python3-yaml`, `python3-requests` packages on Debian-derived systems) ->>>>>>> v5.5.0 * `ogr2ogr` for loading shapefiles into the database (`gdal-bin` on Debian-derived systems) ### Optional development dependencies From 568083280786c51cc49078a7a478d6f7f1a2382b Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sun, 10 Jul 2022 20:59:38 -0700 Subject: [PATCH 37/54] Specify minimum PostGIS version --- INSTALL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index 7ddaa9b41c..1dc4b35a83 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -100,7 +100,7 @@ For development, a style design studio is needed. To display any map a database containing OpenStreetMap data and some utilities are required * [PostgreSQL](https://www.postgresql.org/) -* [PostGIS](https://postgis.net/) +* [PostGIS](https://postgis.net/) 3.1.0 or later with GEOS 3.9.0 or later * [osm2pgsql](https://github.com/openstreetmap/osm2pgsql#installing) 1.4.1 or later to import your data into a PostGIS database * Python 3 with the psycopg2, yaml, and requests libraries (`python3-psycopg2` `python3-yaml` `python3-requests` packages on Debian-derived systems) * `ogr2ogr` for loading shapefiles into the database (`gdal-bin` on Debian-derived systems) From 3283fd8eb12f1fd07e8788977897b8ed33c820af Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sun, 10 Jul 2022 21:12:10 -0700 Subject: [PATCH 38/54] Document flex callbacks and supporting functions --- openstreetmap-carto.lua | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 70513c98da..06fbd9f1c4 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -620,12 +620,17 @@ end phase2_admin_ways = {} --- TODO: Make add_* take object, not object.tags +-- Processing callbacks (https://osm2pgsql.org/doc/manual.html#processing-callbacks) and functions that directly support them + +--- Add an object to the point table +-- @param object parameter table supplied by osm2pgsql function add_point(object) local cols = split_tags(object.tags, columns_map.point) tables.point:add_row(cols) end +--- Add an object to the line table +-- @param object parameter table supplied by osm2pgsql function add_line(object) local cols = split_tags(object.tags, columns_map.line) cols['z_order'] = z_order(object.tags) @@ -633,6 +638,8 @@ function add_line(object) tables.line:add_row(cols) end +--- Add an object to the transport_line table +-- @param object parameter table supplied by osm2pgsql function add_transport_line(object) local cols = split_tags(object.tags, columns_map.transport_line) cols['z_order'] = z_order(object.tags) @@ -640,6 +647,8 @@ function add_transport_line(object) tables.transport_line:add_row(cols) end +--- Add an object to the roads table +-- @param object parameter table supplied by osm2pgsql function add_roads(object) local cols = split_tags(object.tags, columns_map.roads) cols['z_order'] = z_order(object.tags) @@ -647,6 +656,8 @@ function add_roads(object) tables.roads:add_row(cols) end +--- Add an object to the polygon table +-- @param object parameter table supplied by osm2pgsql function add_polygon(object) local cols = split_tags(object.tags, columns_map.polygon) cols['z_order'] = z_order(object.tags) @@ -654,6 +665,8 @@ function add_polygon(object) tables.polygon:add_row(cols) end +--- Add an object to the transport_polygon table +-- @param object parameter table supplied by osm2pgsql function add_transport_polygon(object) local cols = split_tags(object.tags, columns_map.transport_polygon) cols['z_order'] = z_order(object.tags) @@ -661,6 +674,8 @@ function add_transport_polygon(object) tables.transport_polygon:add_row(cols) end +--- Add an object to the route table +-- @param object parameter table supplied by osm2pgsql function add_route(object) for i, member in ipairs(object.members) do if member.type == 'w' then From 912c7ae417ece59c7b7ebf524af7e84e5183bf39 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sun, 10 Jul 2022 21:19:42 -0700 Subject: [PATCH 39/54] Reformat layer and admin_level checks --- openstreetmap-carto.lua | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 06fbd9f1c4..986310d1e2 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -557,8 +557,7 @@ end -- @return The input value if it is an integer between -100 and 100, or nil otherwise function layer (v) return v and -- Check that layer is non-nil - string.find(v, "^-?%d+$") and tonumber(v) < 100 and tonumber(v) > -100 and -- Enforce numeric and range - v or nil -- return v, or nil if fails to match + string.find(v, "^-?%d+$") and tonumber(v) < 100 and tonumber(v) > -100 and v or nil -- Enforce numeric limits and return v, or nil if fails to meet limits end --- Normalizes admin_level tags @@ -566,8 +565,7 @@ end -- @return The input value if it is an integer between 0 and 100, or nil otherwise function admin_level (v) return v and -- Check that admin_level is non-nil - string.find(v, "^%d+$") and tonumber(v) < 100 and tonumber(v) > 0 and -- Enforce numeric range - v or nil -- return v, or nil if fails to match + string.find(v, "^%d+$") and tonumber(v) < 100 and tonumber(v) > 0 and v or nil -- Enforce numeric limits and return v, or nil if fails to meet limits end --- Clean tags of deleted tags From 9c4c9c2fdc9af1e6d3fd3f4e336f4b533d63fa0e Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Wed, 13 Jul 2022 17:34:51 -0700 Subject: [PATCH 40/54] Treat unknown waterway tags as linestrings --- openstreetmap-carto.lua | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 986310d1e2..73fd0863d4 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -275,7 +275,6 @@ local polygon_keys = { 'shop', 'tourism', 'water', - 'waterway', 'wetland' } @@ -288,8 +287,7 @@ local linestring_values = { man_made = {breakwater = true, cutline = true, embankment = true, groyne = true, pipeline = true}, natural = {cliff = true, earth_bank = true, tree_row = true, ridge = true, arete = true}, power = {cable = true, line = true, minor_line = true}, - tourism = {yes = true}, - waterway = {canal = true, derelict_canal = true, ditch = true, drain = true, river = true, stream = true, tidal_channel = true, wadi = true, weir = true} + tourism = {yes = true} } -- Objects with any of the following key/value combinations will be treated as polygon @@ -298,7 +296,8 @@ local polygon_values = { boundary = {aboriginal_lands = true, national_park = true, protected_area= true}, highway = {services = true, rest_area = true}, junction = {yes = true}, - railway = {station = true} + railway = {station = true}, + waterway = {dock = true, boatyard = true, fuel = true, riverbank = true} } -- The following keys will be deleted From 40683e128681e24593fd3cd2dfc8e6c8bdaee6a7 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sat, 16 Jul 2022 15:22:39 -0700 Subject: [PATCH 41/54] Refactor phase 2 data structures By avoid multi-level tables, some memory can be saved for each admin way --- openstreetmap-carto.lua | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 73fd0863d4..e1e67204fe 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -615,7 +615,8 @@ function split_tags(tags, tag_map) return cols end -phase2_admin_ways = {} +phase2_admin_ways_level = {} +phase2_admin_ways_parents = {} -- Processing callbacks (https://osm2pgsql.org/doc/manual.html#processing-callbacks) and functions that directly support them @@ -695,9 +696,10 @@ end function osm2pgsql.process_way(object) if osm2pgsql.stage == 2 then -- Stage two processing is called on ways that are part of admin boundary relations - local props = phase2_admin_ways[object.id] - if props ~= nil then - tables.admin:add_row({admin_level = props.level, multiple_relations = (props.parents > 1), geom = { create = 'line' }}) + if phase2_admin_ways_level[object.id] then + tables.admin:add_row({admin_level = phase2_admin_ways_level[object.id], + multiple_relations = (phase2_admin_ways_parents[object.id] > 1), + geom = { create = 'line' }}) end end if clean_tags(object.tags) then @@ -759,13 +761,15 @@ function osm2pgsql.select_relation_members(relation) if admin ~= nil then for _, ref in ipairs(osm2pgsql.way_member_ids(relation)) do -- Store the lowest admin_level, and how many relations it used in - if phase2_admin_ways[ref] == nil then - phase2_admin_ways[ref] = {level = admin, parents = 1} + if not phase2_admin_ways_level[ref] then + phase2_admin_ways_level[ref] = admin + phase2_admin_ways_parents[ref] = 1 else - if phase2_admin_ways[ref].level == admin then - phase2_admin_ways[ref].parents = phase2_admin_ways[ref].parents + 1 - elseif admin < phase2_admin_ways[ref].level then - phase2_admin_ways[ref] = {level = admin, parents = 1} + if phase2_admin_ways_level[ref] == admin then + phase2_admin_ways_parents[ref] = phase2_admin_ways_parents[ref] + 1 + elseif admin < phase2_admin_ways_level[ref] then + phase2_admin_ways_level[ref] = admin + phase2_admin_ways_parents[ref] = 1 end end end From de95deea83ad62f34df19ca4337388b8bc0d91c0 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sat, 16 Jul 2022 15:24:33 -0700 Subject: [PATCH 42/54] Correct geometry column name --- openstreetmap-carto.lua | 2 +- scripts/lua/test.lua | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index e1e67204fe..1d23e54d77 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -699,7 +699,7 @@ function osm2pgsql.process_way(object) if phase2_admin_ways_level[object.id] then tables.admin:add_row({admin_level = phase2_admin_ways_level[object.id], multiple_relations = (phase2_admin_ways_parents[object.id] > 1), - geom = { create = 'line' }}) + way = { create = 'line' }}) end end if clean_tags(object.tags) then diff --git a/scripts/lua/test.lua b/scripts/lua/test.lua index b8be191084..1af40febb5 100644 --- a/scripts/lua/test.lua +++ b/scripts/lua/test.lua @@ -561,12 +561,12 @@ end -- Because everything is done in a fixed order we can use that to figure out which row is which. This is important because the add_row method doesn't take in the osm_id, and osm2pgsql tracks it separately -assert(deepcompare(table_contents.planet_osm_admin[1], {admin_level = 2, multiple_relations = true, geom = {create = "line" } }), "row 1") -assert(deepcompare(table_contents.planet_osm_admin[2], {admin_level = 2, multiple_relations = true, geom = {create = "line" } }), "row 2") -assert(deepcompare(table_contents.planet_osm_admin[3], {admin_level = 2, multiple_relations = false, geom = {create = "line" } }), "row 3") -assert(deepcompare(table_contents.planet_osm_admin[4], {admin_level = 4, multiple_relations = false, geom = {create = "line" } }), "row 4") -assert(deepcompare(table_contents.planet_osm_admin[5], {admin_level = 2, multiple_relations = false, geom = {create = "line" } }), "row 5") -assert(deepcompare(table_contents.planet_osm_admin[6], {admin_level = 2, multiple_relations = false, geom = {create = "line" } }), "row 6") -assert(deepcompare(table_contents.planet_osm_admin[7], {admin_level = 2, multiple_relations = true, geom = {create = "line" } }), "row 7") -assert(deepcompare(table_contents.planet_osm_admin[8], {admin_level = 2, multiple_relations = true, geom = {create = "line" } }), "row 8") -assert(deepcompare(table_contents.planet_osm_admin[9], {admin_level = 2, multiple_relations = false, geom = {create = "line" } }), "row 9") +assert(deepcompare(table_contents.planet_osm_admin[1], {admin_level = 2, multiple_relations = true, way = {create = "line" } }), "row 1") +assert(deepcompare(table_contents.planet_osm_admin[2], {admin_level = 2, multiple_relations = true, way = {create = "line" } }), "row 2") +assert(deepcompare(table_contents.planet_osm_admin[3], {admin_level = 2, multiple_relations = false, way = {create = "line" } }), "row 3") +assert(deepcompare(table_contents.planet_osm_admin[4], {admin_level = 4, multiple_relations = false, way = {create = "line" } }), "row 4") +assert(deepcompare(table_contents.planet_osm_admin[5], {admin_level = 2, multiple_relations = false, way = {create = "line" } }), "row 5") +assert(deepcompare(table_contents.planet_osm_admin[6], {admin_level = 2, multiple_relations = false, way = {create = "line" } }), "row 6") +assert(deepcompare(table_contents.planet_osm_admin[7], {admin_level = 2, multiple_relations = true, way = {create = "line" } }), "row 7") +assert(deepcompare(table_contents.planet_osm_admin[8], {admin_level = 2, multiple_relations = true, way = {create = "line" } }), "row 8") +assert(deepcompare(table_contents.planet_osm_admin[9], {admin_level = 2, multiple_relations = false, way = {create = "line" } }), "row 9") From f8117a3d565295c8380d8c1b6bad9708e18687d1 Mon Sep 17 00:00:00 2001 From: mboeringa Date: Mon, 9 Jan 2023 21:14:25 +0100 Subject: [PATCH 43/54] Fix for #3504 Remove "virtual" border lines on the antimeridian by excluding all features tagged as closure segment. --- openstreetmap-carto.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 1d23e54d77..4fb59aecc7 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -696,7 +696,7 @@ end function osm2pgsql.process_way(object) if osm2pgsql.stage == 2 then -- Stage two processing is called on ways that are part of admin boundary relations - if phase2_admin_ways_level[object.id] then + if object.tags.closure_segment ~= 'yes' and phase2_admin_ways_level[object.id] then tables.admin:add_row({admin_level = phase2_admin_ways_level[object.id], multiple_relations = (phase2_admin_ways_parents[object.id] > 1), way = { create = 'line' }}) From aa9b75a8927dde7eaf3e5aedca5aee74d76bbe7e Mon Sep 17 00:00:00 2001 From: mboeringa Date: Tue, 10 Jan 2023 18:50:14 +0100 Subject: [PATCH 44/54] Adding test data for removal of "virtual" borders on the antimeridian Adding test data for removal of "virtual" borders on the antimeridian caused by the technical need for ways with "closure_segment=yes" in boundary relations in OpenStreetMap to deal with the antimeridian, as relations may not cross it. --- scripts/lua/test.lua | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/lua/test.lua b/scripts/lua/test.lua index 1af40febb5..a1c86c2bee 100644 --- a/scripts/lua/test.lua +++ b/scripts/lua/test.lua @@ -487,7 +487,7 @@ table_contents.planet_osm_transport_polygon = {} --[[ This sets up an - 1. admin_level=2 relation with ways 1, 2, 3, 5, 7, 8, 9 + 1. admin_level=2 relation with ways 1, 2, 3, 5, 7, 8, 9, 10 2. admin_level=2 relation with ways 1, 2, 6, 7, 8 3. admin_level=4 relation with ways 1, 3, 4 @@ -505,7 +505,8 @@ local test_ways = { { id = 6, tags = { boundary = "administrative", admin_level = "2"} }, { id = 7, tags = { boundary = "administrative", admin_level = "3"} }, -- incorrect tags { id = 8, tags = { } }, -- incorrect tags - { id = 9, tags = { boundary = "administrative", admin_level = "2", highway = "road"} } + { id = 9, tags = { boundary = "administrative", admin_level = "2", highway = "road"} }, + { id = 10, tags = { boundary = "administrative", admin_level = "2", closure_segment = "yes"} } -- closure segments are used on the antimeridian } -- add another way that isn't part of a relation local test_relations = { @@ -517,7 +518,8 @@ local test_relations = { {type = 'w', ref = 5, role = "outer"}, {type = 'w', ref = 7, role = "outer"}, {type = 'w', ref = 8, role = "outer"}, - {type = 'w', ref = 9, role = "outer"} } }, + {type = 'w', ref = 9, role = "outer"} + {type = 'w', ref = 10, role = "outer"} } }, { id = 2, tags = {type = "boundary", boundary = "administrative", admin_level = "2"}, members = { {type = 'w', ref = 1, role = "outer"}, {type = 'w', ref = 2, role = "outer"}, From 598116f7414f78699de0811d7e5714ca02e1853e Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Thu, 19 Jan 2023 14:09:52 -0800 Subject: [PATCH 45/54] Switch to flex output in CI --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 756a79de1b..716f8f4eb3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,9 +39,9 @@ jobs: - name: Wait for database run : sudo pg_ctlcluster 14 main start; until pg_isready; do sleep 0.5; done - name: Setup database - run: sudo -i -u postgres createuser -s $USER && createdb -E utf8 gis && psql -Xq -d gis -c "CREATE EXTENSION postgis; CREATE EXTENSION hstore;" + run: sudo -i -u postgres createuser -s $USER && createdb -E utf8 flex && psql -Xq -d flex -c "CREATE EXTENSION postgis; CREATE EXTENSION hstore;" - name: Import empty file run: | - osm2pgsql -G --hstore --style openstreetmap-carto.style --tag-transform-script openstreetmap-carto.lua -d gis -r xml <(echo '') + osm2pgsql --output flex --style openstreetmap-carto.lua -d flex -r xml <(echo '') - name: Create indexes - run: psql -1Xq -v ON_ERROR_STOP=1 -d gis -f indexes.sql + run: psql -1Xq -v ON_ERROR_STOP=1 -d flex -f indexes.sql From 8623e4b18470144f2f3b414bb56f877abf682424 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Fri, 24 Feb 2023 21:33:52 -0800 Subject: [PATCH 46/54] Rework layer/admin conversion expressions to be clearer --- openstreetmap-carto.lua | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 1d23e54d77..ccb3f7b9d3 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -555,16 +555,18 @@ end -- @param v The layer tag value -- @return The input value if it is an integer between -100 and 100, or nil otherwise function layer (v) - return v and -- Check that layer is non-nil - string.find(v, "^-?%d+$") and tonumber(v) < 100 and tonumber(v) > -100 and v or nil -- Enforce numeric limits and return v, or nil if fails to meet limits + if v and string.find(v, "^-?%d+$") and tonumber(v) < 100 and tonumber(v) > -100 then -- check if value exists, is numeric, and is in range + return v + end end --- Normalizes admin_level tags -- @param v The admin_level tag value -- @return The input value if it is an integer between 0 and 100, or nil otherwise function admin_level (v) - return v and -- Check that admin_level is non-nil - string.find(v, "^%d+$") and tonumber(v) < 100 and tonumber(v) > 0 and v or nil -- Enforce numeric limits and return v, or nil if fails to meet limits + if v and string.find(v, "^%d+$") and tonumber(v) < 100 and tonumber(v) > 0 then + return v + end end --- Clean tags of deleted tags From 4d5abbd9c1210c7049faa9c61276d201fc805422 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Fri, 24 Feb 2023 21:38:57 -0800 Subject: [PATCH 47/54] Fix lua syntax error --- scripts/lua/test.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lua/test.lua b/scripts/lua/test.lua index a1c86c2bee..4ae1b79c48 100644 --- a/scripts/lua/test.lua +++ b/scripts/lua/test.lua @@ -518,7 +518,7 @@ local test_relations = { {type = 'w', ref = 5, role = "outer"}, {type = 'w', ref = 7, role = "outer"}, {type = 'w', ref = 8, role = "outer"}, - {type = 'w', ref = 9, role = "outer"} + {type = 'w', ref = 9, role = "outer"}, {type = 'w', ref = 10, role = "outer"} } }, { id = 2, tags = {type = "boundary", boundary = "administrative", admin_level = "2"}, members = { {type = 'w', ref = 1, role = "outer"}, From ea8cfe005b5aaa2029306c0f4c6af8d35e795543 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Fri, 24 Feb 2023 22:33:09 -0800 Subject: [PATCH 48/54] Fix merge errors in docs --- INSTALL.md | 30 +++--------------------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 85a7955570..344d5f512b 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -14,30 +14,14 @@ createdb flex Enable PostGIS and hstore extensions with -<<<<<<< HEAD -``` -psql -d flex -c 'CREATE EXTENSION postgis; CREATE EXTENSION hstore;' -||||||| 117df85a -``` -psql -d gis -c 'CREATE EXTENSION postgis; CREATE EXTENSION hstore;' -======= ```sh -psql -d gis -c 'CREATE EXTENSION postgis; CREATE EXTENSION hstore;' ->>>>>>> v5.7.0 +psql -d flex -c 'CREATE EXTENSION postgis; CREATE EXTENSION hstore;' ``` then grab some OSM data. It's probably easiest to grab an PBF of OSM data from [Geofabrik](https://download.geofabrik.de/). Once you've done that, import with osm2pgsql: -<<<<<<< HEAD -``` -osm2pgsql --output flex --style openstreetmap-carto.lua -d flex ~/path/to/data.osm.pbf -||||||| 117df85a -``` -osm2pgsql -G --hstore --style openstreetmap-carto.style --tag-transform-script openstreetmap-carto.lua -d gis ~/path/to/data.osm.pbf -======= ```sh -osm2pgsql -G --hstore --style openstreetmap-carto.style --tag-transform-script openstreetmap-carto.lua -d gis ~/path/to/data.osm.pbf ->>>>>>> v5.7.0 +osm2pgsql --output flex --style openstreetmap-carto.lua -d flex ~/path/to/data.osm.pbf ``` You can find a more detailed guide to setting up a database and loading data with osm2pgsql at [switch2osm.org](https://switch2osm.org/serving-tiles/manually-building-a-tile-server-16-04-2-lts/). @@ -53,16 +37,8 @@ JIT can be disabled with `psql -d gis -c 'ALTER SYSTEM SET jit=off;' -c 'SELECT ### Custom indexes Custom indexes are required for rendering performance and are essential on full planet databases. These are generated by the `scripts/indexes.py` script, see `scripts/indexes.py --help` for various advanced options, but the command below will work to create the indexes on a new installation: -<<<<<<< HEAD -``` -psql -d flex -f indexes.sql -||||||| 117df85a -``` -psql -d gis -f indexes.sql -======= ```sh -psql -d gis -f indexes.sql ->>>>>>> v5.7.0 +psql -d flex -f indexes.sql ``` The indexes can be created in parallel with From 816e006708840eb20443a03c5d32e1f5228def02 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Fri, 24 Feb 2023 22:37:20 -0800 Subject: [PATCH 49/54] Use postgres DB for system commands --- INSTALL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index 344d5f512b..bb1ac39d4d 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -32,7 +32,7 @@ We do not recommend [PostgreSQL JIT](https://www.postgresql.org/docs/current/jit Disabling JIT is **essential** for use with Kosmtik and other style development tools. -JIT can be disabled with `psql -d gis -c 'ALTER SYSTEM SET jit=off;' -c 'SELECT pg_reload_conf();'` or any other means of adjusting the PostgreSQL config. +JIT can be disabled with `psql -d postgres -c 'ALTER SYSTEM SET jit=off;' -c 'SELECT pg_reload_conf();'` or any other means of adjusting the PostgreSQL config. ### Custom indexes Custom indexes are required for rendering performance and are essential on full planet databases. These are generated by the `scripts/indexes.py` script, see `scripts/indexes.py --help` for various advanced options, but the command below will work to create the indexes on a new installation: From 800996e4b3955767706681f2b7ba49320cf441b0 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Fri, 24 Feb 2023 22:42:59 -0800 Subject: [PATCH 50/54] Fix merge errors in ci --- .github/workflows/ci.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5bea31e228..f0353be855 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,14 +64,8 @@ jobs: run: | osm2pgsql --output flex --style openstreetmap-carto.lua -d flex -r xml <(echo '') - name: Create indexes -<<<<<<< HEAD run: psql -1Xq -v ON_ERROR_STOP=1 -d flex -f indexes.sql -||||||| 117df85a - run: psql -1Xq -v ON_ERROR_STOP=1 -d gis -f indexes.sql -======= - run: psql -1Xq -v ON_ERROR_STOP=1 -d gis -f indexes.sql - name: Load empty shapefiles run: scripts/get-external-data.py --no-update --cache -D scripts/empty_files - name: Test queries are valid run: scripts/test-queries.py project.mml ->>>>>>> v5.7.0 From 72879d58b9366895ea49111bc69f96e5876c3018 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Fri, 24 Feb 2023 22:48:29 -0800 Subject: [PATCH 51/54] Fix index script when reindexing --- scripts/indexes.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/indexes.py b/scripts/indexes.py index c67b6b638f..64f33f01fe 100755 --- a/scripts/indexes.py +++ b/scripts/indexes.py @@ -45,12 +45,13 @@ def osm2pgsql_parse(index_function): def generate_statement(table, name, function, type, where): return index_statement(table, name, function, type, where, args.concurrent, args.notexist, args.fillfactor) -def generate_reindex_statement(table, name, function, where): +def generate_reindex_statement(table, name, function, type, where): if not args.concurrent: return f'REINDEX planet_osm_{table}_{name};' else: # Rebuilding indexes concurrently requires making a new index, dropping the old one, and renaming. - return f'ALTER INDEX planet_osm_{table}_{name} RENAME TO planet_osm_{table}_{name}_old; {generate_statement(table, name, function, where)} + DROP INDEX planet_osm_{table}_{name}_old;' + return (f'ALTER INDEX planet_osm_{table}_{name} RENAME TO planet_osm_{table}_{name}_old; {generate_statement(table, name, function, type, where)} ' + f'DROP INDEX planet_osm_{table}_{name}_old;') print('-- These are indexes for rendering performance with OpenStreetMap Carto.', end=separator) print('-- This file is generated with {}'.format(' '.join(sys.argv)), end=separator) From 14f668d19d37937b586b43bd3da1f5b3cb1e0b4f Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Fri, 24 Feb 2023 22:48:58 -0800 Subject: [PATCH 52/54] Always return something on layer and admin conversion --- openstreetmap-carto.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index 63f7c48e90..cefa80ded3 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -558,6 +558,7 @@ function layer (v) if v and string.find(v, "^-?%d+$") and tonumber(v) < 100 and tonumber(v) > -100 then -- check if value exists, is numeric, and is in range return v end + return nil end --- Normalizes admin_level tags @@ -567,6 +568,7 @@ function admin_level (v) if v and string.find(v, "^%d+$") and tonumber(v) < 100 and tonumber(v) > 0 then return v end + return nil end --- Clean tags of deleted tags From ba887bdb2ac149e4811cd8963611e40958a6a540 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Sat, 25 Feb 2023 12:06:40 -0800 Subject: [PATCH 53/54] More fixes of database name in docs --- DOCKER.md | 2 +- INSTALL.md | 2 +- scripts/test-queries.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DOCKER.md b/DOCKER.md index b7b66731fe..375fc44470 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -99,5 +99,5 @@ Docker stores its disk image by default in the home directories of the user. If When working with the style's database tables after an import, it can be helpful to log in at the [console](https://www.postgresql.org/docs/current/app-psql.html) to inspect the table structure or view imported data. The following command will open a psql console on the database: ``` -docker-compose exec -e PGUSER=postgres -e PGDATABASE=gis db psql +docker-compose exec -e PGUSER=postgres -e PGDATABASE=flex db psql ``` diff --git a/INSTALL.md b/INSTALL.md index bb1ac39d4d..ac7ee1823a 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -44,7 +44,7 @@ psql -d flex -f indexes.sql The indexes can be created in parallel with ```sh -scripts/indexes.py -0 | xargs -0 -P0 -I{} psql -d gis -c "{}" +scripts/indexes.py -0 | xargs -0 -P0 -I{} psql -d flex -c "{}" ``` ## Scripted download diff --git a/scripts/test-queries.py b/scripts/test-queries.py index 64432987f8..8747e1004c 100755 --- a/scripts/test-queries.py +++ b/scripts/test-queries.py @@ -41,7 +41,7 @@ def main(): description="Test CartoCSS project queries against a database") - parser.add_argument("-d", "--database", action="store", default="gis", + parser.add_argument("-d", "--database", action="store", default="flex", help="Override database name to connect to") parser.add_argument("-H", "--host", action="store", help="Override database server host or socket directory") From cc263e6d32459b9a1b9da5cb2b650e0485edf4b7 Mon Sep 17 00:00:00 2001 From: mboeringa Date: Fri, 28 Jul 2023 12:57:22 +0200 Subject: [PATCH 54/54] Potential fix for the issue raised in https://github.com/gravitystorm/openstreetmap-carto/pull/4431#pullrequestreview-1547693270 Moves creation of dictionaries storing admin_level for stage 2 processing of ways from the 'select_relation_members' function to the 'process_relation' function. --- openstreetmap-carto.lua | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua index cefa80ded3..bb28d2c92d 100644 --- a/openstreetmap-carto.lua +++ b/openstreetmap-carto.lua @@ -738,7 +738,28 @@ function osm2pgsql.process_relation(object) if clean_tags(object.tags) then return end + if type == "boundary" then + if object.tags.boundary == 'administrative' then + local admin = tonumber(admin_level(object.tags.admin_level)) + if admin ~= nil then + for _, ref in ipairs(osm2pgsql.way_member_ids(object)) do + -- Store the lowest admin_level, and how many relations it used in + if not phase2_admin_ways_level[ref] then + phase2_admin_ways_level[ref] = admin + phase2_admin_ways_parents[ref] = 1 + else + if phase2_admin_ways_level[ref] == admin then + phase2_admin_ways_parents[ref] = phase2_admin_ways_parents[ref] + 1 + elseif admin < phase2_admin_ways_level[ref] then + phase2_admin_ways_level[ref] = admin + phase2_admin_ways_parents[ref] = 1 + end + end + end + end + end + add_line(object) if roads(object.tags) then @@ -756,6 +777,7 @@ function osm2pgsql.process_relation(object) elseif type == "route" then add_route(object) end + end function osm2pgsql.select_relation_members(relation) @@ -763,20 +785,6 @@ function osm2pgsql.select_relation_members(relation) and relation.tags.boundary == 'administrative' then local admin = tonumber(admin_level(relation.tags.admin_level)) if admin ~= nil then - for _, ref in ipairs(osm2pgsql.way_member_ids(relation)) do - -- Store the lowest admin_level, and how many relations it used in - if not phase2_admin_ways_level[ref] then - phase2_admin_ways_level[ref] = admin - phase2_admin_ways_parents[ref] = 1 - else - if phase2_admin_ways_level[ref] == admin then - phase2_admin_ways_parents[ref] = phase2_admin_ways_parents[ref] + 1 - elseif admin < phase2_admin_ways_level[ref] then - phase2_admin_ways_level[ref] = admin - phase2_admin_ways_parents[ref] = 1 - end - end - end return { ways = osm2pgsql.way_member_ids(relation) } end end