diff --git a/flex-config/indexes.lua b/flex-config/indexes.lua new file mode 100644 index 000000000..3e08254ac --- /dev/null +++ b/flex-config/indexes.lua @@ -0,0 +1,155 @@ +-- This config example file is released into the Public Domain. + +-- This file shows some options around index creation. + +local tables = {} + +-- When "indexes" is explicitly set to an empty Lua table, there will be no +-- index on this table. (The index for the id column is still built if +-- osm2pgsql needs that for updates.) +tables.pois = osm2pgsql.define_table({ + name = 'pois', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'point', not_null = true }, + }, + indexes = {} +}) + +-- The "indexes" field is not set at all, you get the default, a GIST index on +-- the only (or first) geometry column. +tables.ways = osm2pgsql.define_way_table('ways', { + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'linestring', not_null = true }, +}) + +-- Setting "indexes" explicitly: Two indexes area created, one on the polygon +-- geometry ("geom"), one on the center point geometry ("center"), both use +-- the GIST method. +tables.polygons = osm2pgsql.define_area_table('polygons', { + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry', not_null = true }, + { column = 'center', type = 'point', not_null = true }, +}, { indexes = { + { column = 'geom', method = 'gist' }, + { column = 'center', method = 'gist' } +}}) + +-- You can put an index on any column, not just geometry columns, and use any +-- index method available in your PostgreSQL version. To get a list of methods +-- use: "SELECT amname FROM pg_catalog.pg_am WHERE amtype = 'i';" +tables.pubs = osm2pgsql.define_node_table('pubs', { + { column = 'name', type = 'text' }, + { column = 'geom', type = 'geometry', not_null = true }, +}, { indexes = { + { column = 'geom', method = 'gist' }, + { column = 'name', method = 'btree' } +}}) + +-- You can also create indexes using multiple columns by specifying an array +-- as the "column". And you can add a where condition to the index. Note that +-- the content of the where condition is not checked, but given "as is" to +-- the database. You have to make sure it makes sense. +tables.roads = osm2pgsql.define_way_table('roads', { + { column = 'name', type = 'text' }, + { column = 'type', type = 'text' }, + { column = 'ref', type = 'text' }, + { column = 'geom', type = 'linestring', not_null = true }, +}, { indexes = { + { column = { 'name', 'ref' }, method = 'btree' }, + { column = { 'geom' }, method = 'gist', where = "type='primary'" } +}}) + +-- Instead of on a column (or columns) you can define an index on an expression. +tables.postboxes = osm2pgsql.define_node_table('postboxes', { + { column = 'operator', type = 'text' }, + { column = 'geom', type = 'point', not_null = true }, +}, { indexes = { + { expression = 'lower(operator)', method = 'btree' }, +}}) + +-- Helper function that looks at the tags and decides if this is possibly +-- an area. +function has_area_tags(tags) + if tags.area == 'yes' then + return true + end + if tags.area == 'no' then + return false + end + + return tags.aeroway + or tags.amenity + or tags.building + or tags.harbour + or tags.historic + or tags.landuse + or tags.leisure + or tags.man_made + or tags.military + or tags.natural + or tags.office + or tags.place + or tags.power + or tags.public_transport + or tags.shop + or tags.sport + or tags.tourism + or tags.water + or tags.waterway + or tags.wetland + or tags['abandoned:aeroway'] + or tags['abandoned:amenity'] + or tags['abandoned:building'] + or tags['abandoned:landuse'] + or tags['abandoned:power'] + or tags['area:highway'] +end + +function osm2pgsql.process_node(object) + local geom = object:as_point() + + tables.pois:insert({ + tags = object.tags, + geom = geom + }) + + if object.tags.amenity == 'pub' then + tables.pubs:insert({ + name = object.tags.name, + geom = geom + }) + elseif object.tags.amenity == 'post_box' then + tables.postboxes:insert({ + operator = object.tags.operator, + geom = geom + }) + end +end + +function osm2pgsql.process_way(object) + if object.is_closed and has_area_tags(object.tags) then + local geom = object:as_polygon() + tables.polygons:insert({ + tags = object.tags, + geom = geom, + center = geom:centroid() + }) + else + tables.ways:insert({ + tags = object.tags, + geom = object:as_linestring() + }) + end + + if object.tags.highway then + tables.roads:insert({ + type = object.tags.highway, + name = object.tags.name, + ref = object.tags.ref, + geom = object:as_linestring() + }) + end +end + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 14b81a8b1..59c796588 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -46,6 +46,7 @@ if (WITH_LUA) file(READ "${CMAKE_CURRENT_SOURCE_DIR}/init.lua" LUA_INIT_CODE) configure_file(lua-init.cpp.in lua-init.cpp @ONLY) target_sources(osm2pgsql_lib PRIVATE + flex-index.cpp flex-table.cpp flex-table-column.cpp flex-lua-geom.cpp diff --git a/src/flex-index.cpp b/src/flex-index.cpp new file mode 100644 index 000000000..c41bc06b4 --- /dev/null +++ b/src/flex-index.cpp @@ -0,0 +1,66 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2022 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "flex-index.hpp" +#include "util.hpp" + +std::string flex_index_t::columns() const +{ + return util::join(m_columns, ',', '"', '(', ')'); +} + +std::string flex_index_t::include_columns() const +{ + return util::join(m_include_columns, ',', '"', '(', ')'); +} + +std::string +flex_index_t::create_index(std::string const &qualified_table_name) const +{ + util::string_joiner_t joiner{' '}; + joiner.add("CREATE"); + + if (m_is_unique) { + joiner.add("UNIQUE"); + } + + joiner.add("INDEX ON"); + joiner.add(qualified_table_name); + + joiner.add("USING"); + joiner.add(m_method); + + if (m_expression.empty()) { + joiner.add(columns()); + } else { + joiner.add('(' + m_expression + ')'); + } + + if (!m_include_columns.empty()) { + joiner.add("INCLUDE"); + joiner.add(include_columns()); + } + + if (m_fillfactor != 0) { + joiner.add("WITH"); + joiner.add("(fillfactor = {})"_format(m_fillfactor)); + } + + if (!m_tablespace.empty()) { + joiner.add("TABLESPACE"); + joiner.add("\"" + m_tablespace + "\""); + } + + if (!m_where_condition.empty()) { + joiner.add("WHERE"); + joiner.add(m_where_condition); + } + + return joiner(); +} diff --git a/src/flex-index.hpp b/src/flex-index.hpp new file mode 100644 index 000000000..f62de79a2 --- /dev/null +++ b/src/flex-index.hpp @@ -0,0 +1,100 @@ +#ifndef OSM2PGSQL_FLEX_INDEX_HPP +#define OSM2PGSQL_FLEX_INDEX_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2022 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include +#include +#include +#include + +/** + * This class represents a database index. + */ +class flex_index_t +{ +public: + explicit flex_index_t(std::string method) : m_method(std::move(method)) {} + + std::string const &method() const noexcept { return m_method; } + + std::string columns() const; + + /// Set columns (single-column version) + void set_columns(std::string const &columns) + { + assert(m_columns.empty()); + m_columns.push_back(columns); + } + + /// Set columns (multi-column version) + void set_columns(std::vector const &columns) + { + m_columns = columns; + } + + std::string include_columns() const; + + void set_include_columns(std::vector const &columns) + { + m_include_columns = columns; + } + + std::string const &expression() const noexcept { return m_expression; } + + void set_expression(std::string expression) + { + m_expression = std::move(expression); + } + + std::string const &tablespace() const noexcept { return m_tablespace; } + + void set_tablespace(std::string tablespace) + { + m_tablespace = std::move(tablespace); + } + + std::string const &where_condition() const noexcept + { + return m_where_condition; + } + + void set_where_condition(std::string where_condition) + { + m_where_condition = std::move(where_condition); + } + + void set_fillfactor(uint8_t fillfactor) + { + if (fillfactor < 10 || fillfactor > 100) { + throw std::runtime_error{"Fillfactor must be between 10 and 100."}; + } + m_fillfactor = fillfactor; + } + + bool is_unique() const noexcept { return m_is_unique; } + + void set_is_unique(bool unique) noexcept { m_is_unique = unique; } + + std::string create_index(std::string const &qualified_table_name) const; + +private: + std::vector m_columns; + std::vector m_include_columns; + std::string m_method; + std::string m_expression; + std::string m_tablespace; + std::string m_where_condition; + uint8_t m_fillfactor = 0; + bool m_is_unique = false; + +}; // class flex_index_t + +#endif // OSM2PGSQL_FLEX_INDEX_HPP diff --git a/src/flex-table.cpp b/src/flex-table.cpp index 6d27592c7..e63a2be8b 100644 --- a/src/flex-table.cpp +++ b/src/flex-table.cpp @@ -219,6 +219,11 @@ std::string flex_table_t::build_sql_create_id_index() const full_name(), id_column_names(), tablespace_clause(index_tablespace())); } +flex_index_t &flex_table_t::add_index(std::string method) +{ + return m_indexes.emplace_back(std::move(method)); +} + void table_connection_t::connect(std::string const &conninfo) { assert(!m_db_connection); @@ -367,15 +372,16 @@ void table_connection_t::stop(bool updateable, bool append) } } - if (table().has_geom_column()) { - log_info("Creating geometry index on table '{}'...", table().name()); - - // Use fillfactor 100 for un-updateable imports - m_db_connection->exec( - R"(CREATE INDEX ON {} USING GIST ("{}") {} {})"_format( - table().full_name(), table().geom_column().name(), - (updateable ? "" : "WITH (fillfactor = 100)"), - tablespace_clause(table().index_tablespace()))); + if (table().indexes().empty()) { + log_info("No indexes to create on table '{}'.", table().name()); + } else { + for (auto const &index : table().indexes()) { + log_info("Creating index on table '{}' {}..."_format( + table().name(), index.columns())); + auto const sql = index.create_index( + qualified_name(table().schema(), table().name())); + m_db_connection->exec(sql); + } } if (updateable && table().has_id_column()) { diff --git a/src/flex-table.hpp b/src/flex-table.hpp index 56330001f..9aae735ad 100644 --- a/src/flex-table.hpp +++ b/src/flex-table.hpp @@ -11,6 +11,7 @@ */ #include "db-copy-mgr.hpp" +#include "flex-index.hpp" #include "flex-table-column.hpp" #include "pgsql.hpp" #include "reprojection.hpp" @@ -146,6 +147,13 @@ class flex_table_t return m_has_multiple_geom_columns; } + std::vector const &indexes() const noexcept + { + return m_indexes; + } + + flex_index_t &add_index(std::string method); + private: /// The name of the table std::string m_name; @@ -168,6 +176,11 @@ class flex_table_t */ std::vector m_columns; + /** + * The indexes defined on this table. Does not include the id index. + */ + std::vector m_indexes; + /** * Index of the (first) geometry column in m_columns. Default means no * geometry column. diff --git a/src/lua-utils.cpp b/src/lua-utils.cpp index 6386db114..73ddd0e92 100644 --- a/src/lua-utils.cpp +++ b/src/lua-utils.cpp @@ -138,7 +138,7 @@ char const *luaX_get_table_string(lua_State *lua_state, char const *key, } if (ltype != LUA_TSTRING) { throw std::runtime_error{ - "{} must contain a '{}' string field."_format(error_msg, key)}; + "{} field '{}' must be a string field."_format(error_msg, key)}; } return lua_tostring(lua_state, -1); } @@ -161,7 +161,7 @@ bool luaX_get_table_bool(lua_State *lua_state, char const *key, int table_index, } throw std::runtime_error{ - "{} must contain a '{}' boolean field."_format(error_msg, key)}; + "{} field '{}' must be a boolean field."_format(error_msg, key)}; } diff --git a/src/output-flex.cpp b/src/output-flex.cpp index 8fe7bc3bc..e91336329 100644 --- a/src/output-flex.cpp +++ b/src/output-flex.cpp @@ -9,6 +9,7 @@ #include "db-copy.hpp" #include "expire-tiles.hpp" +#include "flex-index.hpp" #include "flex-lua-geom.hpp" #include "format.hpp" #include "geom-from-osm.hpp" @@ -23,6 +24,7 @@ #include "osmtypes.hpp" #include "output-flex.hpp" #include "pgsql.hpp" +#include "pgsql-capabilities.hpp" #include "reprojection.hpp" #include "thread-pool.hpp" #include "util.hpp" @@ -1174,6 +1176,159 @@ void output_flex_t::setup_flex_table_columns(flex_table_t *table) throw std::runtime_error{ "No columns defined for table '{}'."_format(table->name())}; } + + lua_pop(lua_state(), 1); // "columns" +} + +static void check_and_add_column(flex_table_t const &table, + std::vector *columns, + char const *column_name) +{ + auto const *column = util::find_by_name(table, column_name); + if (!column) { + throw std::runtime_error{"Unknown column '{}' in table '{}'."_format( + column_name, table.name())}; + } + columns->push_back(column_name); +} + +static void check_and_add_columns(flex_table_t const &table, + std::vector *columns, + lua_State *lua_state) +{ + lua_pushnil(lua_state); + while (lua_next(lua_state, -2) != 0) { + if (!lua_isnumber(lua_state, -2)) { + throw std::runtime_error{ + "The 'column' field must contain a string or an array."}; + } + if (!lua_isstring(lua_state, -1)) { + throw std::runtime_error{ + "The entries in the 'column' array must be strings."}; + } + check_and_add_column(table, columns, lua_tostring(lua_state, -1)); + lua_pop(lua_state, 1); // table + } +} + +void output_flex_t::setup_indexes(flex_table_t *table) +{ + assert(table); + + lua_getfield(lua_state(), -1, "indexes"); + if (lua_type(lua_state(), -1) == LUA_TNIL) { + if (table->has_geom_column()) { + auto &index = table->add_index("gist"); + index.set_columns(table->geom_column().name()); + + if (!get_options()->slim || get_options()->droptemp) { + // If database can not be updated, use fillfactor 100. + index.set_fillfactor(100); + } + index.set_tablespace(table->index_tablespace()); + } + lua_pop(lua_state(), 1); // "indexes" + return; + } + + if (lua_type(lua_state(), -1) != LUA_TTABLE) { + throw std::runtime_error{ + "The 'indexes' field in definition of" + " table '{}' is not an array."_format(table->name())}; + } + + lua_pushnil(lua_state()); + while (lua_next(lua_state(), -2) != 0) { + if (!lua_isnumber(lua_state(), -2)) { + throw std::runtime_error{ + "The 'indexes' field must contain an array."}; + } + if (!lua_istable(lua_state(), -1)) { + throw std::runtime_error{ + "The entries in the 'indexes' array must be tables."}; + } + + char const *const method = luaX_get_table_string( + lua_state(), "method", -1, "Index definition"); + if (!has_index_method(method)) { + throw std::runtime_error{ + "Unknown index method '{}'."_format(method)}; + } + + auto &index = table->add_index(method); + + std::vector columns; + lua_getfield(lua_state(), -2, "column"); + if (lua_isstring(lua_state(), -1)) { + check_and_add_column(*table, &columns, + lua_tostring(lua_state(), -1)); + index.set_columns(columns); + } else if (lua_istable(lua_state(), -1)) { + check_and_add_columns(*table, &columns, lua_state()); + if (columns.empty()) { + throw std::runtime_error{ + "The 'column' field in an index definition can not be an " + "empty array."}; + } + index.set_columns(columns); + } else if (!lua_isnil(lua_state(), -1)) { + throw std::runtime_error{ + "The 'column' field in an index definition must contain a " + "string or an array."}; + } + + std::string const expression = luaX_get_table_string( + lua_state(), "expression", -3, "Index definition", ""); + + index.set_expression(expression); + + if (expression.empty() == columns.empty()) { + throw std::runtime_error{"You must set either the 'column' or the " + "'expression' field in index definition."}; + } + + std::vector include_columns; + lua_getfield(lua_state(), -4, "include"); + if (get_database_version() >= 110000) { + if (lua_isstring(lua_state(), -1)) { + check_and_add_column(*table, &include_columns, + lua_tostring(lua_state(), -1)); + } else if (lua_istable(lua_state(), -1)) { + check_and_add_columns(*table, &include_columns, lua_state()); + } else if (!lua_isnil(lua_state(), -1)) { + throw std::runtime_error{ + "The 'include' field in an index definition must contain a " + "string or an array."}; + } + index.set_include_columns(include_columns); + } else if (!lua_isnil(lua_state(), -1)) { + throw std::runtime_error{ + "Database version ({}) doesn't support" + " include columns in indexes."_format(get_database_version())}; + } + + std::string const tablespace = luaX_get_table_string( + lua_state(), "tablespace", -5, "Index definition", ""); + check_identifier(tablespace, "tablespace"); + if (!has_tablespace(tablespace)) { + throw std::runtime_error{ + "Unknown tablespace '{}'."_format(tablespace)}; + } + index.set_tablespace(tablespace.empty() ? table->index_tablespace() + : tablespace); + + index.set_is_unique(luaX_get_table_bool(lua_state(), "unique", -6, + "Index definition", false)); + + index.set_where_condition(luaX_get_table_string( + lua_state(), "where", -7, "Index definition", "")); + + // stack has: "where", "unique", "tablespace", "includes", "expression", + // "column", "method", table + lua_pop(lua_state(), 8); + } + + lua_pop(lua_state(), 1); // "indexes" } int output_flex_t::app_define_table() @@ -1192,6 +1347,7 @@ int output_flex_t::app_define_table() auto &new_table = create_flex_table(); setup_id_columns(&new_table); setup_flex_table_columns(&new_table); + setup_indexes(&new_table); void *ptr = lua_newuserdata(lua_state(), sizeof(std::size_t)); std::size_t *num = new (ptr) std::size_t{}; @@ -2019,6 +2175,30 @@ output_flex_t::output_flex_t(std::shared_ptr const &mid, "No tables defined in Lua config. Nothing to do!"}; } + log_debug("Tables:"); + for (auto const &table : *m_tables) { + log_debug( + "- TABLE {}"_format(qualified_name(table.schema(), table.name()))); + log_debug(" - columns:"); + for (auto const &column : table) { + log_debug(" - \"{}\" {} ({}) not_null={} create_only={}", + column.name(), column.type_name(), column.sql_type_name(), + column.not_null(), column.create_only()); + } + log_debug(" - data_tablespace={}", table.data_tablespace()); + log_debug(" - index_tablespace={}", table.index_tablespace()); + log_debug(" - cluster={}", table.cluster_by_geom()); + for (auto const &index : table.indexes()) { + log_debug(" - INDEX USING {}", index.method()); + log_debug(" - column={}", index.columns()); + log_debug(" - expression={}", index.expression()); + log_debug(" - include={}", index.include_columns()); + log_debug(" - tablespace={}", index.tablespace()); + log_debug(" - unique={}", index.is_unique()); + log_debug(" - where={}", index.where_condition()); + } + } + for (auto &table : *m_tables) { m_table_connections.emplace_back(&table, m_copy_thread); } diff --git a/src/output-flex.hpp b/src/output-flex.hpp index 6001aaf0d..5010d75f3 100644 --- a/src/output-flex.hpp +++ b/src/output-flex.hpp @@ -188,6 +188,7 @@ class output_flex_t : public output_t flex_table_t &create_flex_table(); void setup_id_columns(flex_table_t *table); void setup_flex_table_columns(flex_table_t *table); + void setup_indexes(flex_table_t *table); flex_table_t const &get_table_from_param(); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index db1f5d6bf..11bfc74c7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -98,7 +98,7 @@ if (WITH_LUA) set_test(test-output-flex-validgeom) set_test(test-output-flex-example-configs) - set(FLEX_EXAMPLE_CONFIGS "addresses,attributes,bbox,compatible,data-types,generic,geometries,places,route-relations,simple,unitable") + set(FLEX_EXAMPLE_CONFIGS "addresses,attributes,bbox,compatible,data-types,generic,geometries,indexes,places,route-relations,simple,unitable") # with-schema.lua is not tested because it needs the schema created in the database set_tests_properties(test-output-flex-example-configs PROPERTIES ENVIRONMENT "EXAMPLE_FILES=${FLEX_EXAMPLE_CONFIGS}") endif() diff --git a/tests/bdd/environment.py b/tests/bdd/environment.py index 0425dd983..6b11c2ebf 100644 --- a/tests/bdd/environment.py +++ b/tests/bdd/environment.py @@ -67,6 +67,9 @@ def before_all(context): cur.execute("""SELECT spcname FROM pg_tablespace WHERE spcname = 'tablespacetest'""") context.config.userdata['HAVE_TABLESPACE'] = cur.rowcount > 0 + cur.execute("""SELECT setting FROM pg_settings + WHERE name = 'server_version_num'""") + context.config.userdata['PG_VERSION'] = int(cur.fetchone()[0]) # Get the osm2pgsql configuration proc = subprocess.Popen([str(context.config.userdata['BINARY']), '--version'], @@ -125,3 +128,9 @@ def test_db(context, **kwargs): def working_directory(context, **kwargs): with tempfile.TemporaryDirectory() as tmpdir: yield Path(tmpdir) + +def before_tag(context, tag): + if tag == 'needs-pg-index-includes': + if context.config.userdata['PG_VERSION'] < 110000: + context.scenario.skip("No index includes in PostgreSQL < 11") + diff --git a/tests/bdd/flex/lua-index-definitions.feature b/tests/bdd/flex/lua-index-definitions.feature new file mode 100644 index 000000000..0bf1c3383 --- /dev/null +++ b/tests/bdd/flex/lua-index-definitions.feature @@ -0,0 +1,478 @@ +Feature: Index definitions in Lua file + + Scenario: Indexes field in table definition must be an array + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = true + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + The 'indexes' field in definition of table 'mytable' is not an array. + """ + + Scenario: No indexes field in table definition gets you a default index + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + } + }) + """ + When running osm2pgsql flex + Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING gist (geom)%' + | schemaname | tablename | + | public | mytable | + + Scenario: Empty indexes field in table definition gets you no index + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = {} + }) + """ + When running osm2pgsql flex + Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' + | schemaname | tablename | + + Scenario: Explicitly setting an index column works + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = { + { column = 'name', method = 'btree' } + } + }) + """ + When running osm2pgsql flex + Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' + | schemaname | tablename | + | public | mytable | + + Scenario: Explicitly setting multiple indexes + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = { + { column = 'name', method = 'btree' }, + { column = 'geom', method = 'gist' }, + { column = { 'name', 'tags' }, method = 'btree' } + } + }) + """ + When running osm2pgsql flex + Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' + | schemaname | tablename | + | public | mytable | + And SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING gist (geom)%' + | schemaname | tablename | + | public | mytable | + And SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name, tags)%' + | schemaname | tablename | + | public | mytable | + + Scenario: Method can not be missing + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = { + { column = 'name' } + } + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Index definition must contain a 'method' string field. + """ + + Scenario: Method must be valid + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = { + { column = 'name', method = 'ERROR' } + } + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Unknown index method 'ERROR'. + """ + + Scenario: Column can not be missing + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = { + { method = 'btree' } + } + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + You must set either the 'column' or the 'expression' field in index definition. + """ + + Scenario: Column must exist + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = { + { column = 'foo', method = 'btree' } + } + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Unknown column 'foo' in table 'mytable'. + """ + + Scenario: Column and expression together doesn't work + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = { + { column = 'name', expression = 'lower(name)', method = 'btree' } + } + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + You must set either the 'column' or the 'expression' field in index definition. + """ + + Scenario: Expression indexes work + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = { + { expression = 'lower(name)', method = 'btree' }, + } + }) + """ + When running osm2pgsql flex + Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (lower(name))%' + | schemaname | tablename | + | public | mytable | + + @needs-pg-index-includes + Scenario: Include field must be a string or array + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = { + { column = 'name', method = 'btree', include = true } + } + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + The 'include' field in an index definition must contain a string or an array. + """ + + @needs-pg-index-includes + Scenario: Include field must contain a valid column + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = { + { column = 'name', method = 'btree', include = 'foo' } + } + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Unknown column 'foo' in table 'mytable'. + """ + + @needs-pg-index-includes + Scenario: Include field works with string + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = { + { column = 'name', method = 'btree', include = 'tags' } + } + }) + """ + When running osm2pgsql flex + Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' AND indexdef LIKE '%INCLUDE (tags)%' + | schemaname | tablename | + | public | mytable | + + @needs-pg-index-includes + Scenario: Include field works with array + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = { + { column = 'name', method = 'btree', include = { 'tags' } } + } + }) + """ + When running osm2pgsql flex + Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' AND indexdef LIKE '%INCLUDE (tags)%' + | schemaname | tablename | + | public | mytable | + + Scenario: Tablespace needs a string + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = { + { column = 'name', method = 'btree', tablespace = true } + } + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Index definition field 'tablespace' must be a string field. + """ + + Scenario: Empty tablespace is okay + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = { + { column = 'name', method = 'btree', tablespace = '' } + } + }) + """ + When running osm2pgsql flex + Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' + | schemaname | tablename | + | public | mytable | + + Scenario: Unique needs a boolean + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = { + { column = 'name', method = 'btree', unique = 'foo' } + } + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Index definition field 'unique' must be a boolean field. + """ + + Scenario: Unique index works + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = { + { column = 'name', method = 'btree', unique = true } + } + }) + """ + When running osm2pgsql flex + Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' AND indexdef LIKE '%UNIQUE%' + | schemaname | tablename | + | public | mytable | + + Scenario: Where condition needs a string + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = { + { column = 'name', method = 'btree', where = true } + } + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Index definition field 'where' must be a string field. + """ + + Scenario: Where condition works + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local t = osm2pgsql.define_table({ + name = 'mytable', + ids = { type = 'node', id_column = 'node_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry' }, + }, + indexes = { + { column = 'name', method = 'btree', where = 'name = lower(name)' } + } + }) + """ + When running osm2pgsql flex + Then SELECT schemaname, tablename FROM pg_catalog.pg_indexes WHERE tablename = 'mytable' AND indexdef LIKE '%USING btree (name)%' AND indexdef LIKE '%WHERE (name = lower(name))%' + | schemaname | tablename | + | public | mytable |