Skip to content

Commit f4e64c8

Browse files
lonvialonvia
authored
Merge pull request #1847 from joto/flex-indexes
Flex: Add support for configuring table indexes in the Lua config
2 parents d8dd191 + 5ac3817 commit f4e64c8

12 files changed

+1021
-12
lines changed

‎flex-config/indexes.lua

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
-- This config example file is released into the Public Domain.
2+
3+
-- This file shows some options around index creation.
4+
5+
local tables = {}
6+
7+
-- When "indexes" is explicitly set to an empty Lua table, there will be no
8+
-- index on this table. (The index for the id column is still built if
9+
-- osm2pgsql needs that for updates.)
10+
tables.pois = osm2pgsql.define_table({
11+
name = 'pois',
12+
ids = { type = 'node', id_column = 'node_id' },
13+
columns = {
14+
{ column = 'tags', type = 'jsonb' },
15+
{ column = 'geom', type = 'point', not_null = true },
16+
},
17+
indexes = {}
18+
})
19+
20+
-- The "indexes" field is not set at all, you get the default, a GIST index on
21+
-- the only (or first) geometry column.
22+
tables.ways = osm2pgsql.define_way_table('ways', {
23+
{ column = 'tags', type = 'jsonb' },
24+
{ column = 'geom', type = 'linestring', not_null = true },
25+
})
26+
27+
-- Setting "indexes" explicitly: Two indexes area created, one on the polygon
28+
-- geometry ("geom"), one on the center point geometry ("center"), both use
29+
-- the GIST method.
30+
tables.polygons = osm2pgsql.define_area_table('polygons', {
31+
{ column = 'tags', type = 'jsonb' },
32+
{ column = 'geom', type = 'geometry', not_null = true },
33+
{ column = 'center', type = 'point', not_null = true },
34+
}, { indexes = {
35+
{ column = 'geom', method = 'gist' },
36+
{ column = 'center', method = 'gist' }
37+
}})
38+
39+
-- You can put an index on any column, not just geometry columns, and use any
40+
-- index method available in your PostgreSQL version. To get a list of methods
41+
-- use: "SELECT amname FROM pg_catalog.pg_am WHERE amtype = 'i';"
42+
tables.pubs = osm2pgsql.define_node_table('pubs', {
43+
{ column = 'name', type = 'text' },
44+
{ column = 'geom', type = 'geometry', not_null = true },
45+
}, { indexes = {
46+
{ column = 'geom', method = 'gist' },
47+
{ column = 'name', method = 'btree' }
48+
}})
49+
50+
-- You can also create indexes using multiple columns by specifying an array
51+
-- as the "column". And you can add a where condition to the index. Note that
52+
-- the content of the where condition is not checked, but given "as is" to
53+
-- the database. You have to make sure it makes sense.
54+
tables.roads = osm2pgsql.define_way_table('roads', {
55+
{ column = 'name', type = 'text' },
56+
{ column = 'type', type = 'text' },
57+
{ column = 'ref', type = 'text' },
58+
{ column = 'geom', type = 'linestring', not_null = true },
59+
}, { indexes = {
60+
{ column = { 'name', 'ref' }, method = 'btree' },
61+
{ column = { 'geom' }, method = 'gist', where = "type='primary'" }
62+
}})
63+
64+
-- Instead of on a column (or columns) you can define an index on an expression.
65+
tables.postboxes = osm2pgsql.define_node_table('postboxes', {
66+
{ column = 'operator', type = 'text' },
67+
{ column = 'geom', type = 'point', not_null = true },
68+
}, { indexes = {
69+
{ expression = 'lower(operator)', method = 'btree' },
70+
}})
71+
72+
-- Helper function that looks at the tags and decides if this is possibly
73+
-- an area.
74+
function has_area_tags(tags)
75+
if tags.area == 'yes' then
76+
return true
77+
end
78+
if tags.area == 'no' then
79+
return false
80+
end
81+
82+
return tags.aeroway
83+
or tags.amenity
84+
or tags.building
85+
or tags.harbour
86+
or tags.historic
87+
or tags.landuse
88+
or tags.leisure
89+
or tags.man_made
90+
or tags.military
91+
or tags.natural
92+
or tags.office
93+
or tags.place
94+
or tags.power
95+
or tags.public_transport
96+
or tags.shop
97+
or tags.sport
98+
or tags.tourism
99+
or tags.water
100+
or tags.waterway
101+
or tags.wetland
102+
or tags['abandoned:aeroway']
103+
or tags['abandoned:amenity']
104+
or tags['abandoned:building']
105+
or tags['abandoned:landuse']
106+
or tags['abandoned:power']
107+
or tags['area:highway']
108+
end
109+
110+
function osm2pgsql.process_node(object)
111+
local geom = object:as_point()
112+
113+
tables.pois:insert({
114+
tags = object.tags,
115+
geom = geom
116+
})
117+
118+
if object.tags.amenity == 'pub' then
119+
tables.pubs:insert({
120+
name = object.tags.name,
121+
geom = geom
122+
})
123+
elseif object.tags.amenity == 'post_box' then
124+
tables.postboxes:insert({
125+
operator = object.tags.operator,
126+
geom = geom
127+
})
128+
end
129+
end
130+
131+
function osm2pgsql.process_way(object)
132+
if object.is_closed and has_area_tags(object.tags) then
133+
local geom = object:as_polygon()
134+
tables.polygons:insert({
135+
tags = object.tags,
136+
geom = geom,
137+
center = geom:centroid()
138+
})
139+
else
140+
tables.ways:insert({
141+
tags = object.tags,
142+
geom = object:as_linestring()
143+
})
144+
end
145+
146+
if object.tags.highway then
147+
tables.roads:insert({
148+
type = object.tags.highway,
149+
name = object.tags.name,
150+
ref = object.tags.ref,
151+
geom = object:as_linestring()
152+
})
153+
end
154+
end
155+

‎src/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ if (WITH_LUA)
4646
file(READ "${CMAKE_CURRENT_SOURCE_DIR}/init.lua" LUA_INIT_CODE)
4747
configure_file(lua-init.cpp.in lua-init.cpp @ONLY)
4848
target_sources(osm2pgsql_lib PRIVATE
49+
flex-index.cpp
4950
flex-table.cpp
5051
flex-table-column.cpp
5152
flex-lua-geom.cpp

‎src/flex-index.cpp

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/**
2+
* SPDX-License-Identifier: GPL-2.0-or-later
3+
*
4+
* This file is part of osm2pgsql (https://osm2pgsql.org/).
5+
*
6+
* Copyright (C) 2006-2022 by the osm2pgsql developer community.
7+
* For a full list of authors see the git log.
8+
*/
9+
10+
#include "flex-index.hpp"
11+
#include "util.hpp"
12+
13+
std::string flex_index_t::columns() const
14+
{
15+
return util::join(m_columns, ',', '"', '(', ')');
16+
}
17+
18+
std::string flex_index_t::include_columns() const
19+
{
20+
return util::join(m_include_columns, ',', '"', '(', ')');
21+
}
22+
23+
std::string
24+
flex_index_t::create_index(std::string const &qualified_table_name) const
25+
{
26+
util::string_joiner_t joiner{' '};
27+
joiner.add("CREATE");
28+
29+
if (m_is_unique) {
30+
joiner.add("UNIQUE");
31+
}
32+
33+
joiner.add("INDEX ON");
34+
joiner.add(qualified_table_name);
35+
36+
joiner.add("USING");
37+
joiner.add(m_method);
38+
39+
if (m_expression.empty()) {
40+
joiner.add(columns());
41+
} else {
42+
joiner.add('(' + m_expression + ')');
43+
}
44+
45+
if (!m_include_columns.empty()) {
46+
joiner.add("INCLUDE");
47+
joiner.add(include_columns());
48+
}
49+
50+
if (m_fillfactor != 0) {
51+
joiner.add("WITH");
52+
joiner.add("(fillfactor = {})"_format(m_fillfactor));
53+
}
54+
55+
if (!m_tablespace.empty()) {
56+
joiner.add("TABLESPACE");
57+
joiner.add("\"" + m_tablespace + "\"");
58+
}
59+
60+
if (!m_where_condition.empty()) {
61+
joiner.add("WHERE");
62+
joiner.add(m_where_condition);
63+
}
64+
65+
return joiner();
66+
}

‎src/flex-index.hpp

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
#ifndef OSM2PGSQL_FLEX_INDEX_HPP
2+
#define OSM2PGSQL_FLEX_INDEX_HPP
3+
4+
/**
5+
* SPDX-License-Identifier: GPL-2.0-or-later
6+
*
7+
* This file is part of osm2pgsql (https://osm2pgsql.org/).
8+
*
9+
* Copyright (C) 2006-2022 by the osm2pgsql developer community.
10+
* For a full list of authors see the git log.
11+
*/
12+
13+
#include <cassert>
14+
#include <stdexcept>
15+
#include <string>
16+
#include <vector>
17+
18+
/**
19+
* This class represents a database index.
20+
*/
21+
class flex_index_t
22+
{
23+
public:
24+
explicit flex_index_t(std::string method) : m_method(std::move(method)) {}
25+
26+
std::string const &method() const noexcept { return m_method; }
27+
28+
std::string columns() const;
29+
30+
/// Set columns (single-column version)
31+
void set_columns(std::string const &columns)
32+
{
33+
assert(m_columns.empty());
34+
m_columns.push_back(columns);
35+
}
36+
37+
/// Set columns (multi-column version)
38+
void set_columns(std::vector<std::string> const &columns)
39+
{
40+
m_columns = columns;
41+
}
42+
43+
std::string include_columns() const;
44+
45+
void set_include_columns(std::vector<std::string> const &columns)
46+
{
47+
m_include_columns = columns;
48+
}
49+
50+
std::string const &expression() const noexcept { return m_expression; }
51+
52+
void set_expression(std::string expression)
53+
{
54+
m_expression = std::move(expression);
55+
}
56+
57+
std::string const &tablespace() const noexcept { return m_tablespace; }
58+
59+
void set_tablespace(std::string tablespace)
60+
{
61+
m_tablespace = std::move(tablespace);
62+
}
63+
64+
std::string const &where_condition() const noexcept
65+
{
66+
return m_where_condition;
67+
}
68+
69+
void set_where_condition(std::string where_condition)
70+
{
71+
m_where_condition = std::move(where_condition);
72+
}
73+
74+
void set_fillfactor(uint8_t fillfactor)
75+
{
76+
if (fillfactor < 10 || fillfactor > 100) {
77+
throw std::runtime_error{"Fillfactor must be between 10 and 100."};
78+
}
79+
m_fillfactor = fillfactor;
80+
}
81+
82+
bool is_unique() const noexcept { return m_is_unique; }
83+
84+
void set_is_unique(bool unique) noexcept { m_is_unique = unique; }
85+
86+
std::string create_index(std::string const &qualified_table_name) const;
87+
88+
private:
89+
std::vector<std::string> m_columns;
90+
std::vector<std::string> m_include_columns;
91+
std::string m_method;
92+
std::string m_expression;
93+
std::string m_tablespace;
94+
std::string m_where_condition;
95+
uint8_t m_fillfactor = 0;
96+
bool m_is_unique = false;
97+
98+
}; // class flex_index_t
99+
100+
#endif // OSM2PGSQL_FLEX_INDEX_HPP

‎src/flex-table.cpp

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,11 @@ std::string flex_table_t::build_sql_create_id_index() const
219219
full_name(), id_column_names(), tablespace_clause(index_tablespace()));
220220
}
221221

222+
flex_index_t &flex_table_t::add_index(std::string method)
223+
{
224+
return m_indexes.emplace_back(std::move(method));
225+
}
226+
222227
void table_connection_t::connect(std::string const &conninfo)
223228
{
224229
assert(!m_db_connection);
@@ -367,15 +372,16 @@ void table_connection_t::stop(bool updateable, bool append)
367372
}
368373
}
369374

370-
if (table().has_geom_column()) {
371-
log_info("Creating geometry index on table '{}'...", table().name());
372-
373-
// Use fillfactor 100 for un-updateable imports
374-
m_db_connection->exec(
375-
R"(CREATE INDEX ON {} USING GIST ("{}") {} {})"_format(
376-
table().full_name(), table().geom_column().name(),
377-
(updateable ? "" : "WITH (fillfactor = 100)"),
378-
tablespace_clause(table().index_tablespace())));
375+
if (table().indexes().empty()) {
376+
log_info("No indexes to create on table '{}'.", table().name());
377+
} else {
378+
for (auto const &index : table().indexes()) {
379+
log_info("Creating index on table '{}' {}..."_format(
380+
table().name(), index.columns()));
381+
auto const sql = index.create_index(
382+
qualified_name(table().schema(), table().name()));
383+
m_db_connection->exec(sql);
384+
}
379385
}
380386

381387
if (updateable && table().has_id_column()) {

0 commit comments

Comments
 (0)