Skip to content

[EXPERIMENTAL] Reduced way node index #1058

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

[EXPERIMENTAL] Reduced way node index #1058

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7f140a1
use a bucketed way-node index
lonvia Jan 3, 2020
89eaec4
fix prepared statement
lonvia Jan 4, 2020
504cbf5
fix index lookup id
lonvia Jan 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 13 additions & 6 deletions src/middle-pgsql.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -783,10 +783,16 @@ middle_pgsql_t::middle_pgsql_t(options_t const *options)
"PREPARE get_way (" POSTGRES_OSMID_TYPE ") AS SELECT nodes, tags, array_upper(nodes,1) FROM %p_ways WHERE id = $1;\n"
"PREPARE get_way_list (" POSTGRES_OSMID_TYPE "[]) AS SELECT id, nodes, tags, array_upper(nodes,1) FROM %p_ways WHERE id = ANY($1::" POSTGRES_OSMID_TYPE "[]);\n",
/*prepare_intarray*/
"PREPARE mark_ways_by_node(" POSTGRES_OSMID_TYPE ") AS select id from %p_ways WHERE nodes && ARRAY[$1];\n"
"PREPARE mark_ways_by_rel(" POSTGRES_OSMID_TYPE ") AS select id from %p_ways WHERE id IN (SELECT unnest(parts[way_off+1:rel_off]) FROM %p_rels WHERE id = $1);\n",

/*array_indexes*/ "CREATE INDEX %p_ways_nodes ON %p_ways USING gin (nodes) WITH (FASTUPDATE=OFF) {TABLESPACE %i};\n"
"PREPARE mark_ways_by_node(" POSTGRES_OSMID_TYPE ") AS"
" SELECT id FROM %p_ways w"
" WHERE $1 = any(nodes) and w.bucket_32 && ARRAY[$1/32];\n",

/*array_indexes*/
"CREATE FUNCTION bucket_32(%p_ways) RETURNS bigint[] AS $$\n"
" SELECT array(select distinct unnest(nodes)/32)"
" FROM %p_ways w WHERE w.id = $1.id\n"
"$$ LANGUAGE SQL IMMUTABLE;\n"
"CREATE INDEX %p_ways_nodes ON %p_ways USING gin (bucket_32(%p_ways)) WITH (FASTUPDATE=OFF) {TABLESPACE %i};\n"
);
tables[REL_TABLE] = table_desc(options,
/*table = t_rel,*/
Expand All @@ -796,9 +802,10 @@ middle_pgsql_t::middle_pgsql_t(options_t const *options)
"PREPARE get_rel (" POSTGRES_OSMID_TYPE ") AS SELECT members, tags, array_upper(members,1)/2 FROM %p_rels WHERE id = $1;\n"
"PREPARE rels_using_way(" POSTGRES_OSMID_TYPE ") AS SELECT id FROM %p_rels WHERE parts && ARRAY[$1] AND parts[way_off+1:rel_off] && ARRAY[$1];\n",
/*prepare_intarray*/
"PREPARE mark_rels_by_node(" POSTGRES_OSMID_TYPE ") AS select id from %p_ways WHERE nodes && ARRAY[$1];\n"
"PREPARE mark_rels_by_node(" POSTGRES_OSMID_TYPE ") AS select id from %p_rels WHERE parts && ARRAY[$1] AND parts[1:way_off] && ARRAY[$1];\n"
"PREPARE mark_rels_by_way(" POSTGRES_OSMID_TYPE ") AS select id from %p_rels WHERE parts && ARRAY[$1] AND parts[way_off+1:rel_off] && ARRAY[$1];\n"
"PREPARE mark_rels(" POSTGRES_OSMID_TYPE ") AS select id from %p_rels WHERE parts && ARRAY[$1] AND parts[rel_off+1:array_length(parts,1)] && ARRAY[$1];\n",
"PREPARE mark_rels(" POSTGRES_OSMID_TYPE ") AS select id from %p_rels WHERE parts && ARRAY[$1] AND parts[rel_off+1:array_length(parts,1)] && ARRAY[$1];\n"
"PREPARE mark_ways_by_rel(" POSTGRES_OSMID_TYPE ") AS select id from %p_ways WHERE id IN (SELECT unnest(parts[way_off+1:rel_off]) FROM %p_rels WHERE id = $1);\n",

/*array_indexes*/ "CREATE INDEX %p_rels_parts ON %p_rels USING gin (parts) WITH (FASTUPDATE=OFF) {TABLESPACE %i};\n"
);
Expand Down
6 changes: 5 additions & 1 deletion src/middle-pgsql.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,11 @@ struct middle_pgsql_t : public slim_middle_t
char const *array_indexes = "");

char const *name() const { return m_copy_target->name.c_str(); }
void clear_array_indexes() { m_array_indexes.clear(); }
void clear_array_indexes()
{
m_array_indexes.clear();
m_prepare_intarray.clear();
}

void stop(std::string conninfo, bool droptemp, bool build_indexes);

Expand Down
2 changes: 1 addition & 1 deletion tests/regression.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ def setUpClass(cls):
with psycopg2.connect("dbname='{}'".format(CONFIG['test_database'])) as conn:
with conn.cursor() as cur:
for t in ('nodes', 'ways', 'rels'):
cur.execute("DROP TABLE IF EXISTS planet_osm_" + t)
cur.execute("DROP TABLE IF EXISTS planet_osm_{} CASCADE".format(t))

if cls.import_file:
cls.run_import(cls.get_def_params() + cls.extra_params,
Expand Down