diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 528ff6be1..8ddf41219 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -19,6 +19,7 @@ target_sources(osm2pgsql_lib PRIVATE geom-from-osm.cpp geom-functions.cpp geom-pole-of-inaccessibility.cpp + idlist.cpp input.cpp logging.cpp middle.cpp diff --git a/src/dependency-manager.cpp b/src/dependency-manager.cpp index 72f1fe42e..3a600ff8a 100644 --- a/src/dependency-manager.cpp +++ b/src/dependency-manager.cpp @@ -15,17 +15,17 @@ void full_dependency_manager_t::node_changed(osmid_t id) { - m_changed_nodes.set(id); + m_changed_nodes.push_back(id); } void full_dependency_manager_t::way_changed(osmid_t id) { - m_changed_ways.set(id); + m_changed_ways.push_back(id); } void full_dependency_manager_t::relation_changed(osmid_t id) { - m_changed_relations.set(id); + m_changed_relations.push_back(id); } void full_dependency_manager_t::after_nodes() @@ -39,29 +39,13 @@ void full_dependency_manager_t::after_nodes() m_changed_nodes.clear(); } -static osmium::index::IdSetSmall -set_diff(osmium::index::IdSetSmall const &set, - osmium::index::IdSetSmall const &to_be_removed) -{ - osmium::index::IdSetSmall new_set; - - for (auto const id : set) { - if (!to_be_removed.get_binary_search(id)) { - new_set.set(id); - } - } - - return new_set; -} - void full_dependency_manager_t::after_ways() { if (!m_changed_ways.empty()) { if (!m_ways_pending_tracker.empty()) { // Remove ids from changed ways in the input data from // m_ways_pending_tracker, because they have already been processed. - m_ways_pending_tracker = - set_diff(m_ways_pending_tracker, m_changed_ways); + m_ways_pending_tracker.remove_ids_if_in(m_changed_ways); // Add the list of pending way ids to the list of changed ways, // because we need the parents for them, too. @@ -85,14 +69,13 @@ void full_dependency_manager_t::after_relations() { // Remove ids from changed relations in the input data from // m_rels_pending_tracker, because they have already been processed. - m_rels_pending_tracker = - set_diff(m_rels_pending_tracker, m_changed_relations); + m_rels_pending_tracker.remove_ids_if_in(m_changed_relations); m_changed_relations.clear(); } void full_dependency_manager_t::mark_parent_relations_as_pending( - osmium::index::IdSetSmall const &way_ids) + idlist_t const &way_ids) { assert(m_rels_pending_tracker.empty()); m_object_store->get_way_parents(way_ids, &m_rels_pending_tracker); @@ -102,18 +85,3 @@ bool full_dependency_manager_t::has_pending() const noexcept { return !m_ways_pending_tracker.empty() || !m_rels_pending_tracker.empty(); } - -idlist_t -full_dependency_manager_t::get_ids(osmium::index::IdSetSmall *tracker) -{ - tracker->sort_unique(); - - idlist_t list; - list.reserve(tracker->size()); - - std::copy(tracker->cbegin(), tracker->cend(), std::back_inserter(list)); - - tracker->clear(); - - return list; -} diff --git a/src/dependency-manager.hpp b/src/dependency-manager.hpp index 3afcc88c8..4902f1035 100644 --- a/src/dependency-manager.hpp +++ b/src/dependency-manager.hpp @@ -10,10 +10,9 @@ * For a full list of authors see the git log. */ +#include "idlist.hpp" #include "osmtypes.hpp" -#include - #include #include #include @@ -58,8 +57,7 @@ class dependency_manager_t virtual void after_ways() {} virtual void after_relations() {} - virtual void mark_parent_relations_as_pending( - osmium::index::IdSetSmall const & /*way_ids*/) + virtual void mark_parent_relations_as_pending(idlist_t const & /*way_ids*/) { } @@ -111,24 +109,29 @@ class full_dependency_manager_t : public dependency_manager_t void after_ways() override; void after_relations() override; - void mark_parent_relations_as_pending( - osmium::index::IdSetSmall const &ids) override; + void mark_parent_relations_as_pending(idlist_t const &ids) override; bool has_pending() const noexcept override; idlist_t get_pending_way_ids() override { - return get_ids(&m_ways_pending_tracker); + idlist_t list; + using std::swap; + swap(list, m_ways_pending_tracker); + list.sort_unique(); + return list; } idlist_t get_pending_relation_ids() override { - return get_ids(&m_rels_pending_tracker); + idlist_t list; + using std::swap; + swap(list, m_rels_pending_tracker); + list.sort_unique(); + return list; } private: - static idlist_t get_ids(osmium::index::IdSetSmall *tracker); - std::shared_ptr m_object_store; /** @@ -139,7 +142,7 @@ class full_dependency_manager_t : public dependency_manager_t * the change file, too, and so we don't have to find out which ones they * are. */ - osmium::index::IdSetSmall m_changed_nodes; + idlist_t m_changed_nodes; /** * In append mode all new and changed ways will be added to this. After @@ -148,17 +151,17 @@ class full_dependency_manager_t : public dependency_manager_t * relations that referenced deleted ways must be in the change file, too, * and so we don't have to find out which ones they are. */ - osmium::index::IdSetSmall m_changed_ways; + idlist_t m_changed_ways; /** * In append mode all new and changed relations will be added to this. * This is then used to remove already processed relations from the * pending list. */ - osmium::index::IdSetSmall m_changed_relations; + idlist_t m_changed_relations; - osmium::index::IdSetSmall m_ways_pending_tracker; - osmium::index::IdSetSmall m_rels_pending_tracker; + idlist_t m_ways_pending_tracker; + idlist_t m_rels_pending_tracker; }; #endif // OSM2PGSQL_DEPENDENCY_MANAGER_HPP diff --git a/src/idlist.cpp b/src/idlist.cpp new file mode 100644 index 000000000..d90ab0643 --- /dev/null +++ b/src/idlist.cpp @@ -0,0 +1,54 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "idlist.hpp" + +#include +#include +#include +#include + +osmid_t idlist_t::pop_id() +{ + assert(!m_list.empty()); + auto const id = m_list.back(); + m_list.pop_back(); + return id; +} + +void idlist_t::sort_unique() +{ + std::sort(m_list.begin(), m_list.end()); + const auto last = std::unique(m_list.begin(), m_list.end()); + m_list.erase(last, m_list.end()); +} + +void idlist_t::merge_sorted(idlist_t const &other) +{ + std::vector new_list; + + new_list.reserve(m_list.size() + other.m_list.size()); + std::set_union(m_list.cbegin(), m_list.cend(), other.m_list.cbegin(), + other.m_list.cend(), std::back_inserter(new_list)); + + using std::swap; + swap(new_list, m_list); +} + +void idlist_t::remove_ids_if_in(idlist_t const &other) +{ + std::vector new_list; + + new_list.reserve(m_list.size()); + std::set_difference(m_list.cbegin(), m_list.cend(), other.m_list.cbegin(), + other.m_list.cend(), std::back_inserter(new_list)); + + using std::swap; + swap(new_list, m_list); +} diff --git a/src/idlist.hpp b/src/idlist.hpp new file mode 100644 index 000000000..e719f01a9 --- /dev/null +++ b/src/idlist.hpp @@ -0,0 +1,107 @@ +#ifndef OSM2PGSQL_IDLIST_HPP +#define OSM2PGSQL_IDLIST_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +/** + * \file + * + * This file contains the definition of the idlist_t class. + */ + +#include "osmtypes.hpp" + +#include + +/** + * A list of OSM object ids. Internally this is a vector of ids. + * + * Some operations are only allowed when the list of ids is sorted and + * without duplicates. Call sort_unique() to achieve this. + */ +class idlist_t +{ +public: + using value_type = osmid_t; + + idlist_t() = default; + ~idlist_t() noexcept = default; + + idlist_t(std::initializer_list ids) : m_list(ids) {} + + idlist_t(idlist_t const &) = delete; + idlist_t &operator=(idlist_t const &) = delete; + + idlist_t(idlist_t &&) = default; + idlist_t &operator=(idlist_t &&) = default; + + bool empty() const noexcept { return m_list.empty(); } + + std::size_t size() const noexcept { return m_list.size(); } + + auto begin() const noexcept { return m_list.begin(); } + + auto end() const noexcept { return m_list.end(); } + + auto cbegin() const noexcept { return m_list.cbegin(); } + + auto cend() const noexcept { return m_list.cend(); } + + osmid_t operator[](std::size_t n) const noexcept { return m_list[n]; } + + void clear() noexcept { m_list.clear(); } + + void push_back(osmid_t id) { m_list.push_back(id); } + + void reserve(std::size_t size) { m_list.reserve(size); } + + /** + * Remove id at the end of the list and return it. + * + * \pre \code !m_list.empty()) \endcode + */ + osmid_t pop_id(); + + /// List are equal if they contain the same ids in the same order. + friend bool operator==(idlist_t const &lhs, idlist_t const &rhs) noexcept + { + return lhs.m_list == rhs.m_list; + } + + friend bool operator!=(idlist_t const &lhs, idlist_t const &rhs) noexcept + { + return !(lhs == rhs); + } + + /** + * Sort this list and remove duplicates. + */ + void sort_unique(); + + /** + * Merge other list into this one. + * + * \pre Both lists must be sorted and without duplicates. + */ + void merge_sorted(idlist_t const &other); + + /** + * Remove all ids in this list that are also in the other list. + * + * \pre Both lists must be sorted and without duplicates. + */ + void remove_ids_if_in(idlist_t const &other); + +private: + std::vector m_list; + +}; // class idlist_t + +#endif // OSM2PGSQL_IDLIST_HPP diff --git a/src/middle-pgsql.cpp b/src/middle-pgsql.cpp index 824eba9a9..e441363e8 100644 --- a/src/middle-pgsql.cpp +++ b/src/middle-pgsql.cpp @@ -33,6 +33,7 @@ #include #include "format.hpp" +#include "idlist.hpp" #include "json-writer.hpp" #include "logging.hpp" #include "middle-pgsql.hpp" @@ -55,8 +56,7 @@ static bool check_bucket_index(pg_conn_t const *db_connection, } static void send_id_list(pg_conn_t const &db_connection, - std::string const &table, - osmium::index::IdSetSmall const &ids) + std::string const &table, idlist_t const &ids) { std::string data; for (auto const id : ids) { @@ -70,13 +70,12 @@ static void send_id_list(pg_conn_t const &db_connection, } static void load_id_list(pg_conn_t const &db_connection, - std::string const &table, - osmium::index::IdSetSmall *ids) + std::string const &table, idlist_t *ids) { auto const res = db_connection.exec( fmt::format("SELECT DISTINCT id FROM {} ORDER BY id", table)); for (int n = 0; n < res.num_tuples(); ++n) { - ids->set(osmium::string_to_object_id(res.get_value(n, 0))); + ids->push_back(osmium::string_to_object_id(res.get_value(n, 0))); } } @@ -826,10 +825,9 @@ void middle_pgsql_t::node_delete(osmid_t osm_id) } } -void middle_pgsql_t::get_node_parents( - osmium::index::IdSetSmall const &changed_nodes, - osmium::index::IdSetSmall *parent_ways, - osmium::index::IdSetSmall *parent_relations) const +void middle_pgsql_t::get_node_parents(idlist_t const &changed_nodes, + idlist_t *parent_ways, + idlist_t *parent_relations) const { util::timer_t timer; @@ -922,9 +920,8 @@ INSERT INTO osm2pgsql_changed_relations parent_ways->size(), parent_relations->size()); } -void middle_pgsql_t::get_way_parents( - osmium::index::IdSetSmall const &changed_ways, - osmium::index::IdSetSmall *parent_relations) const +void middle_pgsql_t::get_way_parents(idlist_t const &changed_ways, + idlist_t *parent_relations) const { util::timer_t timer; diff --git a/src/middle-pgsql.hpp b/src/middle-pgsql.hpp index 2e09e9b6b..701af4da6 100644 --- a/src/middle-pgsql.hpp +++ b/src/middle-pgsql.hpp @@ -24,6 +24,7 @@ #include #include "db-copy-mgr.hpp" +#include "idlist.hpp" #include "middle.hpp" #include "pgsql.hpp" @@ -117,14 +118,11 @@ struct middle_pgsql_t : public middle_t void after_ways() override; void after_relations() override; - void get_node_parents( - osmium::index::IdSetSmall const &changed_nodes, - osmium::index::IdSetSmall *parent_ways, - osmium::index::IdSetSmall *parent_relations) const override; + void get_node_parents(idlist_t const &changed_nodes, idlist_t *parent_ways, + idlist_t *parent_relations) const override; - void get_way_parents( - osmium::index::IdSetSmall const &changed_ways, - osmium::index::IdSetSmall *parent_relations) const override; + void get_way_parents(idlist_t const &changed_ways, + idlist_t *parent_relations) const override; class table_desc { diff --git a/src/middle.hpp b/src/middle.hpp index 00b323a7e..2edacfe5c 100644 --- a/src/middle.hpp +++ b/src/middle.hpp @@ -10,7 +10,6 @@ * For a full list of authors see the git log. */ -#include #include #include @@ -19,6 +18,8 @@ #include "osmtypes.hpp" #include "thread-pool.hpp" +class idlist_t; + struct options_t; struct output_requirements; @@ -149,16 +150,14 @@ class middle_t #endif } - virtual void get_node_parents( - osmium::index::IdSetSmall const & /*changed_nodes*/, - osmium::index::IdSetSmall * /*parent_ways*/, - osmium::index::IdSetSmall * /*parent_relations*/) const + virtual void get_node_parents(idlist_t const & /*changed_nodes*/, + idlist_t * /*parent_ways*/, + idlist_t * /*parent_relations*/) const { } - virtual void get_way_parents( - osmium::index::IdSetSmall const & /*changed_ways*/, - osmium::index::IdSetSmall * /*parent_relations*/) const + virtual void get_way_parents(idlist_t const & /*changed_ways*/, + idlist_t * /*parent_relations*/) const { } diff --git a/src/osmdata.cpp b/src/osmdata.cpp index f2a20af2c..30e623e22 100644 --- a/src/osmdata.cpp +++ b/src/osmdata.cpp @@ -244,8 +244,7 @@ class multithreaded_processor std::lock_guard const lock{*mutex}; if (!queue->empty()) { - id = queue->back(); - queue->pop_back(); + id = queue->pop_id(); } return id; @@ -365,7 +364,7 @@ void osmdata_t::process_dependents() const } // stage 1c processing: mark parent relations of marked objects as changed - auto marked_ways = m_output->get_marked_way_ids(); + auto const &marked_ways = m_output->get_marked_way_ids(); if (marked_ways.empty()) { return; } diff --git a/src/osmdata.hpp b/src/osmdata.hpp index 78b6ddee1..f632cd4ee 100644 --- a/src/osmdata.hpp +++ b/src/osmdata.hpp @@ -24,6 +24,7 @@ #include #include "dependency-manager.hpp" +#include "idlist.hpp" #include "osmtypes.hpp" #include "pgsql-params.hpp" diff --git a/src/osmtypes.hpp b/src/osmtypes.hpp index 6eadff83b..b8b2dee66 100644 --- a/src/osmtypes.hpp +++ b/src/osmtypes.hpp @@ -238,23 +238,6 @@ class taglist_t std::vector m_tags; }; // class taglist_t -struct idlist_t : public std::vector -{ - // Get all constructors from std::vector - using vector::vector; - - // Even though we got all constructors from std::vector we need this on - // some compilers/libraries for some reason. - idlist_t() = default; - - explicit idlist_t(osmium::NodeRefList const &list) - { - for (auto const &n : list) { - push_back(n.ref()); - } - } -}; - using rolelist_t = std::vector; #endif // OSM2PGSQL_OSMTYPES_HPP diff --git a/src/output-flex.cpp b/src/output-flex.cpp index 510746be0..030c0055a 100644 --- a/src/output-flex.cpp +++ b/src/output-flex.cpp @@ -998,7 +998,7 @@ void output_flex_t::select_relation_members() "integer way ids."}; } - m_stage2_way_ids->set(id); + m_stage2_way_ids->push_back(id); }); lua_pop(lua_state(), 2); // return value (a table), ways field (a table) @@ -1486,7 +1486,7 @@ void output_flex_t::init_lua(std::string const &filename) lua_remove(lua_state(), 1); // global "osm2pgsql" } -idset_t const &output_flex_t::get_marked_way_ids() +idlist_t const &output_flex_t::get_marked_way_ids() { if (m_stage2_way_ids->empty()) { log_info("Skipping stage 1c (no marked ways)."); diff --git a/src/output-flex.hpp b/src/output-flex.hpp index b64d15e13..d8e07d274 100644 --- a/src/output-flex.hpp +++ b/src/output-flex.hpp @@ -16,9 +16,9 @@ #include "flex-table-column.hpp" #include "flex-table.hpp" #include "geom.hpp" +#include "idlist.hpp" #include "output.hpp" -#include #include #include @@ -35,8 +35,6 @@ class geom_transform_t; class thread_pool_t; struct options_t; -using idset_t = osmium::index::IdSetSmall; - /** * When C++ code is called from the Lua code we sometimes need to know * in what context this happens. These are the possible contexts. @@ -128,7 +126,7 @@ class output_flex_t : public output_t void wait() override; - idset_t const &get_marked_way_ids() override; + idlist_t const &get_marked_way_ids() override; void reprocess_marked() override; void pending_way(osmid_t id) override; @@ -295,7 +293,7 @@ class output_flex_t : public output_t // This is shared between all clones of the output and must only be // accessed while protected using the lua_mutex. - std::shared_ptr m_stage2_way_ids = std::make_shared(); + std::shared_ptr m_stage2_way_ids = std::make_shared(); std::shared_ptr m_copy_thread; diff --git a/src/output.hpp b/src/output.hpp index 847dc3662..d380de3e6 100644 --- a/src/output.hpp +++ b/src/output.hpp @@ -16,8 +16,8 @@ * Common output layer interface. */ -#include - +#include "idlist.hpp" +#include "options.hpp" #include "osmtypes.hpp" #include "output-requirements.hpp" @@ -68,9 +68,9 @@ class output_t virtual void wait() {} - virtual osmium::index::IdSetSmall const &get_marked_way_ids() + virtual idlist_t const &get_marked_way_ids() { - static osmium::index::IdSetSmall const ids{}; + static idlist_t const ids{}; return ids; } diff --git a/src/pgsql-helper.cpp b/src/pgsql-helper.cpp index 8d367a6a5..b438594e4 100644 --- a/src/pgsql-helper.cpp +++ b/src/pgsql-helper.cpp @@ -14,8 +14,9 @@ #include idlist_t get_ids_from_result(pg_result_t const &result) { - idlist_t ids; assert(result.num_tuples() >= 0); + + idlist_t ids; ids.reserve(static_cast(result.num_tuples())); for (int i = 0; i < result.num_tuples(); ++i) { diff --git a/src/pgsql-helper.hpp b/src/pgsql-helper.hpp index 15a46785d..7a38ee753 100644 --- a/src/pgsql-helper.hpp +++ b/src/pgsql-helper.hpp @@ -10,6 +10,7 @@ * For a full list of authors see the git log. */ +#include "idlist.hpp" #include "osmtypes.hpp" #include diff --git a/tests/common-buffer.hpp b/tests/common-buffer.hpp index 251d6c553..a974ef36d 100644 --- a/tests/common-buffer.hpp +++ b/tests/common-buffer.hpp @@ -11,6 +11,7 @@ */ #include "format.hpp" +#include "idlist.hpp" #include "osmtypes.hpp" #include diff --git a/tests/test-middle.cpp b/tests/test-middle.cpp index 9f36536fb..f384672bd 100644 --- a/tests/test-middle.cpp +++ b/tests/test-middle.cpp @@ -1118,7 +1118,7 @@ TEMPLATE_TEST_CASE("middle: change nodes in way", "", options_slim_default, REQUIRE(dependency_manager.has_pending()); idlist_t const way_ids = dependency_manager.get_pending_way_ids(); - REQUIRE_THAT(way_ids, Catch::Equals({20})); + REQUIRE(way_ids == idlist_t{20}); check_way(mid, way20); check_way_nodes(mid, way20.id(), {&node10a, &node11}); @@ -1153,7 +1153,7 @@ TEMPLATE_TEST_CASE("middle: change nodes in way", "", options_slim_default, REQUIRE(dependency_manager.has_pending()); idlist_t const way_ids = dependency_manager.get_pending_way_ids(); - REQUIRE_THAT(way_ids, Catch::Equals({20, 22})); + REQUIRE(way_ids == idlist_t{20, 22}); check_way(mid, way20); check_way_nodes(mid, way20.id(), {&node10a, &node11}); @@ -1265,7 +1265,7 @@ TEMPLATE_TEST_CASE("middle: change nodes in relation", "", options_slim_default, REQUIRE(dependency_manager.has_pending()); idlist_t const rel_ids = dependency_manager.get_pending_relation_ids(); - REQUIRE_THAT(rel_ids, Catch::Equals({30})); + REQUIRE(rel_ids == idlist_t{30}); check_relation(mid, rel30); } @@ -1286,9 +1286,9 @@ TEMPLATE_TEST_CASE("middle: change nodes in relation", "", options_slim_default, REQUIRE(dependency_manager.has_pending()); idlist_t const way_ids = dependency_manager.get_pending_way_ids(); - REQUIRE_THAT(way_ids, Catch::Equals({20})); + REQUIRE(way_ids == idlist_t{20}); idlist_t const rel_ids = dependency_manager.get_pending_relation_ids(); - REQUIRE_THAT(rel_ids, Catch::Equals({31})); + REQUIRE(rel_ids == idlist_t{31}); check_relation(mid, rel31); } }