Skip to content

Commit 52ffd18

Browse files
hjanetzekhjanetzek
committed
wip: early filter/drop unused feature properties
1 parent 014c3c5 commit 52ffd18

5 files changed

Lines changed: 93 additions & 34 deletions

File tree

‎core/include/tangram/data/tileSource.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,12 @@ class TileSource : public std::enable_shared_from_this<TileSource> {
142142

143143
void setFormat(Format format) { m_format = format; }
144144

145+
struct PropertyFilter {
146+
std::vector<std::string> drop;
147+
std::vector<std::string> keep;
148+
};
149+
void setPropertyFilter(PropertyFilter&& filter) { m_propertyFilter = std::move(filter); }
150+
145151
protected:
146152

147153
void createSubTasks(std::shared_ptr<TileTask> _task);
@@ -167,6 +173,8 @@ class TileSource : public std::enable_shared_from_this<TileSource> {
167173
std::vector<std::shared_ptr<TileSource>> m_rasterSources;
168174

169175
std::unique_ptr<DataSource> m_sources;
176+
177+
PropertyFilter m_propertyFilter;
170178
};
171179

172180
}

‎core/src/data/formats/mvt.cpp‎

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -117,12 +117,14 @@ Feature Mvt::getFeature(ParserContext& _ctx, protobuf::message _featureIn) {
117117
}
118118

119119
auto valueKey = tagsMsg.varint();
120-
120+
// Check if the property should be dropped
121+
if (_ctx.keys[tagKey].empty()) {
122+
continue;
123+
}
121124
if( _ctx.values.size() <= valueKey ) {
122125
LOGE("accessing out of bound values");
123126
return feature;
124127
}
125-
126128
_ctx.featureTags[tagKey] = valueKey;
127129
}
128130
break;
@@ -215,7 +217,8 @@ Feature Mvt::getFeature(ParserContext& _ctx, protobuf::message _featureIn) {
215217
return feature;
216218
}
217219

218-
Layer Mvt::getLayer(ParserContext& _ctx, protobuf::message _layerIn) {
220+
Layer Mvt::getLayer(ParserContext& _ctx, protobuf::message _layerIn,
221+
const TileSource::PropertyFilter& filter) {
219222

220223
Layer layer("");
221224

@@ -245,7 +248,29 @@ Layer Mvt::getLayer(ParserContext& _ctx, protobuf::message _layerIn) {
245248
continue;
246249
}
247250
case LAYER_KEY: {
248-
_ctx.keys.push_back(_layerIn.string());
251+
std::string key = _layerIn.string();
252+
// Check whether the key must be kept
253+
if (std::find(std::begin(filter.keep), std::end(filter.keep), key) == std::end(filter.keep)) {
254+
// Check whether the key should be dropped
255+
if (std::find_if(std::begin(filter.drop), std::end(filter.drop),
256+
[&](auto& k) {
257+
if (k.back() == '*' && key.length() >= k.length()-1) {
258+
int n = std::strncmp(key.c_str(), k.c_str(), k.length()-1);
259+
//LOG("check %s / %d / %d", k.c_str(), n, k.length()-1);
260+
return n == 0;
261+
} else {
262+
return key == k;
263+
}}) != std::end(filter.drop)) {
264+
265+
LOG("drop key: %s", key.c_str());
266+
key = "";
267+
//} else {
268+
//LOG("keep key: %s", key.c_str());
269+
}
270+
//} else {
271+
//LOG("keep key: %s", key.c_str());
272+
}
273+
_ctx.keys.emplace_back(std::move(key));
249274
break;
250275
}
251276
case LAYER_VALUE: {
@@ -321,7 +346,8 @@ Layer Mvt::getLayer(ParserContext& _ctx, protobuf::message _layerIn) {
321346
return layer;
322347
}
323348

324-
std::shared_ptr<TileData> Mvt::parseTile(const TileTask& _task, int32_t _sourceId) {
349+
std::shared_ptr<TileData> Mvt::parseTile(const TileTask& _task, int32_t _sourceId,
350+
const TileSource::PropertyFilter& filter) {
325351

326352
auto tileData = std::make_shared<TileData>();
327353

@@ -333,7 +359,7 @@ std::shared_ptr<TileData> Mvt::parseTile(const TileTask& _task, int32_t _sourceI
333359
try {
334360
while(item.next()) {
335361
if(item.tag == LAYER) {
336-
tileData->layers.push_back(getLayer(ctx, item.getMessage()));
362+
tileData->layers.push_back(getLayer(ctx, item.getMessage(), filter));
337363
} else {
338364
item.skip();
339365
}

‎core/src/data/formats/mvt.h‎

Lines changed: 30 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#pragma once
22

33
#include "data/tileData.h"
4+
#include "data/tileSource.h"
45
#include "pbf/pbf.hpp"
56
#include "util/variant.h"
67

@@ -16,41 +17,43 @@ class MapProjection;
1617

1718
namespace Mvt {
1819

19-
struct Geometry {
20-
std::vector<Point> coordinates;
21-
std::vector<int> sizes;
22-
};
20+
struct Geometry {
21+
std::vector<Point> coordinates;
22+
std::vector<int> sizes;
23+
};
2324

24-
struct ParserContext {
25-
ParserContext(int32_t _sourceId) : sourceId(_sourceId){}
25+
struct ParserContext {
26+
explicit ParserContext(int32_t _sourceId) : sourceId(_sourceId){}
2627

27-
int32_t sourceId;
28-
std::vector<std::string> keys;
29-
std::vector<Value> values;
30-
std::vector<protobuf::message> featureMsgs;
31-
Geometry geometry;
32-
// Map Key ID -> Tag values
33-
std::vector<int> featureTags;
34-
// Key IDs sorted by Property key ordering
35-
std::vector<int> orderedKeys;
28+
int32_t sourceId;
29+
std::vector<std::string> keys;
30+
std::vector<Value> values;
31+
std::vector<protobuf::message> featureMsgs;
32+
Geometry geometry;
33+
// Map Key ID -> Tag values
34+
std::vector<int> featureTags;
35+
// Key IDs sorted by Property key ordering
36+
std::vector<int> orderedKeys;
3637

37-
int tileExtent = 0;
38-
int winding = 0;
39-
};
38+
int tileExtent = 0;
39+
int winding = 0;
40+
};
4041

41-
enum GeomCmd {
42-
moveTo = 1,
43-
lineTo = 2,
44-
closePath = 7
45-
};
42+
enum GeomCmd {
43+
moveTo = 1,
44+
lineTo = 2,
45+
closePath = 7
46+
};
4647

47-
Geometry getGeometry(ParserContext& _ctx, protobuf::message _geomIn);
48+
Geometry getGeometry(ParserContext& _ctx, protobuf::message _geomIn);
4849

49-
Feature getFeature(ParserContext& _ctx, protobuf::message _featureIn);
50+
Feature getFeature(ParserContext& _ctx, protobuf::message _featureIn);
5051

51-
Layer getLayer(ParserContext& _ctx, protobuf::message _layerIn);
52+
Layer getLayer(ParserContext& _ctx, protobuf::message _layerIn,
53+
const TileSource::PropertyFilter& filter);
5254

53-
std::shared_ptr<TileData> parseTile(const TileTask& _task, int32_t _sourceId);
55+
std::shared_ptr<TileData> parseTile(const TileTask& _task, int32_t _sourceId,
56+
const TileSource::PropertyFilter& filter);
5457

5558
} // namespace Mvt
5659

‎core/src/data/tileSource.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ std::shared_ptr<TileData> TileSource::parse(const TileTask& _task) const {
106106
switch (m_format) {
107107
case Format::TopoJson: return TopoJson::parseTile(_task, m_id);
108108
case Format::GeoJson: return GeoJson::parseTile(_task, m_id);
109-
case Format::Mvt: return Mvt::parseTile(_task, m_id);
109+
case Format::Mvt: return Mvt::parseTile(_task, m_id, m_propertyFilter);
110110
}
111111
assert(false);
112112
return nullptr;

‎core/src/scene/sceneLoader.cpp‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -969,6 +969,26 @@ void SceneLoader::loadSource(const std::shared_ptr<Platform>& platform, const st
969969
}
970970
}
971971

972+
TileSource::PropertyFilter propFilter;
973+
if (auto n = source["drop_feature_properties"]) {
974+
if (n.IsSequence()) {
975+
for (auto& key : n) {
976+
if (key.IsScalar()) {
977+
propFilter.drop.push_back(key.Scalar());
978+
}
979+
}
980+
}
981+
}
982+
if (auto n = source["keep_feature_properties"]) {
983+
if (n.IsSequence()) {
984+
for (auto& key : n) {
985+
if (key.IsScalar()) {
986+
propFilter.keep.push_back(key.Scalar());
987+
}
988+
}
989+
}
990+
}
991+
972992
// Parse and append any URL parameters.
973993
if (auto urlParamsNode = source["url_params"]) {
974994
std::stringstream urlStream;
@@ -1084,6 +1104,8 @@ void SceneLoader::loadSource(const std::shared_ptr<Platform>& platform, const st
10841104
"This source will be ignored.", name.c_str());
10851105
return;
10861106
}
1107+
1108+
sourcePtr->setPropertyFilter(std::move(propFilter));
10871109
}
10881110

10891111
_scene->tileSources().push_back(sourcePtr);

0 commit comments

Comments
 (0)