Skip to content

schema - optional with defaults - improved behaviour (not injecting the default in the index) #246

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
danovaro wants to merge 2 commits into
base: develop
Choose a base branch
from
+2,015 −217
Open

schema - optional with defaults - improved behaviour (not injecting the default in the index) #246

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d2eb5f4
FDB-649 schema - optional with defaults - improved behaviour (not inj…
danovaro Mar 7, 2026
3c00e56
fix schema
danovaro Mar 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/fdb5/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -475,7 +475,7 @@ list( APPEND fdb5_tools
if ( HAVE_GRIB )
list( APPEND fdb5_tools
fdb-hammer
fdb-patch
# fdb-patch
)
Comment on lines 475 to 479
Copy link

Copilot AI Mar 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fdb-patch has been commented out from the list of built tools under HAVE_GRIB. If this is not intentional, it will remove the executable from builds/installs. If it is intentional, it would help to add a short comment explaining why the tool is being disabled (and ideally gate it behind an option) to avoid an unexpected feature regression.

Copilot uses AI. Check for mistakes.
endif()

Expand Down
18 changes: 17 additions & 1 deletion src/fdb5/api/FDB.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@
#include "eckit/message/Message.h"
#include "eckit/message/Reader.h"

#include "metkit/hypercube/HyperCube.h"
#include "metkit/mars/MarsLanguage.h"
#include "metkit/mars/MarsRequest.h"

#include "fdb5/LibFdb5.h"
#include "fdb5/api/FDBFactory.h"
#include "fdb5/api/helpers/FDBToolRequest.h"
Expand Down Expand Up @@ -237,7 +241,19 @@ eckit::DataHandle* FDB::retrieve(const metkit::mars::MarsRequest& request) {
}

ListIterator FDB::inspect(const metkit::mars::MarsRequest& request) {
return internal_->inspect(request);

ASSERT(!request.empty());

metkit::mars::MarsRequest indexingRequest{request.verb()};
metkit::mars::MarsLanguage language{request.verb()};
// Only copy data parameters, as these are the only ones relevant for the indexing
for (const auto& p : request.parameters()) {
if (!language.isPostProc(p.name()) && !language.isSink(p.name())) {
indexingRequest.setValuesTyped(&p.type(), p.values());
}
}

return internal_->inspect(indexingRequest);
}

ListIterator FDB::list(const FDBToolRequest& request, const ListMode mode, const int level) {
Expand Down
16 changes: 11 additions & 5 deletions src/fdb5/database/IndexAxis.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -399,10 +399,13 @@ void IndexAxis::print(std::ostream& out) const {

const char* sep = "";
out << "{";
for (const auto& kv : axis_) {
out << sep << kv.first << "=(";
for (const auto& [k, vv] : axis_) {
if (vv->size() == 1 && vv->at(0).empty()) {
continue;
}
out << sep << k << "=(";
const char* sep2 = "";
for (const auto& v : *kv.second) {
for (const auto& v : *vv) {
out << sep2 << v;
sep2 = ",";
}
Expand All @@ -415,8 +418,11 @@ void IndexAxis::print(std::ostream& out) const {

void IndexAxis::json(eckit::JSON& json) const {
json.startObject();
for (const auto& kv : axis_) {
json << kv.first << *kv.second;
for (const auto& [k, vv] : axis_) {
if (vv->size() == 1 && vv->at(0).empty()) {
continue;
}
json << k << *vv;
}
json.endObject();
}
Expand Down
4 changes: 3 additions & 1 deletion src/fdb5/rules/MatchOptional.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,11 +71,13 @@ void MatchOptional::fill(Key& key, const std::string& keyword, const std::string

const std::string& MatchOptional::value(const Key& key, const std::string& keyword) const {

static std::string empty{};

if (const auto [iter, found] = key.find(keyword); found) {
return iter->second;
}

return default_[0];
return empty;
}

const std::vector<std::string>& MatchOptional::values(const metkit::mars::MarsRequest& rq,
Expand Down
72 changes: 40 additions & 32 deletions src/fdb5/rules/Rule.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,13 @@ std::optional<Key> Rule::findMatchingKey(const eckit::StringList& values) const
return {};
}

ASSERT(values.size() >= predicates_.size());
size_t numPred = predicates_.size();
for (const auto& p : predicates_) {
if (p->optional()) {
numPred--;
}
}
ASSERT(values.size() >= numPred);

TypedKey key(registry_);

Expand Down Expand Up @@ -282,7 +288,6 @@ std::vector<Key> Rule::findMatchingKeys(const metkit::mars::MarsRequest& request
}
}

/// @todo activate this
graph.canonicalise(registry_);

return graph.makeKeys();
Expand All @@ -295,13 +300,12 @@ std::vector<Key> Rule::findMatchingKeys(const metkit::mars::MarsRequest& request
for (const auto& pred : predicates_) {

const auto& keyword = pred->keyword();
const auto& type = registry_.lookupType(keyword);


const auto& values = pred->values(request);
auto values = pred->values(request);

/// @note do we want to allow empty values?
// if (values.empty() && pred->optional()) { values.push_back(pred->defaultValue()); }
/// if the request does not have the keyword, but the predicate is optional, then use the default value
if (values.empty() && pred->optional()) {
values.push_back(pred->defaultValue());
}

auto& node = graph.push(keyword);

Expand All @@ -311,6 +315,7 @@ std::vector<Key> Rule::findMatchingKeys(const metkit::mars::MarsRequest& request
}
}

/// if there are no matching values for this predicate, then there are no matching keys for the rule
if (node.empty()) {
return {};
}
Expand All @@ -328,46 +333,43 @@ std::vector<Key> Rule::findMatchingKeys(const metkit::mars::MarsRequest& request
for (const auto& pred : predicates_) {

const auto& keyword = pred->keyword();
const auto& type = registry_.lookupType(keyword);

// performance optimization to avoid calling values() on visitor
if (!pred->optional() && request.countValues(keyword) == 0) {
return {};
}

eckit::StringList values;
visitor.values(request, keyword, registry_, values);

if (values.empty() && pred->optional()) {
values.push_back(pred->defaultValue());
if (request.countValues(keyword) == 0) {
if (pred->optional()) {
values.push_back(pred->defaultValue());
if (!pred->defaultValue().empty()) {
values.push_back("");
}
}
else {
return {};
}
}
else {
visitor.values(request, keyword, registry_, values);
}

auto& node = graph.push(keyword);

for (const auto& value : values) {
if (pred->match(value)) {
node.emplace_back(value);
}
}

if (node.empty()) {
return {};
if (pred->optional()) {
node.emplace_back("");
}
else {
return {};
}
}
}

graph.canonicalise(registry_);

auto out = graph.makeKeys();

LOG_DEBUG_LIB(LibFdb5) << "findMatchingKeys " << request << " ==> ";
std::string sep;
for (const auto& k : out) {
LOG_DEBUG_LIB(LibFdb5) << sep << k;
sep = " | ";
}
LOG_DEBUG_LIB(LibFdb5) << std::endl;

return out;
return graph.makeKeys();
}

//----------------------------------------------------------------------------------------------------------------------
Expand Down Expand Up @@ -399,7 +401,13 @@ bool Rule::tryFill(Key& key, const eckit::StringList& values) const {
// --> HACK.
// --> Stick a plaster over the symptom.

ASSERT(values.size() >= predicates_.size()); // Should be equal, except for quantile (FDB-103)
size_t numPred = predicates_.size();
for (const auto& p : predicates_) {
if (p->optional()) {
numPred--;
}
}
ASSERT(values.size() >= numPred); // Should be equal, except for quantile (FDB-103)
ASSERT(values.size() <= predicates_.size() + 1);

auto it_value = values.begin();
Expand Down
1 change: 0 additions & 1 deletion src/fdb5/tools/compare/fdb-compare.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@ class FDBCompare : public eckit::Tool {
std::optional<std::string> config2_;
std::optional<std::string> req1String_;
std::optional<std::string> req2String_;
bool singleFDB_ = false;
};

//---------------------------------------------------------------------------------------------------------------------
Expand Down
1 change: 0 additions & 1 deletion src/fdb5/tools/compare/grib/CompareBitwise.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,6 @@ CompareResult bitComparison(const uint8_t* buffer1, const uint8_t* buffer2, size

// Process sections based on edition
for (int i = 0; i < numSections; ++i) {
// std::cout<<"offset " << offset<<std::endl;
// Read section length
uint32_t sectionLength1 = 0;
uint32_t sectionLength2 = 0;
Expand Down
1 change: 1 addition & 0 deletions tests/fdb/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ add_subdirectory( api )
add_subdirectory( database )
add_subdirectory( type )
add_subdirectory( daos )
add_subdirectory( timespan )

if (HAVE_FDB_BUILD_TOOLS)
add_subdirectory( tools )
Expand Down
6 changes: 3 additions & 3 deletions tests/fdb/api/test_auxiliary.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ eckit::PathName writeAuxiliaryData(const eckit::PathName datapath, const std::st

std::set<eckit::PathName> setup(FDB& fdb, std::set<std::string> auxExtensions = {"foo", "bar"},
const std::vector<std::string>& dates = {"20101010", "20111213"},
const std::vector<std::string>& types = {"fc", "pf"},
const std::vector<std::string>& types = {"fc", "an"},
const std::vector<std::string>& steps = {"1", "2"}

) {
Expand Down Expand Up @@ -126,7 +126,7 @@ CASE("Wipe with extensions") {
EXPECT_EQUAL(element_counts[WipeElementType::STORE_AUX], 8);

// over specified wipe: returns nothing
request = FDBToolRequest::requestsFromString("class=od,expver=xxxx,type=pf,step=1")[0];
request = FDBToolRequest::requestsFromString("class=od,expver=xxxx,type=an,step=1")[0];
doit = true;
iter = fdb.wipe(request, doit);
element_counts.clear();
Expand All @@ -140,7 +140,7 @@ CASE("Wipe with extensions") {


// partial wipe on the second level. Hits half the data files
request = FDBToolRequest::requestsFromString("class=od,expver=xxxx,type=pf")[0];
request = FDBToolRequest::requestsFromString("class=od,expver=xxxx,type=an")[0];
doit = true;
iter = fdb.wipe(request, doit);
element_counts.clear();
Expand Down
2 changes: 1 addition & 1 deletion tests/fdb/api/test_callback.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ CASE("Archive and flush callback") {
keys.push_back(key);
fdb.archive(key, data, length);

key.set("type", "pf");
key.set("type", "an");
keys.push_back(key);
fdb.archive(key, data, length);

Expand Down
2 changes: 1 addition & 1 deletion tests/fdb/api/test_dist.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ CASE("retrieves_distributed_according_to_dist") {

// Do some archiving

metkit::mars::MarsRequest req;
metkit::mars::MarsRequest req{"retrieve"};
req.setValuesTyped(new metkit::mars::TypeAny("class"), std::vector<std::string>{"od"});
req.setValuesTyped(new metkit::mars::TypeAny("expver"), std::vector<std::string>{"xxxx"});
fdb.inspect(req);
Expand Down
9 changes: 7 additions & 2 deletions tests/fdb/api/test_fdb_c.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ int fdb_request_add1(fdb_request_t* req, const char* param, const char* value) {

void key_compare(const std::vector<fdb5::Key>& keys, fdb_listiterator_t* it, bool checkLevel = true) {
const char* k;
const char* v;
const char* v = nullptr;
size_t l;
int err;

Expand All @@ -47,7 +47,12 @@ void key_compare(const std::vector<fdb5::Key>& keys, fdb_listiterator_t* it, boo
size_t level = 0;
for (const auto& key : keys) {
for (const auto& k1 : key) {
int err = fdb_splitkey_next_metadata(sk, &k, &v, checkLevel ? &l : nullptr);
int err;
v = nullptr;
// skip empty values (optional metadata)
while (v == nullptr || strlen(v) == 0) {
Comment on lines +50 to +53
Copy link

Copilot AI Mar 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The loop that skips empty optional metadata values can become infinite if fdb_splitkey_next_metadata() returns FDB_ITERATION_COMPLETE (or any error) without setting v to a non-empty string. It also ignores err until after the loop. Consider checking err on each iteration and breaking/failing on non-FDB_SUCCESS, and/or bounding the loop so iteration-complete cannot spin forever.

Suggested change
int err;
v = nullptr;
// skip empty values (optional metadata)
while (v == nullptr || strlen(v) == 0) {
int err = FDB_SUCCESS;
v = nullptr;
// skip empty values (optional metadata)
while ((v == nullptr || strlen(v) == 0) && err == FDB_SUCCESS) {

Copilot uses AI. Check for mistakes.
err = fdb_splitkey_next_metadata(sk, &k, &v, checkLevel ? &l : nullptr);
}
std::cerr << "k=" << k << " v=" << v << " l=" << l << std::endl;
EXPECT(err == FDB_SUCCESS);
EXPECT(k1.first == k);
Expand Down
23 changes: 19 additions & 4 deletions tests/fdb/api/test_select.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,13 @@

#include "eckit/testing/Test.h"

#include "fdb5/api/helpers/WipeIterator.h"
#include "metkit/mars/MarsLanguage.h"
#include "metkit/mars/MarsParser.h"
#include "metkit/mars/MarsRequest.h"
#include "metkit/mars/TypeAny.h"

#include "fdb5/api/helpers/FDBToolRequest.h"
#include "fdb5/api/helpers/WipeIterator.h"
#include "fdb5/config/Config.h"

#include "ApiSpy.h"
Expand Down Expand Up @@ -153,8 +156,20 @@ CASE("retrieves_distributed_according_to_select") {
// Build FDB from default config

fdb5::FDB fdb(defaultConfig());
fdb.list(metkit::mars::MarsRequest{"class=od"});
fdb.list(metkit::mars::MarsRequest{"class=rd"});
{
std::istringstream in("retrieve,class=od");
metkit::mars::MarsParser parser(in);
auto reqs = parser.parse();
ASSERT(reqs.size() == 1);
fdb.list(reqs[0]);
}
{
std::istringstream in("retrieve,class=rd");
metkit::mars::MarsParser parser(in);
auto reqs = parser.parse();
ASSERT(reqs.size() == 1);
fdb.list(reqs[0]);
}

EXPECT(ApiSpy::knownSpies().size() == 3);
ApiSpy& spy_od(*ApiSpy::knownSpies()[0]);
Expand All @@ -163,7 +178,7 @@ CASE("retrieves_distributed_according_to_select") {

// Do some archiving

metkit::mars::MarsRequest req;
metkit::mars::MarsRequest req{"retrieve"};
req.setValuesTyped(new metkit::mars::TypeAny("class"), std::vector<std::string>{"od"});
req.setValuesTyped(new metkit::mars::TypeAny("expver"), std::vector<std::string>{"xxxx"});
fdb.inspect(req);
Expand Down
6 changes: 3 additions & 3 deletions tests/fdb/daos/test_daos_catalogue.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -527,9 +527,9 @@ CASE("DaosCatalogue tests") {
fdb5::Key db_key({{"a", "11"}, {"b", "22"}});
fdb5::Key index_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}});

fdb5::FDBToolRequest full_req{request_key.request("retrieve"), false, std::vector<std::string>{"a", "b"}};
fdb5::FDBToolRequest index_req{index_key.request("retrieve"), false, std::vector<std::string>{"a", "b"}};
fdb5::FDBToolRequest db_req{db_key.request("retrieve"), false, std::vector<std::string>{"a", "b"}};
fdb5::FDBToolRequest full_req{request_key.request(), false, std::vector<std::string>{"a", "b"}};
fdb5::FDBToolRequest index_req{index_key.request(), false, std::vector<std::string>{"a", "b"}};
fdb5::FDBToolRequest db_req{db_key.request(), false, std::vector<std::string>{"a", "b"}};
fdb5::FDBToolRequest all_req{metkit::mars::MarsRequest{}, true, std::vector<std::string>{}};

// initialise FDB
Expand Down
Loading
Loading