DS-1801: WIP

Signed-off-by: Benjamin Rögner <[email protected]>

Author: roegi

xyz-hub-service/src/main/java/com/here/xyz/hub/rest/FeatureApi.java

@@ -590,45 +590,9 @@ private UpdateStrategy toUpdateStrategy(Space space, IfExists ifExists, IfNotExi
     );
   }
 
-  private String writeHook()
-  {
-   return    // TODO:  paths = [name, jsonpath] needs to be filled from index configuration
-    """
-  feature => {
-
-  const paths = [
-  ["$alias1", "$.properties.name"],
-  ["$alias2", "$.properties.location.lat"],
-  ["$alias3", "$.properties.tags[*]"]
-];
-
-function extractJsonPaths(feature, paths) {
-  const result = {};
-
-  for (const [name, jsonPath] of paths) {
-    try {
-      const value = jp.query(feature, jsonPath);
-      // decide whether to store arrays or single values:
-      result[name] = value.length === 1 ? value[0] : value;
-    } catch (err) {
-      result[name] = null; // or undefined / error message
-      console.error(`Error evaluating JSONPath for ${name}:`, err.message);
-    }
-  }
-
-  return result;
-}
-
- return ["searchable",extractJsonPaths(feature, paths) ];
-}
-
-    """;
-}
-
   private Set<Modification> toModifications(RoutingContext context, Space space, FeatureModificationList featureModificationList,
       boolean versionConflictDetectionEnabled) {
     List<String> featureHooks = new ArrayList<>();
-    List<String> writeHooks = new ArrayList<>();
     List<String> addTags = getAddTags(context);
     List<String> removeTags = getRemoveTags(context);
     String idPrefix = getIdPrefix(context);
@@ -639,20 +603,13 @@ private Set<Modification> toModifications(RoutingContext context, Space space, F
     if (idPrefix != null)
       featureHooks.add("feature => feature.id = \"" + idPrefix + "\" + feature.id");
 
-    boolean useWriteHook = true;
-
-    if( useWriteHook )
-     writeHooks.add(writeHook()); // todo: pass searchable aliases to use within writehook
-
     return featureModificationList.getModifications().stream()
         .map(modification -> new Modification()
             .withFeatureData(modification.getFeatureData())
             .withUpdateStrategy(toUpdateStrategy(space, modification.getOnFeatureExists(), modification.getOnFeatureNotExists(),
                 modification.getOnMergeConflict(), versionConflictDetectionEnabled))
             .withPartialUpdates(modification.getOnFeatureExists() == PATCH)
-            .withFeatureHooks(featureHooks.isEmpty() ? null : featureHooks)
-            .withWriteHooks(writeHooks.isEmpty() ? null : writeHooks)
-            )
+            .withFeatureHooks(featureHooks.isEmpty() ? null : featureHooks))
         .collect(Collectors.toSet());
   }
 

xyz-hub-service/src/main/java/com/here/xyz/hub/task/FeatureHandler.java

@@ -19,8 +19,8 @@
 
 package com.here.xyz.hub.task;
 
-import static com.here.xyz.hub.task.FeatureTaskHandler.injectMinVersion;
 import static com.here.xyz.hub.task.FeatureTask.resolveBranchFor;
+import static com.here.xyz.hub.task.FeatureTaskHandler.injectMinVersion;
 import static com.here.xyz.util.service.rest.TooManyRequestsException.ThrottlingReason.MEMORY;
 import static com.here.xyz.util.service.rest.TooManyRequestsException.ThrottlingReason.STORAGE_QUEUE_FULL;
 import static io.netty.handler.codec.http.HttpResponseStatus.BAD_GATEWAY;
@@ -65,6 +65,7 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.LongAdder;
+import java.util.stream.Collectors;
 import net.jodah.expiringmap.ExpirationPolicy;
 import net.jodah.expiringmap.ExpiringMap;
 import org.apache.logging.log4j.LogManager;
@@ -95,7 +96,12 @@ public static Future<FeatureCollection> writeFeatures(Marker marker, Space space
           .withContext(spaceContext)
           .withAuthor(author)
           .withResponseDataExpected(responseDataExpected)
-          .withRef(baseRef);
+          .withRef(baseRef)
+          .withSearchableProperties(space.getSearchableProperties().entrySet().stream()
+              .collect(Collectors.toMap(
+                  e -> e.getKey().startsWith("$") ? e.getKey().substring(1) : e.getKey(),
+                  e -> e.getKey().startsWith("$") ? toJsonPath(e.getKey().substring(e.getKey().indexOf(":") + 1)) : toJsonPath(e.getKey())
+              )));
 
       return Future.all(injectMinVersion(marker, space.getId(), event), injectSpaceParams(event, space))
           .compose(v -> {
@@ -131,6 +137,15 @@ else if (ar.result() instanceof FeatureCollection featureCollection)
     }
   }
 
+  private static String toJsonPath(String jsonPathPointer) {
+    if (jsonPathPointer.startsWith("$"))
+      //The path pointer is already a JSONPath
+      return jsonPathPointer;
+
+    //TODO: Translate the path pointer like "prop1.prop2[0].prop3" to JSONPath "$.prop1.prop2[0].prop3" for all cases correctly
+    return "$." + jsonPathPointer;
+  }
+
   static void throttle(Space space) throws HttpException {
     Connector storage = space.getResolvedStorageConnector();
     final long GLOBAL_INFLIGHT_REQUEST_MEMORY_SIZE = (long) Service.configuration.GLOBAL_INFLIGHT_REQUEST_MEMORY_SIZE_MB * 1024 * 1024;

xyz-models/src/main/java/com/here/xyz/events/WriteFeaturesEvent.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2017-2024 HERE Europe B.V.
+ * Copyright (C) 2017-2025 HERE Europe B.V.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,11 +21,18 @@
 
 import com.here.xyz.models.geojson.implementation.FeatureCollection;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 public class WriteFeaturesEvent extends ContextAwareEvent<WriteFeaturesEvent> {
   private Set<Modification> modifications;
   private boolean responseDataExpected;
+  /**
+   * A map of properties that are searchable.
+   * The key is the name of the alias / field that should be indexed.
+   * The value is the JSON-Pointer to the property in the feature or a JSONPath expression.
+   */
+  private Map<String, String> searchableProperties;
 
   public Set<Modification> getModifications() {
     return modifications;
@@ -53,13 +60,25 @@ public WriteFeaturesEvent withResponseDataExpected(boolean responseDataExpected)
     return this;
   }
 
+  public Map<String, String> getSearchableProperties() {
+    return searchableProperties;
+  }
+
+  public void setSearchableProperties(Map<String, String> searchableProperties) {
+    this.searchableProperties = searchableProperties;
+  }
+
+  public WriteFeaturesEvent withSearchableProperties(Map<String, String> searchableProperties) {
+    setSearchableProperties(searchableProperties);
+    return this;
+  }
+
   public static class Modification {
     private UpdateStrategy updateStrategy;
     private FeatureCollection featureData;
     private List<String> featureIds; //To be used only for deletions
     private boolean partialUpdates;
     private List<String> featureHooks; //NOTE: The featureHooks will be applied in the given order
-    private List<String> writeHooks; //NOTE: The Hooks will be applied in the given order
 
     public UpdateStrategy getUpdateStrategy() {
       return updateStrategy;
@@ -125,20 +144,5 @@ public Modification withFeatureHooks(List<String> featureHooks) {
       setFeatureHooks(featureHooks);
       return this;
     }
-
-    public List<String> getWriteHooks() {
-      return writeHooks;
-    }
-
-    public void setWriteHooks(List<String> writeHooks) {
-      this.writeHooks = writeHooks;
-    }
-
-    public Modification withWriteHooks(List<String> writeHooks) {
-      setWriteHooks(writeHooks);
-      return this;
-    }
-
-
   }
 }

xyz-psql-connector/src/main/java/com/here/xyz/psql/query/WriteFeatures.java

@@ -42,6 +42,7 @@
 import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -96,6 +97,9 @@ protected SQLQuery buildQuery(WriteFeaturesEvent event) throws ErrorResponseExce
     if (event.getRef() != null && event.getRef().isSingleVersion() && !event.getRef().isHead())
       queryContextBuilder.withBaseVersion(event.getRef().getVersion());
 
+    if (event.getSearchableProperties() != null && !event.getSearchableProperties().isEmpty())
+      queryContextBuilder.with("writeHooks", List.of(writeHook(event.getSearchableProperties())));
+
     return new SQLQuery("SELECT write_features(#{modifications}, 'Modifications', #{author}, #{responseDataExpected});")
         .withLoggingEnabled(false)
         .withContext(queryContextBuilder.build())
@@ -134,4 +138,27 @@ public FeatureCollection handle(ResultSet rs) throws SQLException {
       throw new RuntimeException("Error parsing query result.", e);
     }
   }
+
+  private String writeHook(Map<String, String> searchableProperties) {
+    return """
+        feature => {
+          const searchableProperties = ${searchableProperties};
+          let searchables = {};
+          
+          for (const alias in searchableProperties) {
+            const jsonPath = searchableProperties[alias];
+            try {
+              searchables[alias] = jsonpath_rfc9535.query(feature, jsonPath)[0];
+            }
+            catch (err) {
+              throw new Error(`Error evaluating JSONPath for alias ${alias} and expression ${jsonPath} message: ${err.message}`);
+            }
+          }
+          
+          return {
+            "searchable": searchables
+          };
+        }
+        """.replace("${searchableProperties}", XyzSerializable.serialize(searchableProperties));
+  }
 }

xyz-util/src/main/java/com/here/xyz/util/db/pg/Script.java

@@ -171,10 +171,6 @@ private void install(String targetSchema, boolean deleteBefore) throws SQLExcept
     logger.info("Installing script {} on DB {} into schema {} ...", getScriptName(), getDbId(), targetSchema);
 
     SQLQuery scriptContent = loadSubstitutedScriptContent();
-
-    if( "common.sql".equals(getScriptName()) )
-     scriptContent = addJsLibRegisterFunctions(scriptContent);
-
     List<SQLQuery> installationQueries = new ArrayList<>();
     if (deleteBefore) {
       //TODO: Remove following workaround once "drop schema cascade"-bug creating orphaned functions is fixed in postgres
@@ -315,9 +311,7 @@ private static List<String> scanResourceFolderWA(String resourceFolder, String f
   private static List<String> scanResourceFolder(ScriptResourcePath scriptResourcePath, String fileSuffix) throws IOException {
     String resourceFolder = scriptResourcePath.path();
     //TODO: Remove this workaround once the actual implementation of this method supports scanning folders inside a JAR
-    if (    "/sql".equals(resourceFolder)
-         || "/sql/lib-js".equals(resourceFolder)
-         || "/jobs".equals(resourceFolder))
+    if ("/sql".equals(resourceFolder) || "/jobs".equals(resourceFolder))
       return ensureInitScriptIsFirst(scanResourceFolderWA(resourceFolder, fileSuffix), scriptResourcePath.initScript());
 
     final InputStream folderResource = Script.class.getResourceAsStream(resourceFolder);

xyz-util/src/main/resources/sql/DatabaseWriter.js

@@ -314,6 +314,12 @@ class DatabaseWriter {
       return this.execute()[0];
   }
 
+  createJsonData(inputFeature) {
+    let writeHooks = queryContext().writeHooks;
+    //TODO: Translate the inputFeature into the value for the jsondata column (also apply the write hooks here if some are specified)
+    return inputFeature;
+  }
+
   _PARTITION_SIZE() {
     return queryContext().PARTITION_SIZE || 100000; //TODO: Ensure the partition size is always set in the query context
   }

xyz-util/src/main/resources/sql/FeatureWriter.js

@@ -43,7 +43,6 @@ class FeatureWriter {
   isPartial;
   baseVersion;
   featureHooks;
-  writeHooks;
 
   onExists;
   onNotExists;
@@ -906,11 +905,11 @@ class FeatureWriter {
   /**
    * @returns {FeatureCollection}
    */
-  static writeFeatures(inputFeatures, author, onExists, onNotExists, onVersionConflict, onMergeConflict, isPartial, featureHooks, writeHooks, version = FeatureWriter.getNextVersion()) {
+  static writeFeatures(inputFeatures, author, onExists, onNotExists, onVersionConflict, onMergeConflict, isPartial, featureHooks, version = FeatureWriter.getNextVersion()) {
     FeatureWriter.dbWriter = new DatabaseWriter(queryContext().schema, FeatureWriter._targetTable(), FeatureWriter._tableBaseVersions().at(-1), FW_BATCH_MODE(), queryContext().tableLayout);
     let result = this.newFeatureCollection();
     for (let feature of inputFeatures) {
-      let execution = new FeatureWriter(feature, version, author, onExists, onNotExists, onVersionConflict, onMergeConflict, isPartial, featureHooks, writeHooks).writeFeature();
+      let execution = new FeatureWriter(feature, version, author, onExists, onNotExists, onVersionConflict, onMergeConflict, isPartial, featureHooks).writeFeature();
       this._collectResult(execution, result);
     }
 
@@ -952,18 +951,16 @@ class FeatureWriter {
     let featureCollections = featureModifications.map(modification => FeatureWriter.writeFeatures(this.toFeatureList(modification),
         author, modification.updateStrategy.onExists, modification.updateStrategy.onNotExists,
         modification.updateStrategy.onVersionConflict, modification.updateStrategy.onMergeConflict, modification.partialUpdates,
-        modification.featureHooks && modification.featureHooks.map(hook => eval(hook)),
-        modification.writeHooks && modification.writeHooks.map(hook => eval(hook)),
-        version));
+        modification.featureHooks && modification.featureHooks.map(hook => eval(hook)), version));
     return this.combineResults(featureCollections);
   }
 
   /**
    * @returns {FeatureCollection}
    */
-  static writeFeature(inputFeature, author, onExists, onNotExists, onVersionConflict, onMergeConflict, isPartial, featureHooks, writeHooks, version = undefined) {
+  static writeFeature(inputFeature, author, onExists, onNotExists, onVersionConflict, onMergeConflict, isPartial, featureHooks, version = undefined) {
     return FeatureWriter.writeFeatures([inputFeature], author, onExists, onNotExists, onVersionConflict, onMergeConflict,
-        isPartial, featureHooks, writeHooks, version);
+        isPartial, featureHooks, version);
   }
 }
 

xyz-util/src/main/resources/sql/lib-js/sample_hello.min.js

@@ -88,7 +88,7 @@ class TestFeatureWriter {
   }
 
   run() {
-    let result = FeatureWriter.writeFeature(this.inputFeature, null, "REPLACE", "CREATE", null, null, false, null, null);
+    let result = FeatureWriter.writeFeature(this.inputFeature, null, "REPLACE", "CREATE", null, null, false, null);
     console.log("Returned result from FeatureWriter: ", result);
   }