Handle newline-delimited GeoJSON and large GeoJSON files

planetiler-core/pom.xml

@@ -65,16 +65,6 @@
       <artifactId>gt-epsg-hsql</artifactId>
       <version>${geotools.version}</version>
     </dependency>
-    <dependency>
-      <groupId>org.geotools</groupId>
-      <artifactId>gt-geojson</artifactId>
-      <version>${geotools.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.geotools</groupId>
-      <artifactId>gt-geojson-store</artifactId>
-      <version>${geotools.version}</version>
-    </dependency>
     <dependency>
       <groupId>org.xerial</groupId>
       <artifactId>sqlite-jdbc</artifactId>

planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java

@@ -10,11 +10,11 @@
 import com.onthegomap.planetiler.collection.LongLongMultimap;
 import com.onthegomap.planetiler.config.Arguments;
 import com.onthegomap.planetiler.config.PlanetilerConfig;
-import com.onthegomap.planetiler.reader.GeoJsonReader;
 import com.onthegomap.planetiler.reader.GeoPackageReader;
 import com.onthegomap.planetiler.reader.NaturalEarthReader;
 import com.onthegomap.planetiler.reader.ShapefileReader;
 import com.onthegomap.planetiler.reader.SourceFeature;
+import com.onthegomap.planetiler.reader.geojson.GeoJsonReader;
 import com.onthegomap.planetiler.reader.osm.OsmInputFile;
 import com.onthegomap.planetiler.reader.osm.OsmNodeBoundsProvider;
 import com.onthegomap.planetiler.reader.osm.OsmReader;
@@ -437,7 +437,7 @@ public Planetiler addGeoPackageSource(String name, Path defaultPath, String defa
   }
 
   /**
-   * Adds a new GeoJSON source that will be processed when {@link #run()} is called.
+   * Adds a new GeoJSON or newline-delimited GeoJSON source that will be processed when {@link #run()} is called.
    * <p>
    * If the file does not exist and {@code download=true} argument is set, then the file will first be downloaded from
    * {@code defaultUrl}.
@@ -613,7 +613,9 @@ public Planetiler fetchWikidataNameTranslations(Path defaultWikidataCache) {
   public Translations translations() {
     if (translations == null) {
       boolean transliterate = arguments.getBoolean("transliterate", "attempt to transliterate latin names", true);
-      List<String> languages = arguments.getList("languages", "Languages to include labels for. \"default\" expands to the default set of languages configured by the profile. \"-lang\" excludes \"lang\". \"*\" includes every language not listed.", this.defaultLanguages);
+      List<String> languages = arguments.getList("languages",
+        "Languages to include labels for. \"default\" expands to the default set of languages configured by the profile. \"-lang\" excludes \"lang\". \"*\" includes every language not listed.",
+        this.defaultLanguages);
       if (languages.contains("default")) {
         languages = Stream.concat(
           languages.stream().filter(language -> !language.equals("default")),

planetiler-core/src/main/java/com/onthegomap/planetiler/reader/geojson/GeoJson.java

@@ -0,0 +1,80 @@
+package com.onthegomap.planetiler.reader.geojson;
+
+import com.onthegomap.planetiler.reader.FileFormatException;
+import com.onthegomap.planetiler.util.CloseableIterator;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.stream.Stream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A streaming geojson parser that can handle arbitrarily large regular or newline-delimited geojson files without
+ * loading the whole thing in memory.
+ * <p>
+ * This emits every top-level feature or feature contained within a {@code FeatureCollection}. Invalid json syntax will
+ * throw an exception, but unexpected geojson objects will just log a warning and emit empty geometries.
+ *
+ * @see <a href="https://stevage.github.io/ndgeojson/">Newline-delimted geojson</a>
+ * @see <a href="https://datatracker.ietf.org/doc/html/rfc7946">GeoJSON specification (RFC 7946)</a>
+ */
+public class GeoJson implements Iterable<GeoJsonFeature> {
+  private static final Logger LOGGER = LoggerFactory.getLogger(GeoJson.class);
+
+  private final InputStreamSupplier inputStreamSupplier;
+  private final String name;
+
+  private GeoJson(String name, InputStreamSupplier inputStreamSupplier) {
+    this.inputStreamSupplier = inputStreamSupplier;
+    this.name = name;
+  }
+
+  public static GeoJson from(Path path) {
+    return new GeoJson(path.toString(), () -> Files.newInputStream(path));
+  }
+
+  public static GeoJson from(byte[] json) {
+    return new GeoJson(null, () -> new ByteArrayInputStream(json));
+  }
+
+  public static GeoJson from(String json) {
+    return from(json.getBytes(StandardCharsets.UTF_8));
+  }
+
+  public static GeoJson from(InputStreamSupplier inputStreamSupplier) {
+    return new GeoJson(null, inputStreamSupplier);
+  }
+
+  public Stream<GeoJsonFeature> stream() {
+    return iterator().stream();
+  }
+
+  @Override
+  public CloseableIterator<GeoJsonFeature> iterator() {
+    try {
+      return new GeoJsonFeatureIterator(inputStreamSupplier.get(), name);
+    } catch (IOException e) {
+      throw new FileFormatException("Unable to read geojson file", e);
+    }
+  }
+
+  /** Returns the number of geojson features in this document. */
+  public long count() {
+    try {
+      LOGGER.info("Counting geojson features in {}", name);
+      return GeoJsonFeatureCounter.count(inputStreamSupplier.get());
+    } catch (IOException e) {
+      LOGGER.warn("Unable to feature count", e);
+      return 0;
+    }
+  }
+
+  @FunctionalInterface
+  public interface InputStreamSupplier {
+    InputStream get() throws IOException;
+  }
+}

planetiler-core/src/main/java/com/onthegomap/planetiler/reader/geojson/GeoJsonFeature.java

@@ -0,0 +1,14 @@
+package com.onthegomap.planetiler.reader.geojson;
+
+import com.onthegomap.planetiler.reader.WithTags;
+import java.util.Map;
+import org.locationtech.jts.geom.Geometry;
+
+/**
+ * Feature read from a geojson document.
+ *
+ * @param geometry The parsed JTS geometry from {@code geometry} field, or an empty geometry if it was invalid
+ * @param tags     The parsed map from {@code properties} field, or empty map if properties are missing
+ */
+public record GeoJsonFeature(Geometry geometry, @Override Map<String, Object> tags)
+  implements WithTags {}

planetiler-core/src/main/java/com/onthegomap/planetiler/reader/geojson/GeoJsonFeatureCounter.java

@@ -0,0 +1,43 @@
+package com.onthegomap.planetiler.reader.geojson;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Internal streaming utility to count the number of features in a geojson document.
+ * <p>
+ * To simplify processing, it counts the number of "geometry" fields on objects, and descends into any "features" array
+ * to traverse FeatureCollections. This will result in the correct count for valid geojson, but may be off for invalid
+ * geojson.
+ */
+class GeoJsonFeatureCounter {
+  private GeoJsonFeatureCounter() {}
+
+  static long count(InputStream inputStream) throws IOException {
+    long count = 0;
+    try (
+      JsonParser parser =
+        new JsonFactory().enable(JsonParser.Feature.INCLUDE_SOURCE_IN_LOCATION).createParser(inputStream)
+    ) {
+      while (!parser.isClosed()) {
+        JsonToken token = parser.nextToken();
+        if (token == JsonToken.START_ARRAY) {
+          parser.skipChildren();
+        } else if (token == JsonToken.FIELD_NAME) {
+          String name = parser.currentName();
+          parser.nextToken();
+          if ("geometry".equals(name)) {
+            parser.skipChildren();
+            count++;
+          } else if (!"features".equals(name)) {
+            parser.skipChildren();
+          }
+        }
+      }
+    }
+    return count;
+  }
+}