zarr-developers · JacinthEsther · Nov 3, 2022
diff --git a/.gitignore b/.gitignore
@@ -8,3 +8,16 @@ report.*
 
 # ignore data subdirectories
 data/**/
+.github
+.gitignore
+README.md
+environment.yml
+doc/**/
+examples/**/
+generate_data/jzarr/**/
+generate_data/js/**/
+generate_data/n5-java/**/
+xtensor_zarr/**/
+test/**/
+
+
diff --git a/generate_data/netcdf-java/generate_data.sh b/generate_data/netcdf-java/generate_data.sh
@@ -0,0 +1,19 @@
+# cd to this directory
+# https://stackoverflow.com/a/6393573/2700168
+cd "${0%/*}"
+
+set -e
+set -u
+set -x
+
+MVN_FLAGS=${MVN_FLAGS:-"--no-transfer-progress"}
+mvn "${MVN_FLAGS}" clean package
+
+java -cp target/jzarr-1.0.0.jar zarr_implementations.jzarr.App "$@" && {
+    # Workaround for: https://github.com/bcdev/jzarr/issues/25
+    find ../../data/jzarr* -name .zarray -exec sed -ibak 's/>u1/|u1/' {} \;
+} || {
+    echo jzarr failed
+    exit 2
+}
+
diff --git a/generate_data/netcdf-java/pom.xml b/generate_data/netcdf-java/pom.xml
@@ -0,0 +1,71 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>zarr_implementations</groupId>
+  <artifactId>netcdf-java</artifactId>
+  <packaging>jar</packaging>
+  <version>1.0.0</version>
+  <name>netcdf-java</name>
+  <url>http://maven.apache.org</url>
+  <dependencies>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.13.2</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.bc.zarr</groupId>
+      <artifactId>jzarr</artifactId>
+      <version>0.3.3</version>
+    </dependency>
+  </dependencies>
+  <repositories>
+     <repository>
+        <id>bc-nexus-repo</id>
+        <name>Brockmann-Consult Public Maven Repository</name>
+        <url>https://nexus.senbox.net/nexus/content/groups/public/</url>
+     </repository>
+  </repositories>
+  <build>
+    <plugins>
+      <plugin>
+        <artifactId>maven-enforcer-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>enforce-rules</id>
+            <phase>none</phase>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>3.3</version>
+        <configuration>
+          <source>1.8</source>
+          <target>1.8</target>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <version>2.4.3</version>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <createDependencyReducedPom>true</createDependencyReducedPom>
+              <dependencyReducedPomLocation>
+                ${java.io.tmpdir}/dependency-reduced-pom.xml
+              </dependencyReducedPomLocation>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>
diff --git a/generate_data/netcdf-java/src/main/java/zarr_implementations/netcdf/App.java b/generate_data/netcdf-java/src/main/java/zarr_implementations/netcdf/App.java
@@ -0,0 +1,117 @@
+package zarr_implementations.netcdf;
+
+import com.bc.zarr.*;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+
+
+public class App {
+
+    enum Compression {
+        raw("null"),
+        zlib("zlib");
+//        blosc("blosc");
+
+        private final String value;
+
+        private Compression(final String value) {
+            this.value = value;
+        }
+
+        @Override
+        public String toString() {
+            return value;
+        }
+    }
+
+    // NOTE for now we use 100, 100, 1 as block-size in all examples
+    // maybe it's a better idea to make this more irregular though
+    private static final int WIDTH = 512;
+    private static final int HEIGHT = 512;
+    private static final int CHANNELS = 3;
+    private static final int[] CHUNKS = new int[]{100, 100, 1};
+    private static final int[] SHAPE = new int[] {WIDTH, HEIGHT, CHANNELS};
+    private static final Path IN_PATH = Paths.get("..", "..", "data", "reference_image.png");
+    private static final Path OUT_PATH = Paths.get("..", "..", "data", "jzarr_flat.zr");
+
+    private static int[] getTestData() throws IOException {
+        final BufferedImage image = ImageIO.read(new File("data/reference_image.png"));
+        int[] result = new int[WIDTH * HEIGHT * CHANNELS];
+        for (int i = 0; i < WIDTH; i++) {
+            for (int j = 0; j < HEIGHT; j++) {
+                Color color = new Color(image.getRGB(i, j));
+                int index = (WIDTH*3*j) + (3*i);
+                result[index] = color.getRed();
+                result[index + 1] = color.getGreen();
+                result[index + 2] = color.getBlue();
+            }
+        }
+        return result;
+    }
+
+
+    private static int[] getArrayData(ZarrArray zarr) throws Exception {
+        int[] data = new int[WIDTH * HEIGHT * CHANNELS];
+        zarr.read(data, SHAPE, new int[]{0, 0, 0});
+        int[] unsigned = new int[data.length];
+        for (int i = 0; i < data.length; i++) {
+            unsigned[i] = data[i] & 0xff;
+        }
+        return unsigned;
+    }
+
+    public static void main(String[] args) throws Exception {
+
+        if (args.length != 0 && args.length != 3) {
+            System.out.println("usage: App");
+            System.out.println("usage: App -verify fpath dsname");
+            System.exit(2);  // EARLY EXIT
+        } else if (args.length == 3) {
+            String fpath = args[1];
+            String dsname = args[2];
+            ZarrArray verification = ZarrGroup.open(fpath).openArray(dsname);
+            int[] shape = verification.getShape();
+            if (!Arrays.equals(SHAPE, shape)) {
+                throw new RuntimeException(String.format(
+                            "shape-mismatch expected:%s found:%s",
+                            Arrays.toString(SHAPE), Arrays.toString(shape)
+                ));
+            }
+
+            int[] test = getTestData();
+            int[] verify = getArrayData(verification);
+            if (!Arrays.equals(test, verify)) {
+                throw new RuntimeException("values don't match");
+            }
+            return;  // EARLY EXIT
+        }
+
+        int[] data = getTestData();
+
+        final ZarrGroup container = ZarrGroup.create(OUT_PATH);
+        for (final Compression compressionType : Compression.values()) {
+            ArrayParams arrayParams = new ArrayParams()
+                .shape(SHAPE)
+                .chunks(CHUNKS)
+                .dataType(DataType.u1)
+                // .nested(nested) FIXME: requires a different branch
+                .compressor(CompressorFactory.create(compressionType.toString()));  // jzarr name, "null"
+
+            String dsname = compressionType.name(); // zarr_implementation name, "raw"
+            if ("blosc".equals(dsname)) {
+                dsname = "blosc/lz4"; // FIXME: better workaround?
+            }
+            Path subdir = OUT_PATH.resolve(dsname);
+            ZarrArray zArray = ZarrArray.create(subdir, arrayParams);
+            // final ZarrArray zarr = ZarrArray.open(getRootPath().resolve(pathName));
+            zArray.write(data, SHAPE, new int[]{0, 0, 0});
+        }
+    }
+}