diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e05afcbb7..3564aaa94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,7 +130,7 @@ jobs: sudo apt --yes --no-install-recommends install ninja-build pkgconf libglib2.0-dev libbson-dev libfabric-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse3-dev libopen-trace-format-dev librados-dev if test "${{ matrix.os.dist }}" = 'ubuntu-22.04' then - sudo apt --yes --no-install-recommends install meson + sudo apt --yes --no-install-recommends install meson python3 python3-pip python3-setuptools python3-wheel else sudo apt --yes --no-install-recommends install python3 python3-pip python3-setuptools python3-wheel sudo pip3 install meson @@ -342,6 +342,22 @@ jobs: sleep 10 ./scripts/test.sh -r /hdf5 ./scripts/setup.sh stop + - name: Test julea-h5migrate + if: false + env: + LSAN_OPTIONS: exitcode=0 + run: | + . scripts/environment.sh + pip3 -qqq install h5py numpy + python3 scripts/gen_hdf5_test_file.py + ./scripts/setup.sh start + HDF5_VOL_CONNECTOR=julea-db julea-h5migrate testfile.h5 julea-db://imported.h5 + HDF5_VOL_CONNECTOR=julea-db julea-h5migrate julea-db://imported.h5 exported.h5 + # tail cuts the filename + h5dump testfile.h5 | tail -n+2 > original + h5dump exported.h5 | tail -n+2 > exported + diff original exported + ./scripts/setup.sh stop doxygen: name: Doxygen runs-on: ubuntu-22.04 diff --git a/scripts/gen_hdf5_test_file.py b/scripts/gen_hdf5_test_file.py new file mode 100755 index 000000000..8b08fbfbe --- /dev/null +++ b/scripts/gen_hdf5_test_file.py @@ -0,0 +1,51 @@ +#!/bin/env python3 + +# JULEA - Flexible storage framework +# Copyright (C) 2023 Timm Leon Erxleben +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +# This is a simple script to generate HDF5 files for tests. + +import h5py +import numpy as np + +rng = np.random.default_rng() + +f = h5py.File('testfile.h5', 'w') + +# create some groups +g1 = f.create_group("g1") +g1_1 = g1.create_group("g1_1") +g2 = f.create_group("g2") + +# create some datasets with different data types and sizes +f["2D-f64"] = rng.random(size=(10, 8), dtype=np.float64) +g1["3D-f64"] = rng.random(size=(5, 10, 3), dtype=np.float64) +g1_1["1D-f32"] = rng.random(size=(10), dtype=np.float32) +g2["0D-int"] = rng.integers(0, 500, dtype=np.uint32) +g2["4D-int64"] = rng.integers(0, 500, size=(2,2,2,2), dtype=np.int64) +g1["4D-int32"] = rng.integers(0, 500, size=(2,2,2,2), dtype=np.int32) +g2["string-set"] = np.bytes_("I am a fixed size string!") +#2["var-string-set"] = "I am a variable sized string!" + +# create some attributes +f.attrs["file-attr"] = 42 +#g1.attrs["group1-attr"] = "I am a group attribute and a variable sized string" +g1.attrs["group1-attr-fixed-str"] = np.bytes_("I am a group attribute and a fixed size string") +g2.attrs["group2-attr"] = rng.random(size=(2,2)) +g2["string-set"].attrs["dset-attr"] = 3.14159 + +f.close() + diff --git a/scripts/spack b/scripts/spack index fd3377569..aaea25e35 100644 --- a/scripts/spack +++ b/scripts/spack @@ -156,7 +156,9 @@ spack_get_dependencies () if test -n "${CI}" then + dependencies="${dependencies} python" dependencies="${dependencies} py-gcovr" + dependencies="${dependencies} py-pip" fi #dependencies="${dependencies} enzo@main"