From f3e015b48021099dfadf762b420c738e0921946b Mon Sep 17 00:00:00 2001 From: Timm Erxleben Date: Fri, 17 Mar 2023 18:38:00 +0100 Subject: [PATCH 1/7] test julea-h5migrate in CI --- .github/workflows/ci.yml | 16 ++++++++++- scripts/gen_hdf5_test_file.py | 51 +++++++++++++++++++++++++++++++++++ scripts/spack | 2 ++ 3 files changed, 68 insertions(+), 1 deletion(-) create mode 100755 scripts/gen_hdf5_test_file.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e05afcbb7..ffd5d9688 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,7 +130,7 @@ jobs: sudo apt --yes --no-install-recommends install ninja-build pkgconf libglib2.0-dev libbson-dev libfabric-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse3-dev libopen-trace-format-dev librados-dev if test "${{ matrix.os.dist }}" = 'ubuntu-22.04' then - sudo apt --yes --no-install-recommends install meson + sudo apt --yes --no-install-recommends install meson python3 python3-pip python3-setuptools python3-wheel else sudo apt --yes --no-install-recommends install python3 python3-pip python3-setuptools python3-wheel sudo pip3 install meson @@ -342,6 +342,20 @@ jobs: sleep 10 ./scripts/test.sh -r /hdf5 ./scripts/setup.sh stop + - name: Test julea-h5migrate + env: + LSAN_OPTIONS: exitcode=0 + run: | + . scripts/environment.sh + sudo pip3 install h5py numpy + scripts/gen_hdf5_test_file.py + ./scripts/setup.sh start + HDF5_VOL_CONNECTOR=julea-db julea-h5migrate testfile.h5 julea-db://imported.h5 + HDF5_VOL_CONNECTOR=julea-db julea-h5migrate julea-db://imported.h5 exported.h5 + h5dump testfile.h5 | tail -n+2 > original + h5dump exported.h5 | tail -n+2 > exported + diff original exported + ./scripts/setup.sh stop doxygen: name: Doxygen runs-on: ubuntu-22.04 diff --git a/scripts/gen_hdf5_test_file.py b/scripts/gen_hdf5_test_file.py new file mode 100755 index 000000000..78d3098f8 --- /dev/null +++ b/scripts/gen_hdf5_test_file.py @@ -0,0 +1,51 @@ +#!/bin/python + +# JULEA - Flexible storage framework +# Copyright (C) 2023 Timm Leon Erxleben +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +# This is a simple script to generate HDF5 files for tests. + +import h5py +import numpy as np + +rng = np.random.default_rng() + +f = h5py.File('testfile.h5', 'w') + +# create some groups +g1 = f.create_group("g1") +g1_1 = g1.create_group("g1_1") +g2 = f.create_group("g2") + +# create some datasets with different data types and sizes +f["2D-f64"] = rng.random(size=(10, 8), dtype=np.float64) +g1["3D-f64"] = rng.random(size=(5, 10, 3), dtype=np.float64) +g1_1["1D-f32"] = rng.random(size=(10), dtype=np.float32) +g2["0D-int"] = rng.integers(0, 500, dtype=np.uint32) +g2["4D-int64"] = rng.integers(0, 500, size=(2,2,2,2), dtype=np.int64) +g1["4D-int32"] = rng.integers(0, 500, size=(2,2,2,2), dtype=np.int32) +g2["string-set"] = np.bytes_("I am a fixed size string!") +#2["var-string-set"] = "I am a variable sized string!" + +# create some attributes +f.attrs["file-attr"] = 42 +#g1.attrs["group1-attr"] = "I am a group attribute and a variable sized string" +g1.attrs["group1-attr-fixed-str"] = np.bytes_("I am a group attribute and a fixed size string") +g2.attrs["group2-attr"] = rng.random(size=(2,2)) +g2["string-set"].attrs["dset-attr"] = 3.14159 + +f.close() + diff --git a/scripts/spack b/scripts/spack index fd3377569..0ddd88f4f 100644 --- a/scripts/spack +++ b/scripts/spack @@ -157,6 +157,8 @@ spack_get_dependencies () if test -n "${CI}" then dependencies="${dependencies} py-gcovr" + dependencies="${dependencies} py-pip" + dependencies="${dependencies} python" fi #dependencies="${dependencies} enzo@main" From 8002d6f635c83ec7969c9b6f4887e5d32544a1fb Mon Sep 17 00:00:00 2001 From: Timm Erxleben Date: Mon, 20 Mar 2023 10:24:40 +0100 Subject: [PATCH 2/7] make python script shebang portable --- scripts/gen_hdf5_test_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gen_hdf5_test_file.py b/scripts/gen_hdf5_test_file.py index 78d3098f8..8b08fbfbe 100755 --- a/scripts/gen_hdf5_test_file.py +++ b/scripts/gen_hdf5_test_file.py @@ -1,4 +1,4 @@ -#!/bin/python +#!/bin/env python3 # JULEA - Flexible storage framework # Copyright (C) 2023 Timm Leon Erxleben From 77b7f5daf937cbb5da7ae09b8e34ecb69c4906aa Mon Sep 17 00:00:00 2001 From: Timm Erxleben Date: Mon, 20 Mar 2023 12:22:55 +0100 Subject: [PATCH 3/7] explicitely call python and try pip3 from user --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffd5d9688..7a3c8a5fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -347,8 +347,8 @@ jobs: LSAN_OPTIONS: exitcode=0 run: | . scripts/environment.sh - sudo pip3 install h5py numpy - scripts/gen_hdf5_test_file.py + pip3 install h5py numpy + python3 scripts/gen_hdf5_test_file.py ./scripts/setup.sh start HDF5_VOL_CONNECTOR=julea-db julea-h5migrate testfile.h5 julea-db://imported.h5 HDF5_VOL_CONNECTOR=julea-db julea-h5migrate julea-db://imported.h5 exported.h5 From 64090871c7c89522846177b854dbf2370f48cac4 Mon Sep 17 00:00:00 2001 From: Timm Erxleben Date: Mon, 20 Mar 2023 12:59:54 +0100 Subject: [PATCH 4/7] move python package install to spack since system dependencies are not tested anyway, also deactivate tests for noew --- .github/workflows/ci.yml | 3 ++- scripts/spack | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a3c8a5fd..dd4b6b581 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -317,6 +317,7 @@ jobs: if test "${{ matrix.julea.db }}" = 'mysql'; then JULEA_DB_PATH='127.0.0.1:juleadb:julea:aeluj'; fi julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend="${{ matrix.julea.object }}" --object-component=server --object-path="/tmp/julea/object/${{ matrix.julea.object }}" --kv-backend="${{ matrix.julea.kv }}" --kv-component="${JULEA_KV_COMPONENT}" --kv-path="${JULEA_KV_PATH}" --db-backend="${{ matrix.julea.db }}" --db-component="${JULEA_DB_COMPONENT}" --db-path="${JULEA_DB_PATH}" - name: Tests + if: false env: LSAN_OPTIONS: exitcode=0 run: | @@ -327,6 +328,7 @@ jobs: ./scripts/test.sh ./scripts/setup.sh stop - name: HDF5 Tests + if: false env: LSAN_OPTIONS: exitcode=0 run: | @@ -347,7 +349,6 @@ jobs: LSAN_OPTIONS: exitcode=0 run: | . scripts/environment.sh - pip3 install h5py numpy python3 scripts/gen_hdf5_test_file.py ./scripts/setup.sh start HDF5_VOL_CONNECTOR=julea-db julea-h5migrate testfile.h5 julea-db://imported.h5 diff --git a/scripts/spack b/scripts/spack index 0ddd88f4f..03b065d4c 100644 --- a/scripts/spack +++ b/scripts/spack @@ -156,9 +156,10 @@ spack_get_dependencies () if test -n "${CI}" then - dependencies="${dependencies} py-gcovr" - dependencies="${dependencies} py-pip" dependencies="${dependencies} python" + dependencies="${dependencies} py-gcovr" + dependencies="${dependencies} py-numpy" + dependencies="${dependencies} py-h5py" fi #dependencies="${dependencies} enzo@main" From 9c1d9e2a80adee04519717937595c1f842c15c04 Mon Sep 17 00:00:00 2001 From: Timm Erxleben Date: Mon, 20 Mar 2023 17:43:51 +0100 Subject: [PATCH 5/7] fall back to pip due to failing spack build of openblas with clang --- .github/workflows/ci.yml | 1 + scripts/spack | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd4b6b581..79abe74f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -349,6 +349,7 @@ jobs: LSAN_OPTIONS: exitcode=0 run: | . scripts/environment.sh + pip3 install h5py numpy python3 scripts/gen_hdf5_test_file.py ./scripts/setup.sh start HDF5_VOL_CONNECTOR=julea-db julea-h5migrate testfile.h5 julea-db://imported.h5 diff --git a/scripts/spack b/scripts/spack index 03b065d4c..aaea25e35 100644 --- a/scripts/spack +++ b/scripts/spack @@ -158,8 +158,7 @@ spack_get_dependencies () then dependencies="${dependencies} python" dependencies="${dependencies} py-gcovr" - dependencies="${dependencies} py-numpy" - dependencies="${dependencies} py-h5py" + dependencies="${dependencies} py-pip" fi #dependencies="${dependencies} enzo@main" From cda7ed4f764cf396736a05ead10918ef468fd95e Mon Sep 17 00:00:00 2001 From: Timm Erxleben Date: Tue, 21 Mar 2023 14:11:56 +0100 Subject: [PATCH 6/7] make pip quiet --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79abe74f6..4b264f072 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -349,7 +349,7 @@ jobs: LSAN_OPTIONS: exitcode=0 run: | . scripts/environment.sh - pip3 install h5py numpy + pip3 -qqq install h5py numpy python3 scripts/gen_hdf5_test_file.py ./scripts/setup.sh start HDF5_VOL_CONNECTOR=julea-db julea-h5migrate testfile.h5 julea-db://imported.h5 From 42bd1ab11ba3bd38f2d23433619d276e6777bf73 Mon Sep 17 00:00:00 2001 From: Timm Erxleben Date: Wed, 12 Apr 2023 11:04:36 +0200 Subject: [PATCH 7/7] reactivate tests and deactivate h5migrate for now --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b264f072..3564aaa94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -317,7 +317,6 @@ jobs: if test "${{ matrix.julea.db }}" = 'mysql'; then JULEA_DB_PATH='127.0.0.1:juleadb:julea:aeluj'; fi julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend="${{ matrix.julea.object }}" --object-component=server --object-path="/tmp/julea/object/${{ matrix.julea.object }}" --kv-backend="${{ matrix.julea.kv }}" --kv-component="${JULEA_KV_COMPONENT}" --kv-path="${JULEA_KV_PATH}" --db-backend="${{ matrix.julea.db }}" --db-component="${JULEA_DB_COMPONENT}" --db-path="${JULEA_DB_PATH}" - name: Tests - if: false env: LSAN_OPTIONS: exitcode=0 run: | @@ -328,7 +327,6 @@ jobs: ./scripts/test.sh ./scripts/setup.sh stop - name: HDF5 Tests - if: false env: LSAN_OPTIONS: exitcode=0 run: | @@ -345,6 +343,7 @@ jobs: ./scripts/test.sh -r /hdf5 ./scripts/setup.sh stop - name: Test julea-h5migrate + if: false env: LSAN_OPTIONS: exitcode=0 run: | @@ -354,6 +353,7 @@ jobs: ./scripts/setup.sh start HDF5_VOL_CONNECTOR=julea-db julea-h5migrate testfile.h5 julea-db://imported.h5 HDF5_VOL_CONNECTOR=julea-db julea-h5migrate julea-db://imported.h5 exported.h5 + # tail cuts the filename h5dump testfile.h5 | tail -n+2 > original h5dump exported.h5 | tail -n+2 > exported diff original exported