diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e2b5ff4c..8ca47023 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,8 +20,8 @@ env: HSDS_USERNAME: test_user1 HSDS_PASSWORD: test HSDS_PATH: /home/test_user1/ - HDF5_API_TEST_PATH_PREFIX: /home/test_user1/ HSDS_ENDPOINT: http+unix://%2Ftmp%2Fhs%2Fsn_1.sock + HDF5_API_TEST_PATH_PREFIX: /home/test_user1/ HDF5_VOL_CONNECTOR: REST ROOT_DIR: ${{github.workspace}}/hsdsdata BUCKET_NAME: hsdstest @@ -178,7 +178,7 @@ jobs: ROOT_DIR=${{github.workspace}}/hsdadata ./runall.sh --no-docker 1 & sleep 10 working-directory: ${{github.workspace}}/hsds - + - name: Test HSDS run: | python tests/integ/setup_test.py @@ -211,3 +211,144 @@ jobs: # run: | # ctest -R "vol-rest" -VV # working-directory: ${{github.workspace}}/hdf5/build/ + + build_and_test_windows: + strategy: + fail-fast: false + matrix: + os: [windows-latest] + python-version: ["3.10"] + hdf5-branch: ["hdf5_1_14", "develop"] + env: + HSDS_ENDPOINT: http://127.0.0.1:5101 + runs-on: ${{matrix.os}} + steps: + - name: Get HDF5 + uses: actions/checkout@v4 + with: + repository: HDFGroup/hdf5 + ref: ${{matrix.hdf5-branch}} + path: ${{github.workspace}}/hdf5 + + - name: Get REST VOL + uses: actions/checkout@v3 + with: + path: ${{github.workspace}}/vol-rest + + - name: Get HSDS + uses: actions/checkout@v3 + with: + repository: HDFGroup/hsds + path: ${{github.workspace}}/hsds + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install HSDS dependencies + working-directory: ${{github.workspace}}/hsds + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Install HSDS package + working-directory: ${{github.workspace}}/hsds + shell: bash + run: | + pip install -e . + + - name: Start HSDS + shell: bash + working-directory: ${{github.workspace}}/hsds + run: | + cp admin/config/groups.default admin/config/groups.txt + cp admin/config/passwd.default admin/config/passwd.txt + hsds --root_dir data --host localhost --port 5101 --password_file admin/config/passwd.txt --logfile hs.log --loglevel DEBUG --config_dir=admin/config --count=4 & + sleep 10 + + - name: Test HSDS + Setup Folders + run: | + python tests/integ/setup_test.py + working-directory: ${{github.workspace}}/hsds + + - name: Get curl from source + uses: actions/checkout@v4 + with: + repository: curl/curl + path: ${{github.workspace}}/curl + + - name: Install curl + working-directory: ${{github.workspace}}/curl + run: | + mkdir build + mkdir install + cd build + cmake -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/curl/install ` + -DCMAKE_BUILD_TYPE=Release -DBUILD_STATIC_LIBS=ON ` + -DBUILD_SHARED_LIBS=OFF -DCMAKE_C_FLAGS="/DCURL_STATICLIB" ` + -DCURL_STATIC_CRT=ON -DBUILD_STATIC_CURL=ON ` + -DCURL_USE_LIBIDN2=OFF -DCURL_USE_LIBPSL=OFF .. + cmake --build . --config Release + cmake --install . --config Release + + - name: Get yajl from source + uses: actions/checkout@v4 + with: + repository: lloyd/yajl + path: ${{github.workspace}}/yajl + + - name: Install yajl + working-directory: ${{github.workspace}}/yajl + run: | + mkdir build + mkdir install + cd build + cmake -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/yajl/install ` + -DCMAKE_BUILD_TYPE=Release .. + cmake --build . --config Release + cmake --install . --config Release + + - name: CMake Configure + Build HDF5 + run: | + mkdir build + cd build + cmake ` + -DHDF5_BUILD_HL_LIB=ON ` + -DBUILD_SHARED_LIBS=ON -DHDF5_ENABLE_SZIP_SUPPORT=OFF ` + -DHDF5_TEST_API=ON ` + -DHDF5_ENABLE_Z_LIB_SUPPORT=OFF ` + -DCMAKE_BUILD_TYPE=Release -DHDF5_ENABLE_THREADSAFE=OFF ` + -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/hdf5install ` + .. + cmake --build . -j --config Release + cmake --install . --config Release + working-directory: ${{github.workspace}}/hdf5 + + - name: CMake Configure + Build REST VOL + working-directory: ${{github.workspace}}/vol-rest + run: | + mkdir build + cd build + cmake -DHDF5_ROOT=${{github.workspace}}/hdf5install ` + -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/vol-rest/install ` + -DCURL_ROOT=${{github.workspace}}/curl/install/ -DCMAKE_C_FLAGS="/DCURL_STATICLIB" ` + -DCURL_USE_STATIC_LIBS=ON ` + -DYAJL_ROOT=${{github.workspace}}/yajl/install/ ` + -DBUILD_STATIC_LIBS=OFF -DBUILD_SHARED_LIBS=ON ` + -DCMAKE_BUILD_TYPE=Release .. + cmake --build . -j --config Release + cmake --install . --config Release + + - name: Test REST VOL + working-directory: ${{github.workspace}}\vol-rest\build + env: + HDF5_PLUGIN_PATH: ${{github.workspace}}\vol-rest\install\bin + # Manually append lib to PATH here, since env can change between steps on the runner + run: | + $env:PATH="$env:PATH;${{github.workspace}}\vol-rest\install\bin;${{github.workspace}}\hdf5install\bin;${{github.workspace}}\yajl\install\lib" + ctest . -C Release -VV --output-on-failure + + \ No newline at end of file diff --git a/CMake/FindYAJL.cmake b/CMake/FindYAJL.cmake index 63fb4245..06b1d849 100644 --- a/CMake/FindYAJL.cmake +++ b/CMake/FindYAJL.cmake @@ -47,10 +47,12 @@ if (YAJL_FOUND) endif () set_target_properties(yajl-shared PROPERTIES IMPORTED_LOCATION "${YAJL_SHARED_LIBRARY}" + IMPORTED_IMPLIB "${YAJL_SHARED_LIBRARY}" INTERFACE_INCLUDE_DIRECTORIES "${YAJL_INCLUDE_DIR}" ) set_target_properties(yajl-static PROPERTIES IMPORTED_LOCATION "${YAJL_STATIC_LIBRARY}" + IMPORTED_IMPLIB "${YAJL_STATIC_LIBRARY}" INTERFACE_INCLUDE_DIRECTORIES "${YAJL_INCLUDE_DIR}" ) diff --git a/CMakeLists.txt b/CMakeLists.txt index a31fd37e..35d081be 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ # CMake Build Script for the HDF5 REST VOL connector # #----------------------------------------------------# -cmake_minimum_required (VERSION 3.10) +cmake_minimum_required (VERSION 3.12) project (HDF5_VOL_REST C) # Setup CMake policies. @@ -14,7 +14,6 @@ foreach (policy endif () endforeach () - #----------------------------------------------------------------------------- # Instructions for use : Normal Build # @@ -87,7 +86,6 @@ set (HDF5_VOL_REST_LIBSH_TARGET "${HDF5_VOL_REST_LIB_CORENAME}-shared") set (HDF5_VOL_REST_TEST_LIB_TARGET "${HDF5_VOL_REST_TEST_LIB_CORENAME}-static") set (HDF5_VOL_REST_TEST_LIBSH_TARGET "${HDF5_VOL_REST_TEST_LIB_CORENAME}-shared") - #----------------------------------------------------------------------------- # Define some CMake variables for use later in the project #----------------------------------------------------------------------------- @@ -118,12 +116,15 @@ if (HDF5_ENABLE_THREADSAFE AND NOT Threads_FOUND) find_package(Threads) endif () -find_package (CURL 7.61 REQUIRED) +set (CURL_TARGET_LIBTYPE shared) + +find_package (CURL 7.61 MODULE REQUIRED) + if (CURL_FOUND) include_directories(${CURL_INCLUDE_DIRS}) set (LINK_LIBS ${LINK_LIBS} CURL::libcurl) else () - message (FATAL_ERROR "cURL not found; please check CURL_INCLUDE_DIR") + message (FATAL_ERROR "cURL not found; please check CURL_ROOT") endif () # Include custom module for finding YAJL @@ -283,11 +284,7 @@ endmacro (HDF5_VOL_REST_SET_LIB_OPTIONS) #------------------------------------------------------------------------------- macro (RV_TARGET_C_PROPERTIES wintarget libtype addcompileflags addlinkflags) - if (MSVC) - TARGET_MSVC_PROPERTIES (${wintarget} ${libtype} "${addcompileflags} ${WIN_COMPILE_FLAGS}" "${addlinkflags} ${WIN_LINK_FLAGS}") - else () - set_target_properties (${wintarget} PROPERTIES COMPILE_FLAGS "${addcompileflags}" LINK_FLAGS "${addlinkflags}") - endif () + set_target_properties (${wintarget} PROPERTIES COMPILE_FLAGS "${addcompileflags}" LINK_FLAGS "${addlinkflags}") endmacro () #------------------------------------------------------------------------------- @@ -344,7 +341,7 @@ set (EXTERNAL_LIBRARYDLL_LIST "") #----------------------------------------------------------------------------- # Run all the CMake configuration tests for our build environment #----------------------------------------------------------------------------- -#include (${HDF5_VOL_REST_RESOURCES_DIR}/ConfigureChecks.cmake) +include (${HDF5_VOL_REST_RESOURCES_DIR}/ConfigureChecks.cmake) set (CMAKE_INCLUDE_CURRENT_DIR_IN_INTERFACE ON) diff --git a/README.md b/README.md index d4857444..7b383450 100644 --- a/README.md +++ b/README.md @@ -96,8 +96,6 @@ and can directly be obtained from: `git clone https://github.com/HDFGroup/vol-rest` -For building with the 1.12 or later version of the HDF5 library, use the hdf5_1_12_update branch of this repository. - A source distribution of the HDF5 library has been included in the REST VOL connector source in the `/src/hdf5` directory. @@ -141,9 +139,6 @@ The following configuration options are available to all of the build scripts: mostly useful in helping to diagnose any possible memory leaks or other memory errors within the connector. - -g Enables symbolic debugging of the REST VOL code. (Only available for - `build_vol_autotools.sh`) - -P DIR Specifies where the REST VOL connector should be installed. The default installation prefix is `rest_vol_build` inside the REST VOL connector source root directory. @@ -158,7 +153,15 @@ The following configuration options are available to all of the build scripts: -Y DIR Specifies the top-level directory where YAJL is installed. Used if YAJL is not installed to a system path or used to override -Additionally, the CMake build scripts have the following configuration options: +The following configuration options are specific to `build_vol_autotools.sh`: + + -g Enables symbolic debugging of the REST VOL code. + +The following configuration options are specific to the CMake build scripts: + + -u Specifies that a static cURL library should be used as a dependency. + + -t Specifies that a static YAJL library should be used as a dependency. -B DIR Specifies the directory that CMake should use as the build tree location. The default build tree location is `rest_vol_cmake_build_files` inside the @@ -291,7 +294,8 @@ components mentioned previously cannot be found within the system path. * `HDF5_VOL_REST_ENABLE_MEM_TRACKING` - Enables/Disables memory tracking within the REST VOL connector. This option is mostly useful in helping to diagnose any possible memory leaks or other memory errors within the connector. The default value is `OFF`. * `HDF5_VOL_REST_THREAD_SAFE` - Enables/Disables linking to HDF5 statically compiled with thread safe option. The default value is `OFF`. * `YAJL_USE_STATIC_LIBRARIES` - Indicate if the static YAJL libraries should be used for linking. The default value is `OFF`. - + * `CURL_USE_STATIC_LIBRARIES` - Indicate if the static CURL libraries should be used for linking. The default value is `OFF`. + Note, when setting BUILD_SHARED_LIBS=ON and YAJL_USE_STATIC_LIBRARIES=ON, the static YAJL libraries have be build with the position independent code (PIC) option enabled. In the static YAJL build, this PIC option has been turned off by default. @@ -301,7 +305,7 @@ It is also possible to build the REST VOL as part of the build process for the H ### II.B.v. Build Results -If the build is successful, the following files will be written into the installation directory: +If the build is successful, the following files will be written into the installation directory on a Linux or OSX machine: ``` bin/ @@ -327,6 +331,8 @@ share/ hdf5_vol_rest-targets-.cmake ``` +On a Windows system, the installation directory will have a few differences. If shared libraries are built, the resulting `hdf5_vol_rest.dll` will be found in the `bin` directory. If tests are enabled, `test_rest_vol(-shared).exe` will also be in the `bin` directory. If static libraries are built, `lib` will contain `libhdf5_vol_rest.lib`. + If the REST VOL connector was built using one of the included build scripts, all of the usual files from an HDF5 source build should appear in the respective `bin`, `include`, `lib` and `share` directories in the install directory. Notable among these is `bin/h5cc`, a special-purpose compiler wrapper script that streamlines the process of building HDF5 applications. diff --git a/build_vol_cmake.bat b/build_vol_cmake.bat index 2c58fddb..f029ffe6 100644 --- a/build_vol_cmake.bat +++ b/build_vol_cmake.bat @@ -11,3 +11,223 @@ rem A script used to first configure and build the HDF5 source distribution rem included with the REST VOL connector source code, and then use that built rem HDF5 to build the REST VOL connector itself. +rem Disable showing executed commands to console +@echo off + +rem Do not export variable set in this script to the environment +setlocal + +rem Get the directory of the script itself +set SCRIPT_DIR=%~dp0 + +rem Remove trailing backslash +set SCRIPT_DIR=%SCRIPT_DIR:~0,-1% + +rem Set default install directory +set INSTALL_DIR=%SCRIPT_DIR%\rest_vol_build + +rem Set default build directory +set BUILD_DIR=%SCRIPT_DIR%\rest_vol_cmake_build_files + +rem Determine the number of processors to use when building in parallel +set /A NPROCS=0 + +rem Extra compiler options passed to the various steps, such as -Wall +set COMP_OPTS="-Wall -pedantic -Wunused-macros" + +rem Extra options passed to the REST VOLs CMake script +set CONNECTOR_DEBUG_OPT= +set CURL_DEBUG_OPT= +set MEM_TRACK_OPT= +set THREAD_SAFE_OPT= +set CURL_OPT= +set YAJL_OPT= +set YAJL_LIB_OPT= +set CMAKE_C_FLAGS= +rem On Windows, build type will default to Debug if not specified +set CMAKE_BUILD_TYPE=Release + +echo. +echo ************************* +echo * REST VOL build script * +echo ************************* +echo. + + +call :parse_opts %* + +rem Determine the number of cores to use when parallelizing builds +if %NPROCS% == 0 ( + set NPROCS=%NUMBER_OF_PROCESSORS% +) + +rem Check out vol-tests submodule if subdirectory does not exist +if not exist %SCRIPT_DIR%\test\vol-tests ( + git submodule init + git submodule update +) + +rem Build the REST VOL connector against HDF5 +echo "*******************************************" +echo "* Building REST VOL connector and test suite *" +echo "*******************************************" +echo. + +mkdir %BUILD_DIR% +mkdir %INSTALL_DIR% + +rem Clean out the old CMake cache +del /q %BUILD_DIR%\CMakeCache.txt > nul 2>&1 + +cd %BUILD_DIR% + +echo Configuring build... +cmake -G %CMAKE_GENERATOR% -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% -DCMAKE_C_FLAGS=%CMAKE_C_FLAGS% -DHDF5_ROOT=%HDF5_INSTALL_DIR% -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR% %CURL_OPT% %CURL_LIB_OPT% %YAJL_OPT% %YAJL_LIB_OPT% %CONNECTOR_DEBUG_OPT% %CURL_DEBUG_OPT% %MEM_TRACK_OPT% %THREAD_SAFE_OPT% %SCRIPT_DIR% + +echo Build files generated for CMake generator %CMAKE_GENERATOR% + +cmake --build . -j %NPROCS% --config %CMAKE_BUILD_TYPE% && cmake --install . --config %CMAKE_BUILD_TYPE% + +if %errorlevel% equ 0 ( + echo REST VOL built +) else ( + echo An error occurred during build and installation of REST VOL + EXIT /B 1 +) + +echo Configuring vol-tests + +mkdir %BUILD_DIR%\tests\vol-tests\ +cd %BUILD_DIR%\tests\vol-tests\ + +cmake -G %CMAKE_GENERATOR% -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% -DCMAKE_C_FLAGS=%CMAKE_C_FLAGS% -DHDF5_DIR=%HDF5_INSTALL_DIR% -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR% %CONNECTOR_DEBUG_OPT% %CURL_DEBUG_OPT% %MEM_TRACK_OPT% %THREAD_SAFE_OPT% %SCRIPT_DIR%/test/vol-tests + +echo Build files generated for vol-tests + +cmake --build . -j + +echo VOL tests built + +endlocal +EXIT /B 0 + +:parse_opts +set "optchar=%1" +if "%optchar%"=="" ( + echo Ending arg parse + EXIT /B 0 +) +shift + +if "%optchar%"=="-h" ( + call :usage + EXIT /B 0 +) else if "%optchar%"=="-d" ( + set CONNECTOR_DEBUG_OPT=-DHDF5_VOL_REST_ENABLE_DEBUG=ON + set CMAKE_BUILD_TYPE=Debug + echo Enabled connector debugging + echo. +) else if "%optchar%"=="-c" ( + set CURL_DEBUG_OPT=-DHDF5_VOL_REST_ENABLE_CURL_DEBUG=ON + echo Enabled cURL debugging + echo. +) else if "%optchar%"=="-m" ( + set MEM_TRACK_OPT=-DHDF5_VOL_REST_ENABLE_MEM_TRACKING=ON + echo Enabled connector memory tracking + echo. +) else if "%optchar%"=="-s" ( + set THREAD_SAFE_OPT=-DHDF5_VOL_REST_THREAD_SAFE=ON + echo Enabled connector memory tracking + echo. +) else if "%optchar%"=="-t" ( + set YAJL_LIB_OPT=-DYAJL_USE_STATIC_LIBRARIES=ON + echo Using the static YAJL library + echo. +) else if "%optchar%"=="-u" ( + set CMAKE_C_FLAGS=/DCURL_STATICLIB + set CURL_LIB_OPT=-DCURL_USE_STATIC_LIBRARIES=ON + shift + echo Using the static cURL library + echo. +) else if "%optchar%"=="-G" ( + set CMAKE_GENERATOR=%1 + shift + echo Set CMake generator + echo. +) else if "%optchar%"=="-B" ( + set BUILD_DIR=%1 + shift + echo Set build directory + echo. +) else if "%optchar%"=="-P" ( + set INSTALL_DIR=%1 + shift + echo Set installation prefix + echo. +) else if "%optchar%"=="-H" ( + set HDF5_INSTALL_DIR=%1 + shift + echo Set HDF5 install directory + echo. +) else if "%optchar%"=="-C" ( + set CURL_OPT=-DCURL_ROOT=%1 + shift + echo Set CURL_ROOT + echo. +) else if "%optchar%"=="-Y" ( + set YAJL_OPT=-DYAJL_ROOT=%1 + shift + echo Set YAJL_ROOT + echo. +) else ( + echo ERROR: non-option argument: '%optchar%' >&2 + echo. + call :usage + echo. + EXIT /B 1 +) +goto :parse_opts + +:usage +echo usage: %~dp0 [OPTIONS] +echo. +echo -h Print this help message. +echo. +echo -d Enable debugging output in the REST VOL. +echo -c Enable cURL debugging output in the REST VOL. +echo. +echo -m Enable memory tracking in the REST VOL. +echo. +echo -s Enable linking to thread safe static hdf5 library. +echo. +echo -t Make use of the static YAJL library. Be aware the +echo library should be built with position independent +echo code option enabled. +echo. +echo -u Make use of the static cURL library. +echo. +echo -G Specify the CMake Generator to use for the build +echo files created. +echo. +echo -P DIR Similar to '-DCMAKE_INSTALL_PREFIX=DIR', specifies +echo where the REST VOL should be installed to. Default +echo is 'source directory/rest_vol_build'. +echo. +echo -H DIR To specify a directory where HDF5 has already +echo been installed. Similar to '-DHDF5_ROOT=DIR'. +echo. +echo -B DIR Specifies the directory that CMake should use as +echo the build tree location. Default is +echo 'source directory/rest_vol_cmake_build_files'. +echo Note that the REST VOL does not support in-source +echo CMake builds. +echo. +echo -C DIR To specify the top-level directory where cURL is +echo installed, if cURL was not installed to a system +echo directory. Similar to '-DCURL_ROOT=DIR'. +echo. +echo -Y DIR To specify the top-level directory where YAJL is +echo installed, if YAJL was not installed to a system +echo directory. Similar to '-DYAJL_ROOT=DIR'. +echo. +EXIT /B 0 \ No newline at end of file diff --git a/build_vol_cmake.sh b/build_vol_cmake.sh index 13a519f1..3e7b359a 100755 --- a/build_vol_cmake.sh +++ b/build_vol_cmake.sh @@ -65,6 +65,8 @@ usage() echo " library should be built with position independent" echo " code option enabled." echo + echo " -u Make use of the static cURL library." + echo echo " -G Specify the CMake Generator to use for the build" echo " files created. Default is 'Unix Makefiles'." echo @@ -91,7 +93,7 @@ usage() echo } -optspec=":hctdmstlG:H:C:Y:B:P:-" +optspec=":hctdmstluG:H:C:Y:B:P:-" while getopts "$optspec" optchar; do case "${optchar}" in h) @@ -119,7 +121,11 @@ while getopts "$optspec" optchar; do echo ;; t) YAJL_LIB_OPT="-DYAJL_USE_STATIC_LIBRARIES=ON" - echo "Use the static YAJL library." + echo "Using the static YAJL library." + echo + ;; + u) CURL_LIB_OPT="-DCURL_STATICLIB" + echo "Using the static cURL library." echo ;; G) @@ -189,7 +195,7 @@ rm -f "${BUILD_DIR}/CMakeCache.txt" cd "${BUILD_DIR}" -CFLAGS="-D_POSIX_C_SOURCE=200809L" cmake -G "${CMAKE_GENERATOR}" "-DHDF5_ROOT=${HDF5_INSTALL_DIR}" -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" "${CURL_OPT}" "${YAJL_OPT}" "${YAJL_LIB_OPT}" "${CONNECTOR_DEBUG_OPT}" "${CURL_DEBUG_OPT}" "${MEM_TRACK_OPT}" "${THREAD_SAFE_OPT}" "${SCRIPT_DIR}" +CFLAGS="-D_POSIX_C_SOURCE=200809L" cmake -G "${CMAKE_GENERATOR}" "-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}" "-DHDF5_ROOT=${HDF5_INSTALL_DIR}" -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" "${CURL_OPT}" "${CURL_LIB_OPT}" "${YAJL_OPT}" "${YAJL_LIB_OPT}" "${CONNECTOR_DEBUG_OPT}" "${CURL_DEBUG_OPT}" "${MEM_TRACK_OPT}" "${THREAD_SAFE_OPT}" "${SCRIPT_DIR}" echo "Build files have been generated for CMake generator '${CMAKE_GENERATOR}'" diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake index 468b07be..2b3a699f 100644 --- a/config/cmake/ConfigureChecks.cmake +++ b/config/cmake/ConfigureChecks.cmake @@ -2,7 +2,8 @@ # Include all the necessary files for macros #----------------------------------------------------------------------------- set (HDF_PREFIX "H5") -include (${HDF_RESOURCES_EXT_DIR}/ConfigureChecks.cmake) +include(CheckFunctionExists) +include(CheckLibraryExists) if (NOT DEFINED "H5_DEFAULT_PLUGINDIR") if (WINDOWS) @@ -26,6 +27,24 @@ SET (H5_DEFAULT_VOL H5VL_NATIVE) # END of WINDOWS Hard code Values # ---------------------------------------------------------------------- +# Find the library containing clock_gettime() +CHECK_FUNCTION_EXISTS (clock_gettime CLOCK_GETTIME_IN_LIBC) +CHECK_FUNCTION_EXISTS (gettimeofday RV_HAVE_GETTIMEOFDAY) +CHECK_FUNCTION_EXISTS(strtok_r RV_HAVE_STRTOK_R) +CHECK_FUNCTION_EXISTS(strtok_s RV_HAVE_STRTOK_S) +CHECK_LIBRARY_EXISTS (rt clock_gettime "" CLOCK_GETTIME_IN_LIBRT) +CHECK_LIBRARY_EXISTS (posix4 clock_gettime "" CLOCK_GETTIME_IN_LIBPOSIX4) +if (CLOCK_GETTIME_IN_LIBC) + set (RV_HAVE_CLOCK_GETTIME 1) +elseif (CLOCK_GETTIME_IN_LIBRT) + set (RV_HAVE_CLOCK_GETTIME 1) + list (APPEND LINK_LIBS rt) +elseif (CLOCK_GETTIME_IN_LIBPOSIX4) + set (RV_HAVE_CLOCK_GETTIME 1) + list (APPEND LINK_LIBS posix4) +endif () + + # ----------------------------------------------------------------------- # wrapper script variables # diff --git a/config/cmake/rv_cmake_config.h.in b/config/cmake/rv_cmake_config.h.in index 56d684ed..a616b1ca 100644 --- a/config/cmake/rv_cmake_config.h.in +++ b/config/cmake/rv_cmake_config.h.in @@ -18,6 +18,18 @@ /* Define if the compiler understands the __func__ keyword */ #cmakedefine RV_HAVE_C99_FUNC @RV_HAVE_C99_FUNC@ +/* Define to 1 if you have the `clock_gettime' function. */ +#cmakedefine RV_HAVE_CLOCK_GETTIME @RV_HAVE_CLOCK_GETTIME@ + +/* Define to 1 if you have the `gettimeofday' function. */ +#cmakedefine RV_HAVE_GETTIMEOFDAY @RV_HAVE_GETTIMEOFDAY@ + +/* Define to 1 if you have the `strtok_r` function */ +#cmakedefine RV_HAVE_STRTOK_R @RV_HAVE_STRTOK_R@ + +/* Define to 1 if you have the `strtok_s` function */ +#cmakedefine RV_HAVE_STRTOK_S @RV_HAVE_STRTOK_S@ + /* Define if Darwin or Mac OS X */ #cmakedefine RV_HAVE_DARWIN @RV_HAVE_DARWIN@ diff --git a/configure.ac b/configure.ac index c6adb59d..e3694587 100644 --- a/configure.ac +++ b/configure.ac @@ -610,7 +610,7 @@ AC_CACHE_SAVE ## ---------------------------------------------------------------------- ## Check for functions. -AC_CHECK_FUNCS([snprintf]) +AC_CHECK_FUNCS([snprintf strtok_r strtok_s clock_gettime gettimeofday]) ## ---------------------------------------------------------------------- ## Check compiler characteristics diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 167814e1..2f18c99f 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required (VERSION 3.1.0) +cmake_minimum_required (VERSION 3.12) PROJECT (HDF5_VOL_REST_EXAMPLES) #----------------------------------------------------------------------------- diff --git a/examples/hl/CMakeLists.txt b/examples/hl/CMakeLists.txt index c0a47306..32bfefe2 100644 --- a/examples/hl/CMakeLists.txt +++ b/examples/hl/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required (VERSION 3.1.0) +cmake_minimum_required (VERSION 3.12) PROJECT (HDF5_VOL_REST_HL_EXAMPLES) #----------------------------------------------------------------------------- diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1b25a979..3c3e0f15 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required (VERSION 3.1.0) +cmake_minimum_required (VERSION 3.12) macro (IDE_GENERATED_PROPERTIES SOURCE_PATH HEADERS SOURCES) #set(source_group_path "Source/AIM/${NAME}") diff --git a/src/rest_vol.c b/src/rest_vol.c index a6377254..4b015e14 100644 --- a/src/rest_vol.c +++ b/src/rest_vol.c @@ -30,7 +30,7 @@ #define DATASPACE_SHAPE_BUFFER_DEFAULT_SIZE 256 /* Defines for multi-curl settings */ -#define BACKOFF_INITIAL_DURATION 10000000 /* 10,000,000 ns -> 0.01 sec */ +#define BACKOFF_INITIAL_DURATION 10000 /* 10,000 microseconds -> 0.01 seconds */ #define BACKOFF_SCALE_FACTOR 1.5 #define BACKOFF_MAX_BEFORE_FAIL 3000000000 /* 30,000,000,000 ns -> 30 sec */ @@ -3486,7 +3486,7 @@ RV_parse_server_version(char *HTTP_response, const void *callback_data_in, void FUNC_GOTO_ERROR(H5E_OBJECT, H5E_BADVALUE, FAIL, "server version was NULL"); /* Parse server version into struct */ - if (NULL == (version_field = strtok_r(version_response, ".", &saveptr))) + if (NULL == (version_field = RV_strtok_r(version_response, ".", &saveptr))) FUNC_GOTO_ERROR(H5E_OBJECT, H5E_BADVALUE, FAIL, "server major version field was NULL"); if ((numeric_version_field = (int)strtol(version_field, NULL, 10)) < 0) @@ -3494,7 +3494,7 @@ RV_parse_server_version(char *HTTP_response, const void *callback_data_in, void server_version->major = (size_t)numeric_version_field; - if (NULL == (version_field = strtok_r(NULL, ".", &saveptr))) + if (NULL == (version_field = RV_strtok_r(NULL, ".", &saveptr))) FUNC_GOTO_ERROR(H5E_OBJECT, H5E_BADVALUE, FAIL, "server minor version field was NULL"); if ((numeric_version_field = (int)strtol(version_field, NULL, 10)) < 0) @@ -3502,7 +3502,7 @@ RV_parse_server_version(char *HTTP_response, const void *callback_data_in, void server_version->minor = (size_t)numeric_version_field; - if (NULL == (version_field = strtok_r(NULL, ".", &saveptr))) + if (NULL == (version_field = RV_strtok_r(NULL, ".", &saveptr))) FUNC_GOTO_ERROR(H5E_OBJECT, H5E_BADVALUE, FAIL, "server patch version field was NULL"); if ((numeric_version_field = (int)strtol(version_field, NULL, 10)) < 0) @@ -3732,12 +3732,7 @@ RV_curl_multi_perform(CURL *curl_multi_handle, dataset_transfer_info *transfer_i /* Identify the handle by its original index */ failed_handles_to_retry[handle_index] = curl_multi_msg->easy_handle; - struct timespec tms; - - clock_gettime(CLOCK_MONOTONIC, &tms); - - transfer_info[handle_index].time_of_fail = - (size_t)tms.tv_sec * 1000 * 1000 * 1000 + (size_t)tms.tv_nsec; + transfer_info[handle_index].time_of_fail = (size_t)RV_now_usec(); transfer_info[handle_index].current_backoff_duration = (transfer_info[handle_index].current_backoff_duration == 0) @@ -3877,12 +3872,10 @@ RV_curl_multi_perform(CURL *curl_multi_handle, dataset_transfer_info *transfer_i /* TODO: Replace with an epoll-like structure of some kind, manually iterating this will probably * be slow */ - struct timespec curr_time; - clock_gettime(CLOCK_MONOTONIC, &curr_time); - size_t curr_time_ns = (size_t)curr_time.tv_sec * 1000 * 1000 * 1000 + (size_t)curr_time.tv_nsec; + size_t curr_time_us = (size_t)RV_now_usec(); for (size_t i = 0; i < count; i++) { - if (failed_handles_to_retry[i] && ((curr_time_ns - transfer_info[i].time_of_fail) >= + if (failed_handles_to_retry[i] && ((curr_time_us - transfer_info[i].time_of_fail) >= transfer_info[i].current_backoff_duration)) { if (CURLM_OK != curl_multi_add_handle(curl_multi_handle, failed_handles_to_retry[i])) FUNC_GOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "failed to re-add denied cURL handle"); @@ -4508,3 +4501,68 @@ RV_parse_domain_allocated_size_cb(char *HTTP_response, const void *callback_data return ret_value; } /* end RV_parse_domain_allocated_size_cb */ + +/*------------------------------------------------------------------------- + * Function: RV_now_usec + * + * Purpose: Return the current time as microseconds after the UNIX epoch. + * + * Return: Positive on success, negative on failure. + * + * Programmer: Matthew Larson + * April, 2024 + */ +uint64_t +RV_now_usec(void) +{ + + uint64_t ret_value; + +#ifdef RV_HAVE_CLOCKGETTIME + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + + /* Cast all values in this expression to uint64_t to ensure that all intermediate + * calculations are done in 64 bit, to prevent overflow */ + ret_value = + ((uint64_t)ts.tv_sec * ((uint64_t)1000 * (uint64_t)1000)) + ((uint64_t)ts.tv_nsec / (uint64_t)1000); +#elif RV_HAVE_GETTIMEOFDAY + struct timeval now_tv; + + gettimeofday(&now_tv, NULL); + + /* Cast all values in this expression to uint64_t to ensure that all intermediate + * calculations are done in 64 bit, to prevent overflow */ + ret_value = ((uint64_t)now_tv.tv_sec * ((uint64_t)1000 * (uint64_t)1000)) + (uint64_t)now_tv.tv_usec; +#else + /* Cast all values in this expression to uint64_t to ensure that all intermediate calculations + * are done in 64 bit, to prevent overflow */ + ret_value = ((uint64_t)time(NULL) * ((uint64_t)1000 * (uint64_t)1000)); +#endif + +done: + return (ret_value); +} + +/*------------------------------------------------------------------------- + * Function: RV_strtok_r + * + * Purpose: Helper function to use strtok_r or strtok_r depending on current platform + * + * Return: See strtok_r documentation + * + * Programmer: Matthew Larson + * May, 2024 + */ +char * +RV_strtok_r(char *str, const char *delim, char **saveptr) +{ +#ifdef RV_HAVE_STRTOK_R + return strtok_r(str, delim, saveptr); +#elif RV_HAVE_STRTOK_S + return strtok_s(str, delim, saveptr); +#else + return NULL; +#endif +} \ No newline at end of file diff --git a/src/rest_vol.h b/src/rest_vol.h index 1597fa8a..35763f83 100644 --- a/src/rest_vol.h +++ b/src/rest_vol.h @@ -813,6 +813,12 @@ herr_t RV_get_cmpd_subset_type(hid_t src_type_id, hid_t dst_type_id, RV_subset_t /* Helper to get information about members in dst that are included in src compound */ herr_t RV_get_cmpd_subset_nmembers(hid_t src_type_id, hid_t dst_type_id, size_t *num_cmpd_members); +/* Return the current time as microseconds after the UNIX epoch. */ +uint64_t RV_now_usec(void); + +/* System-indepedent wrapper for strtok_r/strtok_s */ +char *RV_strtok_r(char *str, const char *delim, char **saveptr); + #define SERVER_VERSION_MATCHES_OR_EXCEEDS(version, major_needed, minor_needed, patch_needed) \ (version.major > major_needed) || (version.major == major_needed && version.minor > minor_needed) || \ (version.major == major_needed && version.minor == minor_needed && version.patch >= patch_needed) diff --git a/src/rest_vol_config.h.in b/src/rest_vol_config.h.in index 5e0997d4..98b2405c 100644 --- a/src/rest_vol_config.h.in +++ b/src/rest_vol_config.h.in @@ -30,9 +30,6 @@ /* Define to 1 if you have the header file. */ #undef HAVE_MACH_MACH_TIME_H -/* Define to 1 if you have the header file. */ -#undef HAVE_MEMORY_H - /* Define to 1 if you have the header file. */ #undef HAVE_SETJMP_H @@ -48,6 +45,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_STDINT_H +/* Define to 1 if you have the header file. */ +#undef HAVE_STDIO_H + /* Define to 1 if you have the header file. */ #undef HAVE_STDLIB_H @@ -57,6 +57,12 @@ /* Define to 1 if you have the header file. */ #undef HAVE_STRING_H +/* Define to 1 if you have the `strtok_r' function. */ +#undef HAVE_STRTOK_R + +/* Define to 1 if you have the `strtok_s' function. */ +#undef HAVE_STRTOK_S + /* Define to 1 if you have the header file. */ #undef HAVE_SYS_FILE_H @@ -111,10 +117,13 @@ /* The size of `off_t', as computed by sizeof. */ #undef SIZEOF_OFF_T -/* Define to 1 if you have the ANSI C header files. */ +/* Define to 1 if all of the C90 standard headers exist (not just the ones + required in a freestanding environment). This macro is provided for + backward compatibility; new code need not use it. */ #undef STDC_HEADERS -/* Define to 1 if you can safely include both and . */ +/* Define to 1 if you can safely include both and . This + macro is obsolete. */ #undef TIME_WITH_SYS_TIME /* Define to have the REST VOL track memory usage. */ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 88df5f75..473c53aa 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required (VERSION 3.1.0) +cmake_minimum_required (VERSION 3.12) PROJECT (HDF5_VOL_REST_TEST C) #-----------------------------------------------------------------------------