Add Surrogates and Error Bound Autotuning Implementation

CMakeLists.txt

@@ -1,19 +1,65 @@
+project(H5Z_ROCCI CXX C)
+
 find_package(HDF5 REQUIRED)
 find_package(std_compat REQUIRED)
 find_package(LibPressio REQUIRED)
+find_package(LibPressioOpt REQUIRED)
+find_package(LibDistributed REQUIRED)
+find_library(UUID_LIBRARY NAMES uuid libuuid REQUIRED)
+
+option(USE_SZX "Build SECRE SZx Surrogate" ON)
+option(USE_SZ3 "Build SECRE SZ3 Surrogate" ON)
+option(USE_ZFP "Build SECRE ZFP Surrogate" ON)
+
+if(USE_SZX)
+  list(APPEND secre_src 
+    src/surrogates/SZx/estimate_metric.cc
+    src/surrogates/SZx/sampling.cc
+    src/surrogates/SZx/szx_float.cc
+    src/surrogates/SZx/SZxSurrogate.cc
+  )
+endif()
 
-# include(CMakePrintHelpers)
-# message(HDF5 DIRS:)
-# cmake_print_variables(HDF5_LIBRARIES)
-# cmake_print_variables(HDF5_INCLUDE_DIRS)
+if(USE_SZ3)
+  list(APPEND secre_src 
+    src/surrogates/SZ3/estimate_cr.cc
+    src/surrogates/SZ3/estimate_psnr.cc
+    src/surrogates/SZ3/estimate_ssim.cc
+    src/surrogates/SZ3/CustomHuffmanEncoder.cc
+    src/surrogates/SZ3/SZ3Surrogate.cc
+  )
+endif()
+
+if(USE_ZFP)
+  list(APPEND secre_src 
+    src/surrogates/ZFP/estimate_metric.cc
+    src/surrogates/ZFP/ZFPSurrogate.cc
+  )
+endif()
+
+add_library(SECRE SHARED
+  src/surrogates/qcat_ssim.cc
+  ${secre_src}
+)
+target_include_directories(
+  SECRE
+  PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include/surrogates>
+  $<INSTALL_INTERFACE:SECRE>
+)
+target_link_libraries(
+  SECRE
+  PUBLIC LibPressio::libpressio
+)
 
 add_library(ROCCI SHARED
-  src/rocci.c
-  src/rocci_ByteToolkit.c
-  src/rocci_FileUtil.c
-  src/rocci_utils.c
+  src/rocci.cc
+  src/rocci_ByteToolkit.cc
+  src/rocci_FileUtil.cc
+  src/rocci_utils.cc
   src/inih/ini.c
 )
+set_target_properties(ROCCI PROPERTIES LINKER_LANGUAGE CXX)
 target_include_directories(
   ROCCI
   PUBLIC
@@ -23,12 +69,12 @@ target_include_directories(
 
 target_link_libraries(
   ROCCI
-  PUBLIC LibPressio::libpressio
+  PUBLIC LibPressio::libpressio LibPressioOpt::libpressio_opt LibDistributed::libdistributed ${UUID_LIBRARY} SECRE
 )
 
 add_library(
   hdf5rocci SHARED
-  src/H5Z_ROCCI.c
+  src/H5Z_ROCCI.cc
   )
 target_link_libraries(
   hdf5rocci
@@ -52,4 +98,4 @@ install(TARGETS hdf5rocci EXPORT HDF5ROCCI
   RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
   )
 install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hdf5_rocci)
-export(TARGETS hdf5rocci ROCCI FILE HDF5ROCCI.cmake)
+export(TARGETS hdf5rocci ROCCI SECRE FILE HDF5ROCCI.cmake)

include/H5Z_ROCCI.h

@@ -43,7 +43,7 @@ do                                                    \
 	H5Epush(H5E_DEFAULT,__FILE__,_funcname_,__LINE__, \
 		H5Z_ROCCI_ERRCLASS,MAJ,MIN,MSG);                \
 	retval = RET;                                     \
-	goto done;                                        \
+	return retval;                                        \
 } while(0)
 
 

include/rocci.h

@@ -45,11 +45,25 @@ extern "C" {
 #define ROCCI_SSIM_METRIC 2
 #define ROCCI_AC_METRIC 3
 
+extern const int versionNumber[4];
+
+extern const char* COMPRESSOR_STR[7];
+extern const int COMPRESSOR_OPTIONS[7];
+
+extern const char* CMP_MODE_STR[8];
+extern const int CMP_MODE_OPTIONS[8];
+
+extern const char* METRIC_STR[3];
+extern const int METRIC_OPTIONS[3];
+
+
+
 typedef struct ROCCI_Target
 {
 	int metric; //specify the metric
-	double targetValue_lowbound;
-	double targetValue_upperbound;
+	bool do_opt;
+	double targetValue; // target value for metric
+	double tolerance; // tolerance for early stopping optimization
 } ROCCI_Target;
 
 typedef struct ROCCI_Setting
@@ -65,7 +79,7 @@ typedef struct ROCCI_Setting
 void ROCCI_Init(char* cfgFile);
 int rocciFidelity_multiFields(float** oriData, float** decData, float** fidelity, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
 int rocciFidelity_singleField(float* oriData, float* decData, float** fidelity, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
-int roccifastSearchBestSetting (int metric, int compressorID, ROCCI_Target input, ROCCI_Setting* result);
+int roccifastSearchBestSetting (struct pressio_data* input_data, ROCCI_Target input, ROCCI_Setting* result);
 //int constuctRuntimeCompressor (ROCCI_Setting* input , pressio_compressor* output);
 
 unsigned char* ROCCI_compress_args(int dataType, void* data, size_t* outSize, int error_mode, double abs_error, double rel_error, double pw_rel_error, double psnr, double ratio, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);

include/rocci_ByteToolkit.h

@@ -10,9 +10,6 @@
 #ifndef _rocci_ByteToolkit_H
 #define _rocci_ByteToolkit_H
 
-#ifdef __cplusplus
-extern "C" {
-#endif
 
 #include <stdio.h>
 #include <stdint.h>
@@ -21,8 +18,8 @@ extern "C" {
 
 //ByteToolkit.c
 
-int sysEndianType;
-int dataEndianType;
+static int sysEndianType;
+static int dataEndianType;
 
 typedef union lint16 {
     unsigned short usvalue;
@@ -63,6 +60,9 @@ void symTransform_8bytes(unsigned char data[8]);
 void setSysEndianType(int endianType);
 void setDataEndianType(int endianType);
 
+int getSysEndianType();
+int getDataEndianType();
+
 extern unsigned short bytesToUInt16_bigEndian(unsigned char* bytes);
 extern unsigned int bytesToUInt32_bigEndian(unsigned char* bytes);
 extern uint64_t bytesToUInt64_bigEndian(unsigned char* b);
@@ -115,9 +115,6 @@ void convertULongArrayToBytes(uint64_t* states, size_t stateLength, unsigned cha
 extern size_t bytesToSize(unsigned char* bytes);
 extern void sizeToBytes(unsigned char* outBytes, size_t size);
 
-#ifdef __cplusplus
-}
-#endif
 
 #endif /* ----- #ifndef _ByteToolkit_H  ----- */
 

include/rocci_FileUtil.h

@@ -6,6 +6,13 @@
 #include <stdint.h>
 #include <assert.h>
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 void readfile(const char *file, const size_t num, size_t elem_size, void *data);
 
+#ifdef __cplusplus
+}
+#endif
 #endif

include/rocci_utils.h

@@ -3,8 +3,12 @@
 
 size_t computeNbEle(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
 
+char* append_string(const char* original, const char* append);
+
 enum pressio_dtype rocci_dtype_to_pressio_dtype(int dataType);
 
+char* get_surrogate(ROCCI_Setting rocci_config);
+
 struct pressio_compressor* get_compressor(ROCCI_Setting rocci_config, struct pressio* library);
 
 struct pressio_options* get_compressor_options(ROCCI_Setting rocci_config);

include/surrogates/SZ3/CustomHuffmanEncoder.hpp

@@ -0,0 +1,81 @@
+#ifndef _HUFFMAN_ENCODER_HPP
+#define _HUFFMAN_ENCODER_HPP
+
+#include <vector>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <cstdio>
+#include "SZ3/def.hpp"
+#include "SZ3/encoder/Encoder.hpp"
+
+namespace SZ3 {
+
+template<class T>
+class CustomHuffmanEncoder : public concepts::EncoderInterface<T> {
+public:
+
+    typedef struct node_t {
+        struct node_t *left, *right;
+        size_t freq;
+        char t; //in_node:0; otherwise:1
+        T c;
+    } *node;
+
+    typedef struct HuffmanTree {
+        unsigned int stateNum;
+        unsigned int allNodes;
+        struct node_t *pool;
+        node *qqq, *qq; //the root node of the HuffmanTree is qq[1]
+        int n_nodes; //n_nodes is for compression
+        int qend;
+        uint64_t **code;
+        unsigned char *cout;
+        int n_inode; //n_inode is for decompression
+        int maxBitCount;
+    } HuffmanTree;
+
+    CustomHuffmanEncoder();
+    ~CustomHuffmanEncoder();
+
+    void preprocess_encode(const std::vector<T> &bins, int stateNum);
+    void preprocess_encode(const T *bins, size_t num_bin, int stateNum);
+    void save(unsigned char *&c);
+    size_t size_est();
+    size_t encode(const std::vector<T> &bins, unsigned char *&bytes);
+    size_t encode(const T *bins, size_t num_bin, unsigned char *&bytes);
+    void postprocess_encode();
+    void preprocess_decode();
+    std::vector<T> decode(const unsigned char *&bytes, size_t targetLength);
+    void postprocess_decode();
+    void load(const unsigned char *&c, size_t &remaining_length);
+    bool isLoaded();
+    unsigned char get_cout_for_state(T bin_index);
+
+private:
+
+    HuffmanTree *huffmanTree = NULL;
+    node_t *treeRoot;
+    unsigned int nodeCount;
+    unsigned char sysEndianType;
+    bool loaded;
+    T offset;
+
+    // Private method declarations
+    HuffmanTree *createHuffmanTree(int stateNum);
+    node_t *reconstruct_HuffTree_from_bytes_anyStates(const unsigned char *bytes, uint32_t nodeCount);
+    node_t *new_node(size_t freq, T c, node_t *a, node_t *b);
+    node_t *new_node2(T c, unsigned char t);
+    void qinsert(node_t *n);
+    node_t *qremove();
+    void build_code(node_t *n, int len, uint64_t out1, uint64_t out2);
+    void init(const T *s, size_t length);
+    template<class T1> void pad_tree(T1 *L, T1 *R, T *C, unsigned char *t, unsigned int i, node_t *root);
+    template<class T1> void unpad_tree(T1 *L, T1 *R, T *C, unsigned char *t, unsigned int i, node_t *root);
+    template<class T1> unsigned int convert_HuffTree_to_bytes_anyStates(unsigned int nodeCount, unsigned char *out);
+    void SZ_FreeHuffman();
+};
+
+} // namespace SZ3
+
+#endif

include/surrogates/SZ3/estimate_metric.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#include "SZ3/utils/Config.hpp"
+#include <stdexcept>
+
+double estimate_cr_float(SZ3::Config conf, float *data, double abs, int stride, uint N);
+
+double estimate_psnr_float(SZ3::Config conf, float *data, double abs, int stride, uint N);
+
+double estimate_ssim_float(SZ3::Config conf, float *data, double abs, int stride, int blocksize, uint N);