SNOW-1524259: Add option to build libsnowflakeclient as dynamic library

.github/workflows/build-test.yml

@@ -24,6 +24,7 @@ jobs:
             matrix:
                 build_type: [ 'Debug', 'Release' ]
                 cloud_provider: [ 'AWS', 'AZURE', 'GCP' ]
+                link_type: [ 'Dynamic', 'Static' ]
         steps:
             - uses: actions/checkout@v4
             - name: Restore cached deps
@@ -37,6 +38,7 @@ jobs:
               shell: bash
               env:
                   BUILD_TYPE: ${{ matrix.build_type }}
+                  LINK_TYPE: ${{ matrix.link_type }}
               run: ci/build_linux.sh
             - name: Cache deps
               id: cache-save-deps
@@ -54,6 +56,7 @@ jobs:
               env:
                   BUILD_TYPE: ${{ matrix.build_type }}
                   CLOUD_PROVIDER: ${{ matrix.cloud_provider }}
+                  LINK_TYPE: ${{ matrix.link_type }}
                   PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
               run: ci/test_linux.sh
 
@@ -67,6 +70,7 @@ jobs:
                 build_type: [ 'Debug', 'Release' ]
                 vs_version: [ 'VS16' ]
                 cloud_provider: [ 'AWS', 'AZURE', 'GCP' ]
+                link_type: [ 'Dynamic', 'Static' ]
         steps:
             - uses: actions/checkout@v4
             - name: Install CMake
@@ -89,6 +93,7 @@ jobs:
                   PLATFORM: ${{ matrix.platform }}
                   BUILD_TYPE: ${{ matrix.build_type }}
                   VS_VERSION: ${{ matrix.vs_version }}
+                  LINK_TYPE: ${{ matrix.link_type }}
               run: ci\build_win.bat
             - name: Cache deps
               id: cache-save-deps
@@ -107,6 +112,7 @@ jobs:
                   PLATFORM: ${{ matrix.platform }}
                   BUILD_TYPE: ${{ matrix.build_type }}
                   VS_VERSION: ${{ matrix.vs_version }}
+                  LINK_TYPE: ${{ matrix.link_type }}
                   CLOUD_PROVIDER: ${{ matrix.cloud_provider }}
                   PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
               run: ci\test_win.bat
@@ -120,6 +126,7 @@ jobs:
                 build_type: [ 'Debug', 'Release' ]
                 vs_version: [ 'VS17' ]
                 cloud_provider: [ 'AWS', 'AZURE', 'GCP' ]
+                link_type: [ 'Dynamic', 'Static' ]
         steps:
             - uses: actions/checkout@v4
             - name: Install CMake
@@ -142,6 +149,7 @@ jobs:
                   PLATFORM: ${{ matrix.platform }}
                   BUILD_TYPE: ${{ matrix.build_type }}
                   VS_VERSION: ${{ matrix.vs_version }}
+                  LINK_TYPE: ${{ matrix.link_type }}
               run: ci\build_win.bat
             - name: Cache deps
               id: cache-save-deps
@@ -160,6 +168,7 @@ jobs:
                   PLATFORM: ${{ matrix.platform }}
                   BUILD_TYPE: ${{ matrix.build_type }}
                   VS_VERSION: ${{ matrix.vs_version }}
+                  LINK_TYPE: ${{ matrix.link_type }}
                   CLOUD_PROVIDER: ${{ matrix.cloud_provider }}
                   PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
               run: ci\test_win.bat
@@ -171,6 +180,7 @@ jobs:
             matrix:
                 build_type: [ 'Debug', 'Release' ]
                 cloud_provider: [ 'AWS', 'AZURE', 'GCP' ]
+                link_type: [ 'Dynamic', 'Static' ]
         steps:
             - uses: actions/checkout@v4
             - name: Install Homebrew Bash
@@ -187,6 +197,7 @@ jobs:
               shell: bash
               env:
                 BUILD_TYPE: ${{ matrix.build_type }}
+                LINK_TYPE: ${{ matrix.link_type }}
               run: ./ci/build_mac.sh
             - name: Cache deps
               id: cache-save-deps
@@ -200,6 +211,7 @@ jobs:
               env:
                   BUILD_TYPE: ${{ matrix.build_type }}
                   CLOUD_PROVIDER: ${{ matrix.cloud_provider }}
+                  LINK_TYPE: ${{ matrix.link_type }}
                   PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
               run: ci/test_mac.sh
     build-test-codecov-aws:

CMakeLists.txt

@@ -11,7 +11,7 @@ include(cmake/flags.cmake)
 # we don't need Dart and other tools.
 enable_testing()
 
-add_compile_definitions(LOG_USE_COLOR)
+add_compile_definitions(BUILDING_LIBSFC LOG_USE_COLOR)
 
 option(BUILD_TESTS "True if build tests" on)
 option(MOCK "True if mock should be used" off)
@@ -430,18 +430,30 @@ if (MOCK)
     message("libcmocka is located at " ${CMOCKA_LIB})
 endif ()
 
-add_library(snowflakeclient STATIC ${SOURCE_FILES} ${SOURCE_FILES_PUT_GET} ${SOURCE_FILES_CPP_WRAPPER})
-target_compile_features(snowflakeclient PUBLIC cxx_std_17)
-target_compile_features(snowflakeclient PUBLIC c_std_99)
-if (UNIX)
-    target_compile_definitions(snowflakeclient PUBLIC _LARGEFILE64_SOURCE)
-endif ()
+if (NOT BUILD_SHARED_LIBS)
+    add_compile_definitions(LIBSFC_STATICLIB)
+    add_library(snowflakeclient STATIC ${SOURCE_FILES} ${SOURCE_FILES_PUT_GET} ${SOURCE_FILES_CPP_WRAPPER})
+    target_compile_features(snowflakeclient PUBLIC cxx_std_17)
+    target_compile_features(snowflakeclient PUBLIC c_std_99)
+    if (UNIX)
+        target_compile_definitions(snowflakeclient PUBLIC _LARGEFILE64_SOURCE)
+    endif ()
 
-if (LINUX)
-    target_compile_options(snowflakeclient PUBLIC -pthread)
-    target_link_options(snowflakeclient PUBLIC -pthread)
-endif ()
-#set (CMAKE_CXX_STANDARD 11)
+    if (LINUX)
+        target_compile_options(snowflakeclient PUBLIC -pthread)
+        target_link_options(snowflakeclient PUBLIC -pthread)
+    endif ()
+    #set (CMAKE_CXX_STANDARD 11)
+else()
+    if (WIN32)
+        add_library(snowflakeclient ${SOURCE_FILES} ${SOURCE_FILES_PUT_GET} ${SOURCE_FILES_CPP_WRAPPER})
+    else()
+        add_library(snowflakeclient ${SOURCE_FILES} ${SOURCE_FILES_PUT_GET} ${SOURCE_FILES_CPP_WRAPPER})
+        target_compile_definitions(snowflakeclient PUBLIC _LARGEFILE64_SOURCE)
+    endif()
+    target_compile_features(snowflakeclient PUBLIC cxx_std_17)
+    target_compile_features(snowflakeclient PUBLIC c_std_99)
+endif()
 
 if(LINUX)
     link_directories(
@@ -491,15 +503,93 @@ elseif()
     endif()
 endif()
 
+set(AWS_ALL_LIBS
+        ${AWS_S3_LIB}
+        ${AWS_CORE_LIB}
+        ${AWS_CRT_CPP_LIB}
+        ${AWS_C_EVENT_STREAM_LIB}
+        ${AWS_C_CHECKSUMS_LIB}
+        ${AWS_C_S3_LIB}
+        ${AWS_C_AUTH_LIB}
+        ${AWS_C_HTTP_LIB}
+        ${AWS_C_COMPRESSION_LIB}
+        ${AWS_C_IO_LIB}
+        ${AWS_C_CAL_LIB}
+        ${AWS_C_MQTT_LIB}
+        ${AWS_S2N_LIB}
+        ${AWS_C_COMMON_LIB}
+        ${AWS_C_SDKUTILS_LIB}
+    )
+
 if (LINUX)
-    # Linux
-    target_link_libraries(snowflakeclient rt dl z)
-endif ()
-if (APPLE)
-    # OSX. no librt is required.
-    target_link_libraries(snowflakeclient dl z)
+    set(ARROW_ALL_LIBS
+        ${ARROW_ARROW_LIB}
+        ${BOOST_FILESYSTEM_LIB} ${BOOST_REGEX_LIB} ${BOOST_SYSTEM_LIB}
+        ${ARROW_BROTLIDEC_LIB} ${ARROW_BROTLIENC_LIB} ${ARROW_BROTLICOMMON_LIB}
+        ${ARROW_DOUBLECONVERSION_LIB}
+        ${ARROW_FLATBUFFERS_LIB}
+        ${ARROW_GFLAGS_LIB}
+        ${ARROW_GLOG_LIB}
+        ${ARROW_JEMALLOCPIC_LIB}
+        ${ARROW_LZ4_LIB} ${ARROW_LZO2_LIB}
+        ${ARROW_SNAPPY_LIB}
+        ${ARROW_ZSTD_LIB})
+elseif (APPLE)
+    set(ARROW_ALL_LIBS
+        ${ARROW_ARROW_LIB}
+        ${BOOST_FILESYSTEM_LIB} ${BOOST_REGEX_LIB} ${BOOST_SYSTEM_LIB}
+        ${ARROW_BROTLICOMMON_LIB} ${ARROW_BROTLIDEC_LIB} ${ARROW_BROTLIENC_LIB}
+        ${ARROW_FLATBUFFERS_LIB}
+        ${ARROW_GFLAGS_LIB}
+        ${ARROW_GLOG_LIB}
+        ${ARROW_JEMALLOCPIC_LIB}
+        ${ARROW_LZ4_LIB} ${ARROW_LZO2_LIB}
+        ${ARROW_SNAPPY_LIB}
+        ${ARROW_ZSTD_LIB})
+elseif (WIN32)
+    set(ARROW_ALL_LIBS
+        ${ARROW_ARROW_LIB}
+        ${BOOST_FILESYSTEM_LIB} ${BOOST_REGEX_LIB} ${BOOST_SYSTEM_LIB}
+        ${ARROW_BROTLICOMMON_LIB} ${ARROW_BROTLIDEC_LIB} ${ARROW_BROTLIENC_LIB}
+        ${ARROW_BZ2_LIB}
+        ${ARROW_DOUBLECONVERSION_LIB}
+        ${ARROW_EVENT_LIB} ${ARROW_EVENTCORE_LIB} ${ARROW_EVENTEXTRA_LIB}
+        ${ARROW_FLATBUFFERS_LIB}
+        ${ARROW_GFLAGS_LIB}
+        ${ARROW_GLOG_LIB}
+        ${ARROW_LZ4_LIB}
+        ${ARROW_SNAPPY_LIB}
+        ${ARROW_ZLIB_LIB} ${ARROW_ZSTD_LIB})
 endif ()
 
+if (NOT BUILD_SHARED_LIBS)
+    if (LINUX)
+        # Linux
+        target_link_libraries(snowflakeclient rt dl z)
+    endif ()
+    if (APPLE)
+        # OSX. no librt is required.
+        target_link_libraries(snowflakeclient dl z)
+    endif ()
+else()
+    if (LINUX)
+        set(DYNAMICLIB_LINK_OPTS  rt dl z ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${AZURE_STORAGE_LITE_LIB} ${AWS_ALL_LIBS} -Wl,--whole-archive ${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${UUID_LIB}
+                pthread -Wl,--no-whole-archive -Wl,--as-needed -static-libgcc -static-libstdc++)
+    elseif (APPLE)
+        set(DYNAMICLIB_LINK_OPTS  dl z pthread "-framework CoreFoundation -framework SystemConfiguration -framework CoreServices -framework security"
+                ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${AZURE_STORAGE_LITE_LIB} ${AWS_ALL_LIBS} ${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${UUID_LIB})
+    elseif (WIN32)
+        if (WIN32_DEBUG)
+            set(DYNAMICLIB_LINK_OPTS ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${AWS_ALL_LIBS} ${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB} ${AZURE_STORAGE_LITE_LIB}
+                    Version.lib Userenv.lib Bcrypt.lib ucrtd.lib Secur32.lib Ncrypt.lib Shlwapi.lib)
+        else()
+            set(DYNAMICLIB_LINK_OPTS ${CMOCKA_LIB} ${AWS_ALL_LIBS} ${AZURE_STORAGE_LITE_LIB} ${ARROW_ALL_LIBS} ${CURL_LIB} ${OOB_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB}
+                    Version.lib Userenv.lib Bcrypt.lib Secur32.lib Ncrypt.lib Shlwapi.lib)
+        endif()
+    endif()
+    target_link_libraries(snowflakeclient ${DYNAMICLIB_LINK_OPTS})
+endif()
+
 add_subdirectory(examples)
 
 if (BUILD_TESTS)

README.rst

@@ -52,13 +52,14 @@ Linux and OSX
 
 Windows
 ^^^^^^^^^^
-Set environment variables: PLATFORM: [x64, x86], BUILD_TYPE: [Debug, Release], VS_VERSION: [VS15, VS16, VS17] and run the script.
+Set environment variables: PLATFORM: [x64, x86], BUILD_TYPE: [Debug, Release], VS_VERSION: [VS15, VS16, VS17], LINK_TYPE: [Dynamic, Static] and run the script.
 
 .. code-block:: bash
 
     set platform=x64
     set build_type=Release
     set vs_version=VS17
+    set build_type=Static
 
     .\scripts\build_dependencies.bat
     .\scripts\build_libsnowflakeclient.bat
@@ -122,13 +123,14 @@ Linux and OSX
 Windows
 ^^^^^^^^^^
 
-Set environment variables: PLATFORM: [x64, x86], BUILD_TYPE: [Debug, Release], VS_VERSION: [VS15, VS16, VS17] and run the script.
+Set environment variables: PLATFORM: [x64, x86], BUILD_TYPE: [Debug, Release], VS_VERSION: [VS15, VS16, VS17], LINK_TYPE: [Dynamic, Static] and run the script.
 
 .. code-block:: bash
 
     set platform=x64
     set build_type=Release
     set vs_version=VS17
+    set build_type=Static
 
    .\scripts\run_tests.bat
 

ci/build/build.sh

@@ -9,6 +9,7 @@ CI_BUILD_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 SCRIPTS_DIR=$( cd "$CI_BUILD_DIR/../../scripts" && pwd )
 
 [[ -z "$BUILD_TYPE" ]] && echo "Set BUILD_TYPE: [Debug, Release]" && exit 1
+[[ -z "$LINK_TYPE" ]] && export LINK_TYPE=Static
 
 if [[ -z "$UPLOAD_TO_S3" ]]; then
     if [[ -z "$GITHUB_ACTIONS" ]] && [[ -n "$GIT_BRANCH" ]]; then
@@ -19,7 +20,7 @@ if [[ -z "$UPLOAD_TO_S3" ]]; then
 fi
 echo "UPLOAD_TO_S3=${UPLOAD_TO_S3}"
 
-source $SCRIPTS_DIR/_init.sh -t $BUILD_TYPE "$@"
+source $SCRIPTS_DIR/_init.sh -t $BUILD_TYPE -l $LINK_TYPE "$@"
 source $SCRIPTS_DIR/utils.sh
 init_git_variables
 
@@ -77,10 +78,11 @@ function build_component()
     local component_name=$1
     local component_script=$2
     local build_type=$3
-    local other_args="$4"
+    local link_type=$4
+    local other_args="$5"
 
     echo "=== build: $component_name ==="
-    "$component_script" -t "$build_type" "$other_args"
+    "$component_script" -t "$build_type" -l "$link_type" "$other_args"
     local component_version=$("$component_script" -v)
 
     if [[ "$UPLOAD_TO_S3" == "true" ]]; then
@@ -110,6 +112,6 @@ if [[ -n "$GITHUB_ACTIONS" ]]; then
     rm -rf $SCRIPTS_DIR/../deps/*
 fi
 
-build_component libsnowflakeclient "$SCRIPTS_DIR/build_libsnowflakeclient.sh" "$target" "$@"
+build_component libsnowflakeclient "$SCRIPTS_DIR/build_libsnowflakeclient.sh" "$target" "$linking" "$@"
 
 

ci/build_linux.sh

@@ -45,6 +45,7 @@ docker run \
         -e GITHUB_REF \
         -e CLIENT_CODE_COVERAGE \
         -e USE_EXTRA_WARNINGS \
+        -e LINK_TYPE \
         -w /mnt/host \
         "${BUILD_IMAGE_NAME}" \
         "/mnt/host/ci/build/build.sh"

ci/build_win.bat

@@ -43,6 +43,7 @@ goto :EOF
     set build_type=%~2
     set vs_version=%~3
     set dynamic_runtime=%~4
+
     call "%scriptdir%..\scripts\_init.bat" %platform% %build_type% %vs_version%
     if %ERRORLEVEL% NEQ 0 goto :error
     call :download_build_component zlib "%zlib_build_script%" "%dynamic_runtime%"
@@ -68,6 +69,7 @@ goto :EOF
     if defined GITHUB_ACTIONS (
         rd /S /Q %scriptdir%\..\deps
     )
+
     call :build_component libsnowflakeclient "%libsnowflakeclient_build_script%" "%dynamic_runtime%"
     if %ERRORLEVEL% NEQ 0 goto :error
     if defined GITHUB_ACTIONS (
@@ -140,7 +142,7 @@ goto :EOF
     set dynamic_runtime=%~3
 
     echo === build: %component_name% ===
-    call %build_script% :build %platform% %build_type% %vs_version% %dynamic_runtime% ON
+    call %build_script% :build %platform% %build_type% %vs_version% %dynamic_runtime% ON %link_type%
     if %ERRORLEVEL% NEQ 0 goto :error
 
     call %build_script% :get_version

ci/test/test.sh

@@ -7,8 +7,9 @@ SCRIPTS_DIR=$( cd "$CI_TEST_DIR/../../scripts" && pwd )
 set -o pipefail
 
 [[ -z "$BUILD_TYPE" ]] && echo "Set BUILD_TYPE: [Debug, Release]" && exit 1
+[[ -z "$LINK_TYPE" ]] && export LINK_TYPE=Static
 
-source $SCRIPTS_DIR/_init.sh -t $BUILD_TYPE
+source $SCRIPTS_DIR/_init.sh -t $BUILD_TYPE -l $LINK_TYPE
 echo "CMAKE: $CMAKE, CTEST: $CTEST"
 source $SCRIPTS_DIR/utils.sh
 
@@ -33,6 +34,7 @@ function test_component()
     local build_type=$3
 
     echo "build_type: $build_type"
+    echo "link_type: $LINK_TYPE"
     local component_version=$("$component_script" -v)
     local cmake_file_name=$(get_cmake_file_name $component_name $component_version $build_type)
     local cmake_dir=cmake-build-$build_type

ci/test_linux.sh

@@ -58,6 +58,7 @@ docker run \
         -e GITHUB_SHA \
         -e RUNNER_TRACKING_ID \
         -e CLIENT_CODE_COVERAGE \
+        -e LINK_TYPE \
         -w /mnt/host \
         "${TEST_IMAGE_NAME}" \
         "/mnt/host/ci/test/test.sh"