Self-contained dependencies (#84)

This improves reliability of CI and builds in general.

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "lib/googletest"]
-	path = lib/googletest
-	url = https://github.com/google/googletest.git

CMakeLists.txt

@@ -5,6 +5,10 @@ project(TrustWalletCore)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
+
+include_directories(${CMAKE_SOURCE_DIR}/build/local/include)
+link_directories(${CMAKE_SOURCE_DIR}/build/local/lib)
+
 add_subdirectory(trezor-crypto)
 
 macro(find_host_package)
@@ -19,29 +23,9 @@ endmacro(find_host_package)
 
 find_host_package(Boost REQUIRED)
 
-# Submodules
-find_package(Git QUIET)
-if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
-    option(GIT_SUBMODULE "Check submodules during build" ON)
-    if(GIT_SUBMODULE)
-        message(STATUS "Submodule update")
-        execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init
-                        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-                        RESULT_VARIABLE GIT_SUBMOD_RESULT)
-        if(NOT GIT_SUBMOD_RESULT EQUAL "0")
-            message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules")
-        endif()
-    endif()
-endif()
-
-if(NOT EXISTS "${PROJECT_SOURCE_DIR}/lib/googletest/CMakeLists.txt")
-    message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
-endif()
-
 include(ExternalProject)
 
 # Dependencies
-include(cmake/JSON.cmake)
 include(cmake/Protobuf.cmake)
 
 # Source files
@@ -59,7 +43,6 @@ else()
 
     target_link_libraries(TrustWalletCore PRIVATE TrezorCrypto protobuf Boost::boost)
 endif()
-add_dependencies(TrustWalletCore nlohmann_json)
 set_property(TARGET TrustWalletCore PROPERTY CXX_STANDARD 17)
 
 # Define headers for this library. PUBLIC headers are used for compiling the
@@ -71,7 +54,7 @@ target_include_directories(TrustWalletCore
     PRIVATE
         ${CMAKE_CURRENT_SOURCE_DIR}/src
         ${CMAKE_CURRENT_SOURCE_DIR}/jni/cpp
-        ${json_INCLUDE_DIRS}
+        ${CMAKE_CURRENT_SOURCE_DIR}/build/local/include
 )
 
 if(NOT ANDROID AND NOT IOS_PLATFORM)

Dockerfile

@@ -45,19 +45,6 @@ RUN cd /usr/local/src \
     && rm -rf cmake*
 RUN cmake --version
 
-# Install Protobuf
-ENV PROTOBUF_VERSION=$PROTOBUF_VERSION
-RUN curl -fSsOL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-java-$PROTOBUF_VERSION.tar.gz \
-    && tar xzf protobuf-java-$PROTOBUF_VERSION.tar.gz \
-    && cd protobuf-$PROTOBUF_VERSION \
-    && ./configure \
-    && make \
-    && make install \
-    && ldconfig \
-    && cd .. \
-    && rm -rf protobuf*
-RUN protoc --version
-
 # Clean Up
 RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
@@ -66,7 +53,6 @@ ENV CC=/usr/bin/clang
 ENV CXX=/usr/bin/clang++
 RUN git clone https://github.com/TrustWallet/wallet-core.git \
     && cd wallet-core \
-    && git submodule update --init \
-    && cmake -H. -Bbuild -DCMAKE_BUILD_TYPE=Debug -DGIT_SUBMODULE=OFF
+    && tools/install-dependencies
 
 CMD ["/bin/bash"]

TrustWalletCore.podspec

@@ -18,8 +18,8 @@ Pod::Spec.new do |s|
     submodules: true
   }
 
-  protobuf_dir = 'build/protobuf/src/protobuf_ext'
-  json_dir = 'build/nlohmann/src/nlohmann_json'
+  protobuf_dir = 'build/protobuf/staging/protobuf-3.7.0'
+  json_dir = 'build/local/include/nlohmann'
   s.source_files =
     'src/**/*.{c,cc,cpp,h}',
     'include/**/*.h',
@@ -158,7 +158,7 @@ Pod::Spec.new do |s|
   s.pod_target_xcconfig = {
     'SYSTEM_HEADER_SEARCH_PATHS' => '$(inherited) /usr/local/include'
   }
-  s.prepare_command = 'tools/generate-files && cmake -H. -Bbuild && make -Cbuild protobuf_ext nlohmann_json'
+  s.prepare_command = 'tools/install-dependencies && tools/generate-files'
 
   s.dependency 'SwiftProtobuf', '~> 1.3.0'
 end

bootstrap.sh

@@ -4,15 +4,14 @@
 set -e
 
 echo "#### Initializing... ####"
-git submodule update --init
-cmake -H. -Bbuild -DCMAKE_BUILD_TYPE=Debug -DGIT_SUBMODULE=OFF
+tools/install-dependencies
 
 echo "#### Generating files... ####"
-make -Cbuild protobuf_ext
 tools/generate-files
 
 echo "#### Building... ####"
+cmake -H. -Bbuild -DCMAKE_BUILD_TYPE=Debug
 make -Cbuild tests
 
 echo "#### Testing... ####"
-build/tests/tests
+build/tests/tests tests

cmake/Protobuf.cmake

@@ -1,23 +1,4 @@
-set(protobuf_PREFIX ${PROJECT_SOURCE_DIR}/lib/protobuf)
-file(MAKE_DIRECTORY ${protobuf_PREFIX})
-
-ExternalProject_Add(
-    protobuf_ext
-
-    URL https://github.com/protocolbuffers/protobuf/releases/download/v3.7.0/protobuf-cpp-3.7.0.tar.gz
-    URL_HASH SHA256=797e759e6daf644be4c09070d4777d178b538dca310cc4956fde4b51b02ba2d3
-
-    PREFIX "protobuf"
-    INSTALL_DIR ${protobuf_PREFIX}
-
-    UPDATE_COMMAND ""
-    PATCH_COMMAND ""
-    CONFIGURE_COMMAND <SOURCE_DIR>/configure --prefix <INSTALL_DIR>
-    BUILD_COMMAND ""
-    INSTALL_COMMAND ""
-)
-ExternalProject_Get_Property(protobuf_ext source_dir)
-set(protobuf_SOURCE_DIR ${source_dir})
+set(protobuf_SOURCE_DIR ${CMAKE_SOURCE_DIR}/build/protobuf/staging/protobuf-3.7.0)
 
 set(protobuf_SOURCE_FILES
     ${protobuf_SOURCE_DIR}/src/google/protobuf/stubs/bytestream.cc
@@ -136,13 +117,11 @@ set(protobuf_SOURCE_FILES
     ${protobuf_SOURCE_DIR}/src/google/protobuf/util/time_util.cc
     ${protobuf_SOURCE_DIR}/src/google/protobuf/util/type_resolver_util.cc
 )
-set_source_files_properties(${protobuf_SOURCE_FILES} PROPERTIES GENERATED TRUE)
 
 file(GLOB_RECURSE protobuf_HEADER_FILES ${protobuf_SOURCE_DIR}/src/**/*.h)
 include_directories(${protobuf_SOURCE_DIR}/src)
 
 add_library(protobuf ${protobuf_SOURCE_FILES})
-add_dependencies(protobuf protobuf_ext)
 set_target_properties(
     protobuf
     PROPERTIES

docs/Contributing.md

@@ -19,7 +19,7 @@ Keep this in mind when adding to the library:
 
 * Install Xcode
 * Install Xcode command line tools: `xcode-select --install`
-* Install CMake, boost, protobuf: `brew install cmake ninja boost protobuf swift-protobuf autoconf automake libtool`
+* Install CMake, boost, protobuf: `brew install cmake ninja boost autoconf automake libtool`
 * Install [Android Studio](https://developer.android.com/studio/index.html)
 * Install the [Android NDK](https://developer.android.com/ndk/guides/)
 
@@ -44,19 +44,19 @@ This project has a number of different pieces. Each piece lives in its own subfo
 
 Use the `bootstrap.sh` script in the root folder to quickly build and test.
 
-The build pipeline uses CMake. If you add or rename files you need to re-run cmake: `cmake -H. -Bbuild -DCMAKE_BUILD_TYPE=Debug -DGIT_SUBMODULE=OFF`. If you only change existing files and want to run the tests you only need to run make: `make -C build tests`.
+The build pipeline uses CMake. If you add or rename files you need to re-run cmake: `cmake -H. -Bbuild -DCMAKE_BUILD_TYPE=Debug`. If you only change existing files and want to run the tests you only need to run make: `make -C build tests`.
 
 If you change interface files in the include folder you need to regenerate the interface code: `codegen/bin/codegen`. Run `codegen/bin/codegen -h` to get usage information on the tool. Note that currently if you add a new interface header file you need to manually add that file as a public header to the iOS project, otherwise iOS tests will fail.
 
 ## Testing
 
-Use the `bootstrap.sh` script in the root folder to quickly build and test. After you have run either `bootstrap.sh` or `cmake`, run `make -C build tests && build/tests/tests`. This will run all the C++ tests. To run integration tests on each platform run the respective script in the tools folder:
+Use the `bootstrap.sh` script in the root folder to quickly build and test. After you have run either `bootstrap.sh` or `cmake`, run `make -C build tests && build/tests/tests tests`. This will run all the C++ tests. To run integration tests on each platform run the respective script in the tools folder:
 * Android: `tools/android-test`
 * iOS: `tools/ios-test`
 
 How to generate a Xcode project:
 
-`cmake -Bxcode -GXcode -DCMAKE_BUILD_TYPE=Debug -DGIT_SUBMODULE=OFF`
+`cmake -Bxcode -GXcode -DCMAKE_BUILD_TYPE=Debug`
 
 ## C Headers
 

protobuf-plugin/CMakeLists.txt

@@ -1,11 +1,16 @@
 cmake_minimum_required(VERSION 3.2 FATAL_ERROR)
 project(TrustWalletCoreProtobufPlugin)
 
-set (CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
 
-find_package(Protobuf REQUIRED)
+include_directories(${CMAKE_SOURCE_DIR}/build/local/include)
+link_directories(${CMAKE_SOURCE_DIR}/build/local/lib)
+
+find_library(Protobuf REQUIRED PATH ${CMAKE_SOURCE_DIR}/build/local/lib/pkgconfig NO_DEFAULT_PATH)
 include_directories(${Protobuf_INCLUDE_DIRS})
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
 add_executable(protoc-gen-int main.cc ${PROTO_SRCS} ${PROTO_HDRS})
-target_link_libraries(protoc-gen-int ${Protobuf_LIBRARIES} ${Protobuf_PROTOC_LIBRARIES})
+target_link_libraries(protoc-gen-int protobuf protoc)

swift/TrustWalletCore.xcodeproj/project.pbxproj

@@ -1704,7 +1704,7 @@
 				61A9B31321FF978800E601B3 /* wrappers.pb.cc */,
 			);
 			name = protobuf;
-			path = ../build/protobuf/src/protobuf_ext/src/google/protobuf;
+			path = "../build/protobuf/staging/protobuf-3.7.0/src/google/protobuf";
 			sourceTree = "<group>";
 		};
 		61CAC6AE21F113C60086D9A0 /* stubs */ = {
@@ -2765,8 +2765,7 @@
 					"$(inherited)",
 					"${SRCROOT}/../include",
 					"${SRCROOT}/../trezor-crypto/include",
-					"${SRCROOT}/../lib/protobuf/src",
-					"${SRCROOT}/../lib/json/include",
+					"${SRCROOT}/../build/local/include",
 				);
 				TARGETED_DEVICE_FAMILY = "1,2";
 			};
@@ -2807,8 +2806,7 @@
 					"$(inherited)",
 					"${SRCROOT}/../include",
 					"${SRCROOT}/../trezor-crypto/include",
-					"${SRCROOT}/../lib/protobuf/src",
-					"${SRCROOT}/../lib/json/include",
+					"${SRCROOT}/../build/local/include",
 				);
 				TARGETED_DEVICE_FAMILY = "1,2";
 			};
@@ -2820,7 +2818,6 @@
 				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
 				CODE_SIGN_STYLE = Automatic;
 				DEVELOPMENT_TEAM = 9873B38DWV;
-				HEADER_SEARCH_PATHS = "${SRCROOT}/../build/protobuf/src/protobuf_ext/src";
 				OTHER_CFLAGS = "-DHAVE_PTHREAD=1";
 				OTHER_LDFLAGS = "-ObjC";
 				PRODUCT_NAME = "$(TARGET_NAME)";
@@ -2835,7 +2832,6 @@
 				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
 				CODE_SIGN_STYLE = Automatic;
 				DEVELOPMENT_TEAM = 9873B38DWV;
-				HEADER_SEARCH_PATHS = "${SRCROOT}/../build/protobuf/src/protobuf_ext/src";
 				OTHER_CFLAGS = "-DHAVE_PTHREAD=1";
 				OTHER_LDFLAGS = "-ObjC";
 				PRODUCT_NAME = "$(TARGET_NAME)";

swift/cpp.xcconfig.in

@@ -4,4 +4,5 @@
 // terms governing use, modification, and redistribution, is contained in the
 // file LICENSE at the root of the source code distribution tree.
 
-HEADER_SEARCH_PATHS = $SRCROOT/../src @Boost_INCLUDE_DIRS@ @JSON_INCLUDE_DIR@
+HEADER_SEARCH_PATHS = $(SRCROOT)/../src @Boost_INCLUDE_DIRS@
+SYSTEM_HEADER_SEARCH_PATHS = $(SRCROOT)/../src/build/local/include $(SRCROOT)/../build/protobuf/staging/protobuf-3.7.0/src

tests/CMakeLists.txt

@@ -6,25 +6,18 @@ set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
 
 # Add googletest directly to our build. This defines
 # the gtest and gtest_main targets.
-add_subdirectory(${CMAKE_SOURCE_DIR}/lib/googletest
+add_subdirectory(${CMAKE_SOURCE_DIR}/build/gtest/staging/googletest-release-1.8.1
                  ${CMAKE_CURRENT_BINARY_DIR}/googletest-build
                  EXCLUDE_FROM_ALL)
 
-# The gtest/gtest_main targets carry header search path
-# dependencies automatically when using CMake 2.8.11 or
-# later. Otherwise we have to add them here ourselves.
-if (CMAKE_VERSION VERSION_LESS 2.8.11)
-  include_directories("${gtest_SOURCE_DIR}/include")
-endif()
-
 # Protobuf
-find_library(Protobuf REQUIRED)
+find_library(Protobuf REQUIRED PATH ${CMAKE_SOURCE_DIR}/build/local/lib/pkgconfig NO_DEFAULT_PATH)
 include_directories(${Protobuf_INCLUDE_DIRS})
 
 # Test executable
 file(GLOB_RECURSE test_sources *.cpp **/*.cpp)
 add_executable(tests ${test_sources})
-target_compile_features(tests PRIVATE cxx_std_14)
+target_compile_features(tests PRIVATE cxx_std_17)
 target_link_libraries(tests gtest_main TrustWalletCore protobuf Boost::boost)
 target_include_directories(tests PRIVATE ${CMAKE_SOURCE_DIR}/src)
 

tools/generate-files

@@ -10,17 +10,20 @@
 
 set -e
 
-PROTOBUF_DIR="$PWD/build/protobuf/src/protobuf_ext/"
+ROOT="$PWD"
+PREFIX="$ROOT/build/local"
+PATH="$PREFIX/bin":$PATH
+PROTOC="$PREFIX/bin/protoc"
 
 # Generate Java, C++ and Swift Protobuf files
 if [ -x "$(command -v protoc-gen-swift)" ]; then
-    protoc -I=$PROTOBUF_DIR/src -I=src/proto --cpp_out=src/proto --java_out=JNI/java --swift_out=swift/Sources --swift_opt=Visibility=Public src/proto/*.proto
+    "$PROTOC" -I=$PREFIX/include -I=src/proto --cpp_out=src/proto --java_out=JNI/java --swift_out=swift/Sources --swift_opt=Visibility=Public src/proto/*.proto
 else
-    protoc -I=$PROTOBUF_DIR/src -I=src/proto --cpp_out=src/proto --java_out=JNI/java src/proto/*.proto
+    "$PROTOC" -I=$PREFIX/include -I=src/proto --cpp_out=src/proto --java_out=JNI/java src/proto/*.proto
 fi
 
 # Generate internal Protobuf files
-protoc -I=$PROTOBUF_DIR/src -I=src/Tron/Protobuf --cpp_out=src/Tron/Protobuf src/Tron/Protobuf/*.proto
+"$PROTOC" -I=$PREFIX/include -I=src/Tron/Protobuf --cpp_out=src/Tron/Protobuf src/Tron/Protobuf/*.proto
 
 # Generate JS Protobuf files
 pushd js
@@ -30,9 +33,6 @@ npm run generate
 popd
 
 # Generate Proto interface file
-pushd protobuf-plugin
-mkdir -p build
 cmake -H. -Bbuild
-make -Cbuild
-protoc -I=$PROTOBUF_DIR/src -I=../src/proto --plugin=build/protoc-gen-int --int_out ../include/TrustWalletCore ../src/proto/*.proto
-popd
+make -Cbuild protoc-gen-int
+"$PROTOC" -I=$PREFIX/include -I=src/proto --plugin=build/protobuf-plugin/protoc-gen-int --int_out include/TrustWalletCore src/proto/*.proto