Supports ropsten ethereum header source for test usages (#66)

* Merge rustify into ropsten

* feat(eth): supports default ropsten rpc

* chore(tests): remove optional deps

* chore(crate): renamme package name to darwinia-shadow

* feat(lib): add readme to lib.rs

* fix(go): rename internal pkg to shadow

* fix(lint): optimize code style

* chore(docker): update dockerfile and docker-compose.yml

* fix(mmr): fix the genesis mmr size

* fix(linker): declare links in Cargo.toml

* fix(example): the linker in proof example

* perf(linker): supports dynamic link in examples

* fix(example): exports ffi for test usages

* perf(ffi): move tests into module

* perf(build): add go.mod and go.sum into manifest

* fix(pkg): remove the missing internal pkg

* fix(cgo): enable cgo mod and add libc6-compat to alpine

* perf(linker): use dynamic linker to invoke diesel libraries in musl

* feat(docker): complete the dockerfile of alpine

* chore(bump): bump versions to v.0.2.0

* fix(ci): update LD_LIBRARY_PATH for ubuntu

* fix(linker): use rustc-link-search to find libdarwinia_shadow in ubuntu

* fix(ci): use OUT_DIR to customize the output of libdarwinia_shadow_ext

* fix(dylib): adapt the loading process of dynamic link for common linux

* fix(docker): remove the --out-dir flag for building with stable version rustc

* fix(dylib): adapt LD_LIBRARY_PATH for unix

* fix(dylib): force move and overwrite old libdarwinia_shadow_ext

* chore(dylib): use sudo inside build.rs to try to fix perm in ubuntu

* chore(dylib): use sudo inside build.rs to fix perm in ubuntu...again

* fix(ci): ldconfig in ci after installing

* feat(eth): optimzie proof process and add dockerignore

* chore(README): remove outdated lines

* fix(docs.rs): removed all sudo commands, supports both dylink and static link for unix

* chore(chains): remove outdated modules

* fix(ci): force link-search out_dir while building static library

* fix(typo): fix typo in badge link

Author: clearloop

.dockerignore

@@ -0,0 +1,3 @@
+target
+examples
+tests

.github/workflows/mmr.yml .github/workflows/shadow.yml.github/workflows/mmr.yml renamed to .github/workflows/shadow.yml

@@ -1,4 +1,4 @@
-name: Rust MMR
+name: shadow
 
 on:
   push:
@@ -23,6 +23,7 @@ jobs:
           else
               sudo apt-get update -y
               sudo apt-get install -y libsqlite3-dev libdbus-1-dev
+              export LD_LIBRARY_PATH=/lib:/usr/lib:/usr/local/lib
           fi
       # Cache MMR Library
       - name: Cache MMR Library
@@ -33,6 +34,8 @@ jobs:
           path: ./target
           key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('./Cargo.lock') }}
       - name: Build
-        run: cargo build --verbose
+        run: cargo build --release -vv
+      - name: Check Binary
+        run: ./target/release/shadow --version
       - name: Run tests
-        run: cargo test --verbose
+        run: cargo test --release -vv

Cargo.toml

@@ -1,8 +1,25 @@
 [package]
-name = "shadow"
-version = "0.1.0"
+name = "darwinia-shadow"
+version = "0.2.0"
 authors = ["clearloop <[email protected]>"]
 edition = "2018"
+description = "The shadow service for relayers and verify workers to retrieve header data and generate proof."
+repository = "https://github.com/darwinia-network/shadow"
+license = "GPL-3.0-or-later"
+documentation = "https://docs.rs/darwinia-shadow"
+homepage = "https://github.com/darwinia-network/shadow"
+include = [
+  "go.mod",
+  "go.sum",
+  "build.rs",
+  "pkg/**/*",
+  "src/**/*",
+  "Cargo.toml",
+  "./README.md",
+]
+keywords = ["darwinia", "substrate", "ethereum", "service"]
+readme = './README.md'
+links = "darwinia_shadow"
 
 [[bin]]
 name = "shadow"
@@ -38,12 +55,3 @@ features = ["derive"]
 [dependencies.diesel]
 version = "1.4.4"
 features = ["r2d2", "sqlite"]
-
-[dependencies.eth]
-git = "https://github.com/darwinia-network/darwinia-common.git"
-package = "ethereum-primitives"
-optional = true
-
-[features]
-darwinia = [ "eth" ]
-default = []

Dockerfile

@@ -1,25 +1,31 @@
-# Build MMR in a stock rust builder container
-FROM rust:alpine as mmr
+# Build Shadow in a stock rust builder container
+FROM rust:alpine as shadow
 ARG DEBIAN_FRONTEND=noninteractive
 ENV TZ=America/Los_Angeles
 COPY . shadow
-RUN apk add --no-cache sqlite-dev bash musl-dev \
-    && cd shadow \
-    && cargo build --release
 
-# Build Shadow in a stock Go builder container
-FROM golang:1.14-alpine as shadow
-ARG DEBIAN_FRONTEND=noninteractive
-ENV TZ=America/Los_Angeles
-COPY --from=mmr /shadow/target/release/libmmr.a /usr/local/lib/
-COPY . shadow
-RUN apk add --no-cache sqlite-dev sqlite-libs musl-dev gcc bash \
-    && mkdir /outputs \
+# Required dynamic libraries
+#
+# libdarwinia_shadow.so => /usr/local/lib/libdarwinia_shadow.so (0x7fd26af02000)
+# libssl.so.1.1 => /lib/libssl.so.1.1 (0x7fd26ae81000)
+# libcrypto.so.1.1 => /lib/libcrypto.so.1.1 (0x7fd26ac02000)
+# libsqlite3.so.0 => /usr/lib/libsqlite3.so.0 (0x7fd26ab1a000)
+# libc.musl-x86_64.so.1 => /lib/ld64.so.1 (0x7fd26bebb000)
+RUN apk add --no-cache gcc go openssl-dev sqlite-dev\
     && cd shadow \
-    && go build -o /usr/local/bin/shadow -v bin/main.go
+    && cargo build --release -vv \
+    && mkdir /target \
+    && cp target/release/shadow /target/ \
+    && cp /usr/lib/libsqlite3.so.0 /target/libsqlite3.so.0 \
+    && cp /usr/local/lib/libdarwinia_shadow.so /target/libdarwinia_shadow.so
 
-# Pull Geth and Shadow into a third stage deploy alpine container
+# Pull Shadow into a second stage deploy alpine container
 FROM alpine:latest
-COPY --from=shadow /usr/local/bin/shadow /usr/local/bin/shadow
+COPY --from=shadow /target /target
+RUN mv /target/shadow /usr/local/bin/shadow \
+    && mv /target/libsqlite3.so.0 /usr/lib/libsqlite3.so.0 \
+    && mv /target/libdarwinia_shadow.so /usr/local/lib/libdarwinia_shadow.so \
+    && cp /lib/libc.musl-x86_64.so.1 /lib/ld64.so.1 \
+    && rm -rf /target
 EXPOSE 3000
 ENTRYPOINT ["shadow"]

README.md

@@ -1,13 +1,17 @@
 # Shadow
 
-[![Golang CI][workflow-badge]][github]
+[![Shadow][workflow-badge]][github]
+[![crate](https://img.shields.io/crates/v/darwinia-shadow.svg)](https://crates.io/crates/darwinia_shadow)
+[![doc](https://img.shields.io/badge/current-docs-brightgreen.svg)](https://docs.rs/darwinia_shadow/)
+[![LICENSE](https://img.shields.io/crates/l/darwinia-shadow.svg)](https://choosealicense.com/licenses/gpl-3.0/)
 
 The shadow service for relayers and verify workers to retrieve header data and generate proof. Shadow will index the data it needs from blockchain nodes, such as Ethereum and Darwinia.
 
+
 ## Usage
 
 ```sh
-shadow 0.1.0
+shadow 0.2.0
 
 USAGE:
     shadow <SUBCOMMAND>
@@ -23,36 +27,30 @@ SUBCOMMANDS:
     trim     Trim mmr from target leaf
 ```
 
-## Contribute and Build
 
-Downloads shadow service
+## Download
 
-```
-git clone https://github.com/darwinia-network/shadow.git
+```sh
+$ cargo install darwinia-shadow
 ```
 
-Starts shadow service:
 
-```
-# Starts shadow serives at port 3000
-$ cargo run -p 3000
+### Note
+
++ Please make sure you have `golang` installed in your machine
++ Please make sure you have `sqlite3` installed in your machine
 
-# If you have fast eth node:
-$ ETHEREUM_RPC=<your-api> cargo run -p 3000
-```
 
 ## Trouble Shooting
 
 Everytime you run `proof` in error, please delete `~/.ethashproof` and `~/.ethash` 
 and retry.
 
+
 ## LICENSE
 
 GPL-3.0
 
 
-[infura]: https://infura.io
 [github]: https://github.com/darwinia-network/shadow
-[spec]: https://github.com/darwinia-network/darwinia/wiki/Darwinia-offchain-worker-shadow-service-spec
-[workflow-badge]: https://github.com/darwinia-network/shadow/workflows/Golang%20CI/badge.svg
-[api]: https://darwinia-network.github.io/shadow
+[workflow-badge]: https://github.com/darwinia-network/shadow/workflows/shadow/badge.svg

build.rs

@@ -1,32 +1,59 @@
-use std::process::Command;
+use std::{env, process::Command};
 
 fn main() {
-    let os = Command::new("uname").output().unwrap();
-    let ext = match String::from_utf8_lossy(os.stdout.as_slice())
-        .into_owned()
-        .trim_end()
-        .as_ref()
-    {
-        "Darwin" => "dylib",
-        _ => "so",
-    };
+    // Pre-check
+    println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-changed=path/to/Cargo.lock");
 
-    let profile = match std::env::var("PROFILE").unwrap().as_str() {
-        "release" => "release",
-        _ => "debug",
-    };
+    // Declare build args
+    let mut dynamic = true;
+    let out_dir = env::var("OUT_DIR").unwrap();
+    let staticlib = format!("{}/libdarwinia_shadow.a", out_dir);
+    let dylib = format!(
+        "/usr/local/lib/libdarwinia_shadow.{}",
+        match String::from_utf8_lossy(Command::new("uname").output().unwrap().stdout.as_slice())
+            .into_owned()
+            .trim_end()
+            .as_ref()
+        {
+            "Darwin" => "dylib",
+            _ => "so",
+        }
+    );
 
-    let lib = format!("target/{}/libeth.{}", profile, ext);
-    Command::new("go")
-        .args(&[
+    let args = |dynamic: bool| {
+        vec![
             "build",
             "-o",
-            &lib,
-            "-buildmode=c-shared",
-            "pkg/internal/ffi/mod.go",
-        ])
+            if dynamic { &dylib } else { &staticlib },
+            if dynamic {
+                "-buildmode=c-shared"
+            } else {
+                "-buildmode=c-archive"
+            },
+            "-v",
+            "pkg/shadow/ffi/mod.go",
+        ]
+    };
+
+    // Build the link library
+    if !Command::new("go")
+        .args(&args(dynamic))
         .status()
-        .unwrap();
+        .unwrap()
+        .success()
+    {
+        dynamic = false;
+        Command::new("go").args(&args(dynamic)).status().unwrap();
+        println!("built static library at {}", out_dir);
+    }
 
-    println!(r"cargo:rustc-link-search=target/debug");
+    // Post-check
+    if dynamic {
+        println!("cargo:rustc-link-search=/usr/local/lib");
+        println!("cargo:rustc-link-lib=dylib=darwinia_shadow");
+    } else {
+        println!("cargo:rustc-link-search={}", out_dir);
+        println!("cargo:rustc-link-lib=static=darwinia_shadow");
+    }
 }

docker-compose.yml

@@ -16,13 +16,6 @@ services:
       - data:/root
     depends_on:
       - geth
-  mmr:
-    image: quay.io/darwinia-network/shadow:next
-    command: run -vm --no-fetch --no-api
-    volumes:
-      - data:/root
-    depends_on:
-      - fetcher
 
 volumes:
   data:

examples/doctor.rs

@@ -1,9 +1,8 @@
 //! Mock the uncle block
 use cmmr::{leaf_index_to_mmr_size, MMR};
-use shadow::{
-    hash::{MergeHash, H256},
-    pool,
-    store::Store,
+use darwinia_shadow::{
+    db::pool,
+    mmr::{MergeHash, Store, H256},
 };
 
 fn main() {

examples/header.rs

@@ -1,4 +1,4 @@
-use shadow::chain::eth::{EthHeader, EthHeaderRPCResp};
+use darwinia_shadow::chain::eth::{EthHeader, EthHeaderRPCResp};
 
 fn main() {
     let client = reqwest::Client::new();