feat(#major); add sol balance substream

Cargo.toml

@@ -11,14 +11,15 @@ members = [
     "substreams-helper",
     "uniswap-v2",
     "solana-sample",
+    "sol-balance",
     "ens-names",
 ]
 exclude = ["messari-cli"]
 
 [workspace.dependencies]
 substreams = "0.3.2"
 substreams-ethereum = "0.6.2"
-substreams-solana = "0.1.0"
+substreams-solana = "0.2.0"
 
 [build]
 target = "wasm32-unknown-unknown"

Makefile

@@ -10,6 +10,7 @@ build-all:
 	$(MAKE) -C network build
 	$(MAKE) -C solana-sample build
 	$(MAKE) -C eth-balance build
+	$(MAKE) -C sol-balance build
 	$(MAKE) -C ens-names build
 
 .PHONY: run-all
@@ -19,6 +20,7 @@ run-all:
 	$(MAKE) -C erc20-market-cap run
 	$(MAKE) -C erc721 run
 	$(MAKE) -C network run
+	$(MAKE) -C sol-balance run
 	$(MAKE) -C ens-names run
 
 .PHONY: test

README.md

@@ -43,6 +43,7 @@
 | ENS Look Up      | 🔨 | ENS records for lookup and reverse lookup  |
 | Uniswap v2       | 🔨 | Substreams for Uniswap v2                  |
 | Compound v2      | 🔨 | Substreams for Compound v2                 |
+| SOL Balance      | 🛠 | SOL balance for every Solana address       |
 
 ## Workflow
 

common/proto/sol_token.proto

@@ -0,0 +1,47 @@
+syntax = "proto3";
+
+package messari.sol_token.v1;
+
+message Transfers {
+  repeated Transfer transfers = 1;
+}
+
+message Transfer {
+  string signature = 1;
+  string from = 2;
+  string to = 3;
+  TokenAccount token = 4;
+  string mint = 5;
+  uint64 native_amount = 6;
+  uint64 amount = 7;
+  BalanceChanges balance_changes = 8;
+}
+
+message BalanceChanges {
+  repeated TokenBalance items = 1;
+}
+
+// balance change
+message TokenBalance {
+  TokenAccount token = 1;
+  string transaction_id = 2;
+  uint64 block_height = 3;
+  string address = 4; // account address of the balance change
+  string pre_balance = 5; // BigInt, in token's native amount
+  string post_balance = 6; // BigInt, in token's native amount
+}
+
+message Tokens {
+  repeated TokenAccount tokens = 1;
+}
+
+message TokenAccount {
+  string address = 1;
+  string name = 2;
+  string symbol = 3;
+  uint64 decimals = 4;
+  string freeze_authority = 5;
+  string mint_authority = 6;
+  string tx_created = 7;
+  uint64 block_created = 8;
+}

erc20-holdings/src/pb.rs

@@ -1,39 +1,9 @@
 #[rustfmt::skip]
-#[path = "../target/pb/messari.chainlink.v1.rs"]
-pub(in crate::pb) mod chainlink_v1;
+#[path = "../target/pb/messari.sol_token.v1.rs"]
+pub(in crate::pb) mod sol_token_v1;
 
-pub mod chainlink {
+pub mod sol_token {
     pub mod v1 {
-        pub use super::super::chainlink_v1::*;
-    }
-}
-
-#[rustfmt::skip]
-#[path = "../target/pb/messari.common.v1.rs"]
-pub(in crate::pb) mod common_v1;
-
-pub mod common {
-    pub mod v1 {
-        pub use super::super::common_v1::*;
-    }
-}
-
-#[rustfmt::skip]
-#[path = "../target/pb/messari.erc20.v1.rs"]
-pub(in crate::pb) mod erc20_v1;
-
-pub mod erc20 {
-    pub mod v1 {
-        pub use super::super::erc20_v1::*;
-    }
-}
-
-#[rustfmt::skip]
-#[path = "../target/pb/messari.erc20_price.v1.rs"]
-pub(in crate::pb) mod erc20_price_v1;
-
-pub mod erc20_price {
-    pub mod v1 {
-        pub use super::super::erc20_price_v1::*;
+        pub use super::super::sol_token_v1::*;
     }
 }

erc20-market-cap/src/pb.rs

@@ -37,3 +37,13 @@ pub mod erc20_price {
         pub use super::super::erc20_price_v1::*;
     }
 }
+
+#[rustfmt::skip]
+#[path = "../target/pb/messari.uniswap.v1.rs"]
+pub(in crate::pb) mod uniswap_v1;
+
+pub mod uniswap {
+    pub mod v1 {
+        pub use super::super::uniswap_v1::*;
+    }
+}

sol-balance/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "sol-balance"
+version = "0.1.0"
+description = "Messari's substream for solana SOL balances"
+edition = "2021"
+repository = "https://github.com/messari/substreams"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+bs58 = "0.4.0"
+prost = "0.11.2"
+ethabi = "17.2.0"
+num-bigint = "0.4"
+hex-literal = "0.3.4"
+substreams = { workspace = true }
+substreams-solana = { workspace = true }
+substreams-helper = { path = "../substreams-helper" }
+
+[build-dependencies]
+anyhow = "1"
+substreams-common = { path = "../common" }

sol-balance/Makefile

@@ -0,0 +1,7 @@
+.PHONY: build
+build:
+	cargo build --target wasm32-unknown-unknown --release
+
+.PHONY: run
+run:
+	substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml map_balances -s 174257750 -t +1

sol-balance/README.md

@@ -0,0 +1,10 @@
+# SOL Balance Substream
+
+This substream is designed to store the SOL balance of every account.
+
+### Notes  
+
+- We are not able fully map all the accounts to SOL balance changes.
+  - There is a missing field [`address_table_lookup`](https://github.com/streamingfast/firehose-solana/blob/develop/proto/sf/solana/type/v1/type.proto#L38) that stores the rest of the addresses.
+  - See the issue filed [here](https://github.com/streamingfast/substreams/issues/144) to track the status.
+  - This field is available, but there is decoding needed on the substream part in order to actually get that data. Next steps are understanding the Solana data structure better and seeing firesol output to get a better understanding of where this data lives.

sol-balance/build.rs

@@ -0,0 +1,10 @@
+use anyhow::{Ok, Result};
+use substreams_common::codegen;
+
+fn main() -> Result<(), anyhow::Error> {
+    println!("cargo:rerun-if-changed=proto");
+    println!("cargo:rerun-if-changed=abi");
+    codegen::generate(None)?;
+
+    Ok(())
+}

sol-balance/rust-toolchain.toml

@@ -0,0 +1,4 @@
+[toolchain]
+channel = "1.64.0"
+components = [ "rustfmt" ]
+targets = [ "wasm32-unknown-unknown" ]

sol-balance/src/abi.rs

@@ -0,0 +1 @@
+// DO NOT EDIT - the file is generated by build script

sol-balance/src/lib.rs

@@ -0,0 +1,41 @@
+use bs58;
+use substreams_helper::pb::sol_token::v1 as proto;
+use substreams_helper::token::get_sol_token;
+use substreams_solana::pb::sol as solana;
+
+#[substreams::handlers::map]
+fn map_balances(
+    block: solana::v1::Block,
+) -> Result<proto::BalanceChanges, substreams::errors::Error> {
+    let mut balances = vec![];
+    for tx in block.transactions {
+        if let Some(meta) = tx.meta {
+            if let Some(_) = meta.err {
+                continue;
+            }
+            if let Some(transaction) = tx.transaction {
+                if let Some(msg) = transaction.message {
+                    for i in 0..meta.post_balances.len() {
+                        let account_id: String;
+                        if i < msg.account_keys.len() {
+                            account_id = bs58::encode(&msg.account_keys[i]).into_string();
+                        } else {
+                            account_id = "TODO".to_string();
+                            // TODO: use msg.address_table_lookup table, but it is not avail
+                        }
+                        balances.push(proto::TokenBalance {
+                            token: get_sol_token(),
+                            transaction_id: bs58::encode(&transaction.signatures[0]).into_string(),
+                            block_height: block.block_height.as_ref().unwrap().block_height,
+                            address: account_id,
+                            pre_balance: meta.pre_balances[i].to_string(),
+                            post_balance: meta.post_balances[i].to_string(),
+                        });
+                    }
+                }
+            }
+        }
+    }
+
+    Ok(proto::BalanceChanges { items: balances })
+}

sol-balance/src/pb.rs

@@ -0,0 +1,19 @@
+#[rustfmt::skip]
+#[path = "../target/pb/messari.sol_token.v1.rs"]
+pub(in crate::pb) mod sol_token_v1;
+
+pub mod sol_token {
+    pub mod v1 {
+        pub use super::super::sol_token_v1::*;
+    }
+}
+
+#[rustfmt::skip]
+#[path = "../target/pb/messari.solana.type.rs"]
+pub(in crate::pb) mod solana_type;
+
+pub mod solana {
+    pub mod type {
+        pub use super::super::solana_type::*;
+    }
+}

sol-balance/substreams.yaml

@@ -0,0 +1,27 @@
+specVersion: v0.1.0
+package:
+  name: sol_balance
+  version: v0.1.0
+
+imports:
+  sol: https://github.com/streamingfast/firehose-solana/releases/download/v0.1.0/solana-v0.1.0.spkg
+
+protobuf:
+  files:
+    - sol_token.proto
+  importPaths:
+    - ../common/proto
+
+binaries:
+  default:
+    type: wasm/rust-v1
+    file: "../target/wasm32-unknown-unknown/release/sol_balance.wasm"
+
+modules:
+  # get the SOL balance change of each account in each transaction in a block
+  - name: map_balances
+    kind: map
+    inputs:
+      - source: sf.solana.type.v1.Block
+    output:
+      type: proto:messari.sol.token.BalanceChanges

substreams-helper/build.rs

@@ -2,6 +2,7 @@ use anyhow::{Ok, Result};
 use substreams_common::codegen;
 
 fn main() -> Result<(), anyhow::Error> {
+    codegen::generate(None);
     codegen::generate_abi(None)?;
     Ok(())
 }

substreams-helper/src/lib.rs

@@ -2,6 +2,8 @@ pub mod abi;
 pub mod erc20;
 pub mod keyer;
 pub mod math;
+pub mod pb;
 pub mod price;
+pub mod token;
 pub mod types;
 pub mod utils;

substreams-helper/src/pb.rs

@@ -0,0 +1,9 @@
+#[rustfmt::skip]
+#[path = "../target/pb/messari.sol_token.v1.rs"]
+pub(in crate::pb) mod sol_token_v1;
+
+pub mod sol_token {
+    pub mod v1 {
+        pub use super::super::sol_token_v1::*;
+    }
+}