diff --git a/.github/workflows/build-common.yml b/.github/workflows/build-common.yml index 9d998181..76f150dd 100644 --- a/.github/workflows/build-common.yml +++ b/.github/workflows/build-common.yml @@ -30,9 +30,6 @@ jobs: sudo dnf install -y protobuf-compiler sudo dnf install -y systemd-devel - - name: Clone protosol - run: make fetch_proto - - name: Check lints and clippy run: | cargo fmt --all -- --check diff --git a/Cargo.lock b/Cargo.lock index 0f3eb643..0b83c48f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5158,6 +5158,15 @@ dependencies = [ "autotools", ] +[[package]] +name = "protosol" +version = "1.0.1" +source = "git+https://github.com/firedancer-io/protosol?tag=v1.0.4#72fb79f2a24e60a55d392c386c1ac7c2d15c13a4" +dependencies = [ + "prost 0.11.9", + "prost-build 0.11.9", +] + [[package]] name = "qstring" version = "0.7.2" @@ -10681,6 +10690,7 @@ dependencies = [ "prost-build 0.13.5", "prost-types 0.11.9", "protobuf-src", + "protosol", "qstring", "qualifier_attr", "quinn", diff --git a/Cargo.toml b/Cargo.toml index 80293b0d..f9288fc6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -518,6 +518,7 @@ winapi = "0.3.8" x509-parser = "0.14.0" zeroize = { version = "1.7", default-features = false } zstd = "0.13.3" +protosol = {git = "https://github.com/firedancer-io/protosol", tag = "v1.0.4"} solfuzz-agave-macro = { path = "macro" } once_cell = "1.21.3" lazy_static = "1.5.0" diff --git a/Makefile b/Makefile index a7c33080..ad9235fe 100644 --- a/Makefile +++ b/Makefile @@ -17,17 +17,14 @@ CC:=clang CARGO?=cargo -.PHONY: build clean binaries shared_obj fetch_proto +.PHONY: build clean binaries shared_obj -all: | fetch_proto shared_obj binaries +all: | shared_obj binaries # Alias for backwards compatibility -build: | fetch_proto shared_obj +build: | shared_obj -conformance: | fetch_proto shared_obj_debug - -fetch_proto: - ./scripts/fetch_proto.sh +conformance: | shared_obj_debug shared_obj: RUSTFLAGS="$(RUSTFLAGS)" $(CARGO) build --target x86_64-unknown-linux-gnu --release --lib diff --git a/README.md b/README.md index 45a0d333..c0cf0901 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,19 @@ apt install libudev-dev protobuf-compiler pkg-config Running with a local copy of Agave (for easily adding print statements): ```sh -pip3 install tomlkit -python3 scripts/generate_cargo.py -p -o Cargo.toml +./scripts/generate_cargo.py -p -o Cargo.toml ``` +Running with a specific Agave commit: +```sh +./scripts/generate_cargo.py -c -o Cargo.toml +``` + +Running with a specific protosol version: +```sh +./scripts/generate_cargo.py -c -v -o Cargo.toml +# Example: ./scripts/generate_cargo.py -c -v 1.0.4 -o Cargo.toml +``` Check and test: @@ -64,7 +73,7 @@ $ nm -D target/x86_64-unknown-linux-gnu/release/libsolfuzz_agave.so | grep '__sa U __sanitizer_cov_trace_pc_indir ``` -**Note:** You may have to periodically run `make build` to ensure that Protobuf definitions stay in sync with [Protosol](https://github.com/firedancer-io/protosol/). Alternatively, you can run `./scripts/fetch_proto.sh` to keep Protosol up to date. Maintainers are expected to bump the `PROTO_VERSION` corresponding to the versioned git tag of the protosol repository when staging new protobuf changes. +**Note:** Protobuf definitions are now managed through the `protosol` Rust crate dependency. The `protosol` crate is automatically included in the generated Cargo.toml with a specific version tag. When protobuf schema changes are needed, maintainers should update the `PROTOSOL_VERSION_TAG` in `scripts/generate_cargo.py` to the appropriate version tag from the [protosol repository](https://github.com/firedancer-io/protosol/). ## Building Targets with Core BPF Programs diff --git a/build.rs b/build.rs index f61b0f4f..d126eb73 100644 --- a/build.rs +++ b/build.rs @@ -1,6 +1,6 @@ -use std::io::Result; +use std::{env, fs, path::PathBuf}; -fn main() -> Result<()> { +fn main() -> Result<(), Box> { // Tells rustc to recompile the `load_core_bpf_program!` macro if either // of the required environment variables has changed. println!("cargo:rerun-if-env-changed=CORE_BPF_PROGRAM_ID"); @@ -12,36 +12,40 @@ fn main() -> Result<()> { println!("cargo:rerun-if-changed=force_rebuild"); } - let proto_base_path = std::path::PathBuf::from("protosol/proto"); - - let protos = &[ - proto_base_path.join("invoke.proto"), - proto_base_path.join("vm.proto"), - proto_base_path.join("txn.proto"), - proto_base_path.join("elf.proto"), - proto_base_path.join("shred.proto"), - proto_base_path.join("pack.proto"), - proto_base_path.join("block.proto"), - proto_base_path.join("type.proto"), - ]; - - protos - .iter() - .for_each(|proto| println!("cargo:rerun-if-changed={}", proto.display())); - - for proto in protos { - if !proto.exists() { - return Err(std::io::Error::new( - std::io::ErrorKind::NotFound, - format!( - "Proto file does not exist: {}. Run ./scripts/fetch_proto.sh", - proto.display() - ), - )); + // Get absolute proto dir from producer + let proto_dir = PathBuf::from( + env::var("DEP_PROTOSOL_PROTO_DIR") + .expect("protosol did not expose PROTO_DIR, did protosol build.rs run first?"), + ); + + println!("cargo:rerun-if-env-changed=DEP_PROTOSOL_PROTO_DIR"); + println!("cargo:rerun-if-changed={}", proto_dir.display()); + + // Collect absolute .proto paths + let mut proto_files = vec![]; + for entry in fs::read_dir(&proto_dir)? { + let path = entry?.path(); + if path.extension().and_then(|e| e.to_str()) == Some("proto") { + println!("cargo:rerun-if-changed={}", path.display()); + proto_files.push(path); } } - prost_build::compile_protos(protos, &[proto_base_path])?; + // Ensure deterministic order for rebuilds + proto_files.sort(); + + // Compile protos into Rust + let out_dir = PathBuf::from(env::var("OUT_DIR")?); + let mut config = prost_build::Config::new(); + config.out_dir(&out_dir); + + config.compile_protos( + &proto_files + .iter() + .map(|p| p.display().to_string()) + .collect::>(), + &[proto_dir.to_str().unwrap()], + )?; Ok(()) } diff --git a/scripts/fetch_proto.sh b/scripts/fetch_proto.sh deleted file mode 100755 index 3ba481aa..00000000 --- a/scripts/fetch_proto.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Allow overriding proto version; default pinned -PROTO_VERSION="${PROTO_VERSION:-v1.0.4}" - -# Fetch protosol at specified tag/branch -if [ ! -d protosol ]; then - git clone --depth=1 --branch "$PROTO_VERSION" https://github.com/firedancer-io/protosol.git -else - cd protosol - git fetch --tags - git checkout "$PROTO_VERSION" - cd .. -fi diff --git a/scripts/generate_cargo.py b/scripts/generate_cargo.py index 8b4a363d..d8c556d6 100755 --- a/scripts/generate_cargo.py +++ b/scripts/generate_cargo.py @@ -11,6 +11,9 @@ import os import subprocess +# NOTE: this needs bumped with schema version upgrades of the protocol +PROTOSOL_VERSION_TAG = "v1.0.4" + def replace_path_with_git_rev(toml_data, git_url, rev): """ Recursively process the TOML data to replace path with git and rev, @@ -78,6 +81,7 @@ def parse_toml_file(file_path): return parse(f.read()) def main(): + global PROTOSOL_VERSION_TAG parser = argparse.ArgumentParser(description="Process input files.") @@ -85,8 +89,15 @@ def main(): parser.add_argument("--commit", "-c", help="Commit in firedancer-io/agave to use") parser.add_argument("--agave-path", "-p", help="Commit in firedancer-io/agave to use") parser.add_argument("--output", "-o", help="Path to the output file") + parser.add_argument("--version", "-v", help=f"Protosol version to use (e.g. \"{PROTOSOL_VERSION_TAG}\")") args = parser.parse_args() + if args.version: + PROTOSOL_VERSION_TAG = args.version + # Prepend 'v' if not already present + if not PROTOSOL_VERSION_TAG.startswith('v'): + PROTOSOL_VERSION_TAG = 'v' + PROTOSOL_VERSION_TAG + print(f"Using protosol version: {PROTOSOL_VERSION_TAG}") if args.agave_path: toml_data = parse_toml_file(args.agave_path + "/Cargo.toml") @@ -117,6 +128,20 @@ def main(): if patch_to_remove in toml_data.get("patch", {}).get("crates-io", {}): del toml_data["patch"]["crates-io"][patch_to_remove] + # Add protosol dependency (deduped if already present) + if "dependencies" not in toml_data: + toml_data["dependencies"] = table() + + # Remove existing protosol if present to ensure we use the correct version + if "protosol" in toml_data["dependencies"]: + del toml_data["dependencies"]["protosol"] + + # Add the required protosol dependency + protosol_dep = inline_table() + protosol_dep["git"] = "https://github.com/firedancer-io/protosol" + protosol_dep["tag"] = PROTOSOL_VERSION_TAG + toml_data["dependencies"]["protosol"] = protosol_dep + # add required solfuzz-agave added configurations solfuzz_agave_config = parse_toml_file("solfuzz_agave.toml") for section, values in solfuzz_agave_config.items(): @@ -141,4 +166,4 @@ def main(): f.write(toml_data.as_string()) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/run_test_vectors.sh b/scripts/run_test_vectors.sh index 282f05cc..2d106fe9 100755 --- a/scripts/run_test_vectors.sh +++ b/scripts/run_test_vectors.sh @@ -20,9 +20,6 @@ mkdir -p dump # Show the commit hashes being used repo_commit=$(git rev-parse HEAD) echo "Using repo commit: $repo_commit" -protosol_commit=$(cd protosol && git rev-parse HEAD) -protosol_tag=$(cd protosol && git describe --tags --exact-match 2>/dev/null || echo "") -echo "Using protosol commit: $protosol_commit (tag: $protosol_tag)" # Fetch/update test-vectors repo if [ ! -d dump/test-vectors ]; then