Skip to content

Commit 4824f5f

Browse files
KasamaKasama
committed
add initial cargo ci
1 parent 9f59118 commit 4824f5f

File tree

6 files changed

+206
-69
lines changed

6 files changed

+206
-69
lines changed

‎.cargo/config.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ rustflags = ["--cfg", "tokio_unstable"]
33

44
[alias]
55
bump-versions = "run -p upgrade-version --"
6+
ci = "run -p ci --"

‎.github/workflows/ci.yml‎

Lines changed: 16 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ jobs:
2121
include:
2222
- { runner: spacetimedb-runner, smoketest_args: --docker }
2323
- { runner: windows-latest, smoketest_args: --no-build-cli }
24-
runner: [ spacetimedb-runner, windows-latest ]
24+
runner: [spacetimedb-runner, windows-latest]
2525
runs-on: ${{ matrix.runner }}
2626
steps:
2727
- name: Find Git ref
@@ -76,8 +76,8 @@ jobs:
7676
- name: Install psycopg2
7777
run: python -m pip install psycopg2-binary
7878
- name: Run smoketests
79-
# Note: clear_database and replication only work in private
80-
run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication
79+
run: cargo ci smoketests -- ${{ matrix.smoketest_args }}
80+
8181
- name: Stop containers (Linux)
8282
if: always() && runner.os == 'Linux'
8383
run: docker compose down
@@ -109,26 +109,9 @@ jobs:
109109
with:
110110
global-json-file: global.json
111111

112-
- name: Create /stdb dir
113-
run: |
114-
sudo mkdir /stdb
115-
sudo chmod 777 /stdb
116-
117112
- name: Run cargo test
118113
#Note: Unreal tests will be run separately
119-
run: cargo test --all -- --skip unreal
120-
121-
- name: Check that the test outputs are up-to-date
122-
run: bash tools/check-diff.sh
123-
124-
- name: Ensure C# autogen bindings are up-to-date
125-
run: |
126-
cargo run -p spacetimedb-codegen --example regen-csharp-moduledef
127-
bash tools/check-diff.sh crates/bindings-csharp
128-
129-
- name: C# bindings tests
130-
working-directory: crates/bindings-csharp
131-
run: dotnet test -warnaserror
114+
run: cargo ci test
132115

133116
lints:
134117
name: Lints
@@ -145,29 +128,7 @@ jobs:
145128
global-json-file: global.json
146129

147130
- name: Run cargo fmt
148-
run: cargo fmt --all -- --check
149-
150-
- name: Run cargo clippy
151-
run: cargo clippy --all --tests --benches -- -D warnings
152-
153-
- name: Run C# formatting check
154-
working-directory: crates/bindings-csharp
155-
run: |
156-
dotnet tool restore
157-
dotnet csharpier --check .
158-
159-
- name: Run `cargo doc` for bindings crate
160-
# `bindings` is the only crate we care strongly about documenting,
161-
# since we link to its docs.rs from our website.
162-
# We won't pass `--no-deps`, though,
163-
# since we want everything reachable through it to also work.
164-
# This includes `sats` and `lib`.
165-
working-directory: crates/bindings
166-
env:
167-
# Make `cargo doc` exit with error on warnings, most notably broken links
168-
RUSTDOCFLAGS: '--deny warnings'
169-
run: |
170-
cargo doc
131+
run: cargo ci lint
171132

172133
wasm_bindings:
173134
name: Build and test wasm bindings
@@ -179,20 +140,7 @@ jobs:
179140
- run: echo ::add-matcher::.github/workflows/rust_matcher.json
180141

181142
- name: Run bindgen tests
182-
run: cargo test -p spacetimedb-codegen
183-
184-
# Make sure the `Cargo.lock` file reflects the latest available versions.
185-
# This is what users would end up with on a fresh module, so we want to
186-
# catch any compile errors arising from a different transitive closure
187-
# of dependencies than what is in the workspace lock file.
188-
#
189-
# For context see also: https://github.com/clockworklabs/SpacetimeDB/pull/2714
190-
- name: Update dependencies
191-
run: cargo update
192-
193-
- name: Build module-test
194-
run: cargo run -p spacetimedb-cli -- build --project-path modules/module-test
195-
143+
run: cargo ci wasm-bindings
196144

197145
publish_checks:
198146
name: Check that packages are publishable
@@ -245,19 +193,9 @@ jobs:
245193
run: sudo apt install -y libssl-dev
246194

247195
- name: Build spacetimedb-update
248-
run: cargo build --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update
249-
250-
- name: Run self-install
251196
env:
252197
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
253-
shell: bash
254-
run: |
255-
ROOT_DIR="$(mktemp -d)"
256-
# NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`.
257-
# My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it
258-
# happens very frequently on the `macos-runner`, but we haven't seen it on any others).
259-
cargo run --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update -- self-install --root-dir="${ROOT_DIR}" --yes
260-
"${ROOT_DIR}"/spacetime --root-dir="${ROOT_DIR}" help
198+
run: cargo ci update-flow --target=${{ matrix.target }}
261199

262200
unreal_engine_tests:
263201
name: Unreal Engine Tests
@@ -296,6 +234,15 @@ jobs:
296234
with:
297235
ref: ${{ env.GIT_REF }}
298236
- uses: dsherret/rust-toolchain-file@v1
237+
- name: Install unreal engine test dependencies
238+
working-directory: sdks/unreal
239+
env:
240+
UE_ROOT_PATH: /home/ue4/UnrealEngine
241+
run: |
242+
243+
apt-get update
244+
apt-get install -y acl curl ca-certificates
245+
299246
- name: Run Unreal Engine tests
300247
working-directory: sdks/unreal
301248
env:

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ members = [
4646
"sdks/rust/tests/test-client",
4747
"sdks/rust/tests/test-counter",
4848
"sdks/rust/tests/connect_disconnect_client",
49+
"tools/ci",
4950
"tools/upgrade-version",
5051
"tools/license-check",
5152
"crates/bindings-typescript/test-app/server",

‎tools/ci/Cargo.toml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[package]
2+
name = "ci"
3+
version = "0.1.0"
4+
edition.workspace = true
5+
6+
[dependencies]
7+
log.workspace = true
8+
anyhow.workspace = true
9+
chrono = { workspace = true, features=["clock"] }
10+
clap.workspace = true
11+
regex.workspace = true
12+
duct.workspace = true

‎tools/ci/build.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#[allow(clippy::disallowed_macros)]
2+
fn main() {
3+
println!("cargo:rustc-env=TARGET={}", std::env::var("TARGET").unwrap_or_default());
4+
}

‎tools/ci/src/main.rs‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
use anyhow::Result;
2+
use clap::{CommandFactory, Parser, Subcommand};
3+
use duct::cmd;
4+
use std::collections::HashMap;
5+
use std::env;
6+
7+
#[derive(Parser)]
8+
#[command(
9+
name = "spacetimedb-ci",
10+
about = "SpacetimeDB CI tasks",
11+
subcommand_required = false,
12+
arg_required_else_help = false
13+
)]
14+
struct Cli {
15+
#[command(subcommand)]
16+
cmd: Option<CiCmd>,
17+
18+
#[arg(long)]
19+
skip: Vec<String>,
20+
}
21+
22+
#[derive(Subcommand)]
23+
enum CiCmd {
24+
Test,
25+
Lints,
26+
WasmBindings,
27+
Smoketests {
28+
#[arg(trailing_var_arg = true)]
29+
args: Vec<String>,
30+
},
31+
UpdateFlow {
32+
#[arg(long)]
33+
target: Option<String>,
34+
#[arg(long, default_value = "true")]
35+
github_token_auth: bool,
36+
},
37+
UnrealTests,
38+
CliDocs,
39+
}
40+
41+
macro_rules! run {
42+
($cmdline:expr) => {
43+
run_command($cmdline, &Vec::new())
44+
};
45+
($cmdline:expr, $envs:expr) => {
46+
run_command($cmdline, $envs)
47+
};
48+
}
49+
50+
fn run_all_clap_subcommands(skips: &[String]) -> Result<()> {
51+
let subcmds = Cli::command()
52+
.get_subcommands()
53+
.map(|sc| sc.get_name().to_string())
54+
.collect::<Vec<_>>();
55+
56+
for subcmd in subcmds {
57+
if skips.contains(&subcmd) {
58+
log::info!("skipping {subcmd} as requested");
59+
continue;
60+
}
61+
log::info!("executing cargo ci {subcmd}");
62+
run!(&format!("cargo ci {subcmd}"))?;
63+
}
64+
65+
Ok(())
66+
}
67+
68+
fn run_command(cmdline: &str, additional_env: &[(&str, &str)]) -> Result<()> {
69+
let mut env = env::vars().collect::<HashMap<_, _>>();
70+
env.extend(additional_env.iter().map(|(k, v)| (k.to_string(), v.to_string())));
71+
log::debug!("$ {cmdline}");
72+
let status = cmd!("bash", "-lc", cmdline).full_env(env).run()?;
73+
if !status.status.success() {
74+
let e = anyhow::anyhow!("command failed: {cmdline}");
75+
log::error!("{e}");
76+
return Err(e);
77+
}
78+
Ok(())
79+
}
80+
81+
fn main() -> Result<()> {
82+
let cli = Cli::parse();
83+
84+
match cli.cmd {
85+
Some(CiCmd::Test) => {
86+
run!("sudo mkdir -p /stdb && sudo chmod 777 /stdb")?;
87+
run!("cargo test --all -- --skip unreal")?;
88+
run!("bash tools/check-diff.sh")?;
89+
run!("cargo run -p spacetimedb-codegen --example regen-csharp-moduledef && bash tools/check-diff.sh crates/bindings-csharp")?;
90+
run!("(cd crates/bindings-csharp && dotnet test -warnaserror)")?;
91+
}
92+
93+
Some(CiCmd::Lints) => {
94+
run!("cargo fmt --all -- --check")?;
95+
run!("cargo clippy --all --tests --benches -- -D warnings")?;
96+
run!("(cd crates/bindings-csharp && dotnet tool restore && dotnet csharpier --check .)")?;
97+
run!("cd crates/bindings && cargo doc", &[("RUSTDOCFLAGS", "hey")])?;
98+
}
99+
100+
Some(CiCmd::WasmBindings) => {
101+
run!("cargo test -p spacetimedb-codegen")?;
102+
run!("cargo update")?;
103+
run!("cargo run -p spacetimedb-cli -- build --project-path modules/module-test")?;
104+
}
105+
106+
Some(CiCmd::Smoketests { args }) => {
107+
// Note: clear_database and replication only work in private
108+
run!(&format!("python -m smoketests {} -x clear_database replication", args.join(" ")))?;
109+
}
110+
111+
Some(CiCmd::UpdateFlow {
112+
target,
113+
github_token_auth,
114+
}) => {
115+
let target = target.unwrap_or_else(|| env!("TARGET").to_string());
116+
let github_token_auth_flag = if github_token_auth {
117+
"--features github-token-auth "
118+
} else {
119+
""
120+
};
121+
122+
run!(&format!("echo 'checking update flow for target: {target}'"))?;
123+
run!(&format!(
124+
"cargo build {github_token_auth_flag}--target {target} -p spacetimedb-update"
125+
))?;
126+
run!(&format!(
127+
r#"
128+
ROOT_DIR="$(mktemp -d)"
129+
# NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`.
130+
# My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it
131+
# happens very frequently on the `macos-runner`, but we haven't seen it on any others).
132+
cargo run {github_token_auth_flag}--target {target} -p spacetimedb-update -- self-install --root-dir="${{ROOT_DIR}}" --yes
133+
"${{ROOT_DIR}}"/spacetime --root-dir="${{ROOT_DIR}}" help
134+
"#
135+
))?;
136+
}
137+
138+
Some(CiCmd::UnrealTests) => {
139+
run!("for p in \"$GITHUB_WORKSPACE\" \"${RUNNER_TEMP:-/__t}\" \"${RUNNER_TOOL_CACHE:-/__t}\"; do [ -d \"$p\" ] && setfacl -R -m u:ue4:rwX -m d:u:ue4:rwX \"$p\" || true; done")?;
140+
141+
run!("export CARGO_HOME=\"${RUNNER_TOOL_CACHE:-/__t}/cargo\"")?;
142+
run!("export RUSTUP_HOME=\"${RUNNER_TOOL_CACHE:-/__t}/rustup\"")?;
143+
run!("mkdir -p \"$CARGO_HOME\" \"$RUSTUP_HOME\"")?;
144+
145+
run!("chmod a+rx \"$UE_ROOT_PATH\" \"$UE_ROOT_PATH/Engine\" \"$UE_ROOT_PATH/Engine/Build\" \"$UE_ROOT_PATH/Engine/Build/BatchFiles/Linux\" || true")?;
146+
run!("chmod a+rx \"$UE_ROOT_PATH/Engine/Build/BatchFiles/Linux/Build.sh\" || true")?;
147+
148+
run!("sudo -E -H -u ue4 env HOME=/home/ue4 CARGO_HOME=\"$CARGO_HOME\" RUSTUP_HOME=\"$RUSTUP_HOME\" PATH=\"$CARGO_HOME/bin:$PATH\" bash -lc 'set -euxo pipefail; if ! command -v cargo >/dev/null 2>&1; then curl -sSf https://sh.rustup.rs | sh -s -- -y; fi; rustup show >/dev/null; git config --global --add safe.directory \"$GITHUB_WORKSPACE\" || true; cd \"$GITHUB_WORKSPACE/sdks/unreal\"; cargo --version; cargo test'")?;
149+
}
150+
151+
Some(CiCmd::CliDocs) => {
152+
run!("pnpm install --recursive")?;
153+
run!("cargo run --features markdown-docs -p spacetimedb-cli > docs/docs/cli-reference.md")?;
154+
run!("pnpm format")?;
155+
run!("git status")?;
156+
run!(
157+
r#"
158+
if git diff --exit-code HEAD; then
159+
echo "No docs changes detected"
160+
else
161+
echo "It looks like the CLI docs have changed:"
162+
exit 1
163+
fi
164+
"#
165+
)?;
166+
}
167+
168+
None => run_all_clap_subcommands(&cli.skip)?,
169+
}
170+
171+
Ok(())
172+
}

0 commit comments

Comments
 (0)