diff --git a/Cargo.lock b/Cargo.lock index d219910..aa1933f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -523,6 +523,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "find-msvc-tools" version = "0.1.7" @@ -983,7 +989,7 @@ dependencies = [ [[package]] name = "iii-tools" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "clap", @@ -1149,6 +1155,12 @@ dependencies = [ "libc", ] +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + [[package]] name = "litemap" version = "0.8.1" @@ -1220,7 +1232,7 @@ dependencies = [ [[package]] name = "motia-tools" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "clap", @@ -1635,6 +1647,19 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +[[package]] +name = "rustix" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.36" @@ -1733,7 +1758,7 @@ dependencies = [ [[package]] name = "scaffolder-core" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "clap", @@ -1748,10 +1773,9 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "sha2", + "tempfile", "thiserror 2.0.17", "tokio", - "toml", "url", "uuid", "walkdir", @@ -1846,15 +1870,6 @@ dependencies = [ "zmij", ] -[[package]] -name = "serde_spanned" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" -dependencies = [ - "serde", -] - [[package]] name = "serde_yaml" version = "0.9.34+deprecated" @@ -2010,6 +2025,19 @@ dependencies = [ "libc", ] +[[package]] +name = "tempfile" +version = "3.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1" +dependencies = [ + "fastrand", + "getrandom 0.4.1", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "textwrap" version = "0.16.2" @@ -2157,47 +2185,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "toml" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.22.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" -dependencies = [ - "indexmap", - "serde", - "serde_spanned", - "toml_datetime", - "toml_write", - "winnow", -] - -[[package]] -name = "toml_write" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" - [[package]] name = "tower" version = "0.5.3" @@ -2802,15 +2789,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" -[[package]] -name = "winnow" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" -dependencies = [ - "memchr", -] - [[package]] name = "wiremock" version = "0.6.5" diff --git a/Cargo.toml b/Cargo.toml index 84aeb81..4e600d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,12 +54,11 @@ zip = "8.1" # Open URLs in browser open = "5" -# Telemetry / project identity (shared with iii engine) +# Telemetry / project identity (reads engine's telemetry.yaml) dirs = "6" -toml = "0.8" uuid = { version = "1", features = ["v4"] } serde_json = "1" -sha2 = "0.10" +tempfile = "3" wiremock = "0.6" # Local crates diff --git a/crates/scaffolder-core/Cargo.toml b/crates/scaffolder-core/Cargo.toml index 6609215..bb7d050 100644 --- a/crates/scaffolder-core/Cargo.toml +++ b/crates/scaffolder-core/Cargo.toml @@ -25,10 +25,8 @@ zip = { workspace = true } open = { workspace = true } colored = { workspace = true } dirs = { workspace = true } -toml = { workspace = true } uuid = { workspace = true } serde_json = { workspace = true } -sha2 = { workspace = true } # Optional: only for tui feature cliclack = { workspace = true, optional = true } @@ -39,3 +37,4 @@ ctrlc = { workspace = true, optional = true } url = { workspace = true } wiremock = { workspace = true } tokio = { workspace = true, features = ["full", "macros"] } +tempfile = { workspace = true } diff --git a/crates/scaffolder-core/src/telemetry.rs b/crates/scaffolder-core/src/telemetry.rs index ac4d18d..d3030f9 100644 --- a/crates/scaffolder-core/src/telemetry.rs +++ b/crates/scaffolder-core/src/telemetry.rs @@ -1,16 +1,15 @@ -use std::collections::BTreeMap; use std::path::Path; use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result}; -use serde::Serialize; -use sha2::{Digest, Sha256}; +use serde::{Deserialize, Serialize}; use tokio::fs; use crate::runtime::check::Language; const API_KEY: &str = "a7182ac460dde671c8f2e1318b517228"; const AMPLITUDE_ENDPOINT: &str = "https://api2.amplitude.com/2/httpapi"; +const TELEMETRY_SCHEMA_VERSION: u8 = 2; #[cfg(test)] fn resolve_endpoint() -> String { @@ -22,72 +21,36 @@ fn resolve_endpoint() -> String { AMPLITUDE_ENDPOINT.to_string() } -type TomlSections = BTreeMap>; - -fn iii_dir() -> std::path::PathBuf { +fn telemetry_yaml_path() -> std::path::PathBuf { dirs::home_dir() .unwrap_or_else(std::env::temp_dir) .join(".iii") + .join("telemetry.yaml") } -fn telemetry_toml_path() -> std::path::PathBuf { - iii_dir().join("telemetry.toml") -} - -fn write_atomic(path: &Path, content: &str) { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent).ok(); - } - let tmp = path.with_extension("tmp"); - if std::fs::write(&tmp, content).is_ok() { - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let perms = std::fs::Permissions::from_mode(0o600); - std::fs::set_permissions(&tmp, perms).ok(); - } - std::fs::rename(&tmp, path).ok(); - } +/// Only the fields we need to read from the engine's `~/.iii/telemetry.yaml`. +#[derive(Deserialize)] +struct TelemetryYaml { + version: Option, + #[serde(default)] + identity: IdentitySection, } -fn read_toml_key(section: &str, key: &str) -> Option { - let contents = std::fs::read_to_string(telemetry_toml_path()).ok()?; - let sections: TomlSections = toml::from_str(&contents).ok()?; - sections - .get(section)? - .get(key) - .filter(|v| !v.is_empty()) - .cloned() +#[derive(Deserialize, Default)] +struct IdentitySection { + #[serde(default)] + device_id: Option, } -fn set_toml_key(section: &str, key: &str, value: &str) { - let path = telemetry_toml_path(); - let contents = std::fs::read_to_string(&path).unwrap_or_default(); - let mut sections: TomlSections = toml::from_str(&contents).unwrap_or_default(); - sections - .entry(section.to_string()) - .or_default() - .insert(key.to_string(), value.to_string()); - if let Ok(serialized) = toml::to_string(§ions) { - write_atomic(&path, &serialized); - } -} - -pub fn get_or_create_telemetry_id() -> String { - if let Some(id) = read_toml_key("identity", "id") { - return id; - } - let legacy_path = iii_dir().join("telemetry_id"); - if let Ok(raw) = std::fs::read_to_string(&legacy_path) { - let id = raw.trim().to_string(); - if !id.is_empty() { - set_toml_key("identity", "id", &id); - return id; - } +/// Reads the device_id from the engine-managed `~/.iii/telemetry.yaml`. +/// Returns `None` if the file is missing, malformed, or has no device_id. +fn read_device_id() -> Option { + let contents = std::fs::read_to_string(telemetry_yaml_path()).ok()?; + let state: TelemetryYaml = serde_yaml::from_str(&contents).ok()?; + if state.version != Some(TELEMETRY_SCHEMA_VERSION) { + return None; } - let id = format!("auto-{}", uuid::Uuid::new_v4()); - set_toml_key("identity", "id", &id); - id + state.identity.device_id.filter(|id| !id.is_empty()) } pub fn is_telemetry_disabled() -> bool { @@ -116,14 +79,6 @@ pub fn is_telemetry_disabled() -> bool { CI_VARS.iter().any(|v| std::env::var(v).is_ok()) } -fn detect_machine_id() -> String { - let hostname = std::env::var("HOSTNAME").unwrap_or_else(|_| "unknown".to_string()); - let mut hasher = Sha256::new(); - hasher.update(hostname.as_bytes()); - let result = hasher.finalize(); - result[..8].iter().map(|b| format!("{:02x}", b)).collect() -} - fn detect_is_container() -> bool { if std::env::var("III_CONTAINER").is_ok() { return true; @@ -153,13 +108,13 @@ fn detect_install_method() -> &'static str { "manual" } -fn build_user_properties(tools_version: &str) -> serde_json::Value { +fn build_user_properties(tools_version: &str, device_id: &str) -> serde_json::Value { serde_json::json!({ "environment.os": std::env::consts::OS, "environment.arch": std::env::consts::ARCH, "environment.cpu_cores": std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1), "environment.timezone": std::env::var("TZ").unwrap_or_else(|_| "Unknown".to_string()), - "environment.machine_id": detect_machine_id(), + "environment.machine_id": device_id, "environment.is_container": detect_is_container(), "env": std::env::var("III_ENV").unwrap_or_else(|_| "unknown".to_string()), "install_method": detect_install_method(), @@ -178,6 +133,7 @@ fn millis_epoch() -> i64 { #[derive(Serialize)] struct AmplitudeEvent { device_id: String, + #[serde(skip_serializing_if = "Option::is_none")] user_id: Option, event_type: String, event_properties: serde_json::Value, @@ -196,6 +152,46 @@ struct AmplitudePayload<'a> { events: Vec, } +fn build_amplitude_client() -> Option { + reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(5)) + .build() + .ok() +} + +async fn post_amplitude(endpoint: &str, payload: &AmplitudePayload<'_>) { + let Some(client) = build_amplitude_client() else { + return; + }; + let _ = client.post(endpoint).json(payload).send().await; +} + +/// Sends a lightweight failure event when telemetry.yaml is missing. +/// Uses a throwaway device_id since we have no identity to attach to. +async fn send_telemetry_failed(endpoint: &str, platform: &str, tools_version: &str) { + let event = AmplitudeEvent { + device_id: format!("unknown-{}", uuid::Uuid::new_v4()), + user_id: None, + event_type: "iii_tools_telemetry_failed".to_string(), + event_properties: serde_json::json!({ + "reason": "telemetry_yaml_missing", + "path": telemetry_yaml_path().to_string_lossy(), + }), + user_properties: None, + platform: platform.to_string(), + os_name: std::env::consts::OS.to_string(), + app_version: tools_version.to_string(), + time: millis_epoch(), + insert_id: uuid::Uuid::new_v4().to_string(), + ip: Some("$remote".to_string()), + }; + let payload = AmplitudePayload { + api_key: API_KEY, + events: vec![event], + }; + post_amplitude(endpoint, &payload).await; +} + async fn send_amplitude_to( endpoint: &str, event_type: &str, @@ -203,13 +199,16 @@ async fn send_amplitude_to( tools_version: &str, event_properties: serde_json::Value, ) { - let telemetry_id = get_or_create_telemetry_id(); + let Some(device_id) = read_device_id() else { + send_telemetry_failed(endpoint, platform, tools_version).await; + return; + }; let event = AmplitudeEvent { - device_id: telemetry_id.clone(), - user_id: Some(telemetry_id), + device_id: device_id.clone(), + user_id: None, event_type: event_type.to_string(), event_properties, - user_properties: Some(build_user_properties(tools_version)), + user_properties: Some(build_user_properties(tools_version, &device_id)), platform: platform.to_string(), os_name: std::env::consts::OS.to_string(), app_version: tools_version.to_string(), @@ -221,14 +220,7 @@ async fn send_amplitude_to( api_key: API_KEY, events: vec![event], }; - let client = match reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(5)) - .build() - { - Ok(c) => c, - Err(_) => return, - }; - let _ = client.post(endpoint).json(&payload).send().await; + post_amplitude(endpoint, &payload).await; } async fn send_amplitude( @@ -352,83 +344,52 @@ mod tests { assert!(s.contains("project_name=my-app")); } - #[tokio::test] - async fn sends_project_created_event() { - let mock_server = MockServer::start().await; - - Mock::given(method("POST")) - .and(path("/2/httpapi")) - .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({"code": 200}))) - .expect(1) - .mount(&mock_server) - .await; - - let endpoint = format!("{}/2/httpapi", mock_server.uri()); - - send_amplitude_to( - &endpoint, - "project_created", - "motia-tools", - "0.3.0", - serde_json::json!({ - "project_id": "test-id", - "project_name": "my-project", - "template": "quickstart", - "product": "motia", - }), - ) - .await; - - // Mock expectation of exactly 1 call is verified on drop + #[test] + fn read_device_id_returns_none_when_no_file() { + // Unless the engine has written telemetry.yaml, this may be None. + // We just verify it doesn't panic. + let _ = read_device_id(); } #[tokio::test] - async fn sends_project_initialized_event() { + async fn sends_failed_event_when_yaml_missing() { let mock_server = MockServer::start().await; Mock::given(method("POST")) .and(path("/2/httpapi")) - .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({"code": 200}))) + .respond_with(ResponseTemplate::new(200)) .expect(1) .mount(&mock_server) .await; let endpoint = format!("{}/2/httpapi", mock_server.uri()); + // Point HOME at a temp dir so telemetry.yaml won't exist + let tmp = tempfile::tempdir().unwrap(); + unsafe { std::env::set_var("HOME", tmp.path()); } + send_amplitude_to( &endpoint, - "project_initialized", - "motia-tools", + "project_created", + "iii-tools", "0.3.0", - serde_json::json!({ - "project_id": "test-id", - "project_name": "my-project", - "template": "quickstart", - "product": "motia", - }), + serde_json::json!({"project_id": "test-id"}), ) .await; - } - #[tokio::test] - async fn does_not_send_quickstart_event() { - let mock_server = MockServer::start().await; + unsafe { std::env::remove_var("HOME"); } - Mock::given(method("POST")) - .and(path("/2/httpapi")) - .respond_with(ResponseTemplate::new(200)) - .expect(0) - .mount(&mock_server) - .await; - - // We intentionally do NOT call send_amplitude_to with "quickstart" - // because the codebase never sends it — this test documents that fact. + let requests: Vec = mock_server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 1); - // Verified: 0 calls received on drop + let body: serde_json::Value = serde_json::from_slice(&requests[0].body).unwrap(); + let event = &body["events"][0]; + assert_eq!(event["event_type"], "iii_tools_telemetry_failed"); + assert_eq!(event["event_properties"]["reason"], "telemetry_yaml_missing"); } #[tokio::test] - async fn payload_contains_correct_event_type_and_api_key() { + async fn sends_normal_event_when_yaml_exists() { let mock_server = MockServer::start().await; Mock::given(method("POST")) @@ -438,30 +399,46 @@ mod tests { .mount(&mock_server) .await; + let tmp = tempfile::tempdir().unwrap(); + let iii_dir = tmp.path().join(".iii"); + std::fs::create_dir_all(&iii_dir).unwrap(); + std::fs::write( + iii_dir.join("telemetry.yaml"), + "version: 2\nidentity:\n device_id: test-device-abc\n", + ) + .unwrap(); + + unsafe { std::env::set_var("HOME", tmp.path()); } + let endpoint = format!("{}/2/httpapi", mock_server.uri()); send_amplitude_to( &endpoint, "project_created", - "motia-tools", + "iii-tools", "0.3.0", serde_json::json!({ "project_id": "test-id", "project_name": "my-project", "template": "quickstart", - "product": "motia", + "product": "iii", }), ) .await; + unsafe { std::env::remove_var("HOME"); } + let requests: Vec = mock_server.received_requests().await.unwrap(); assert_eq!(requests.len(), 1); let body: serde_json::Value = serde_json::from_slice(&requests[0].body).unwrap(); - assert_eq!(body["api_key"], API_KEY); - assert_eq!(body["events"][0]["event_type"], "project_created"); - assert_eq!(body["events"][0]["platform"], "motia-tools"); - assert_eq!(body["events"][0]["event_properties"]["project_id"], "test-id"); - assert_eq!(body["events"][0]["event_properties"]["template"], "quickstart"); + let event = &body["events"][0]; + assert_eq!(event["event_type"], "project_created"); + assert_eq!(event["device_id"], "test-device-abc"); + assert!( + event.get("user_id").is_none() || event["user_id"].is_null(), + "user_id should not be sent" + ); + assert_eq!(event["event_properties"]["project_id"], "test-id"); } } diff --git a/templates/iii/default-iii-config.yaml b/templates/iii/default-iii-config.yaml index ac2deb5..e9a7e30 100644 --- a/templates/iii/default-iii-config.yaml +++ b/templates/iii/default-iii-config.yaml @@ -1,4 +1,8 @@ modules: + - class: modules::worker::WorkerModule + config: + port: 49134 # WebSocket protocol port — workers connect here + - class: modules::stream::StreamModule config: port: ${STREAM_PORT:3112} diff --git a/templates/iii/quickstart.zip b/templates/iii/quickstart.zip index 964c8d6..f3a32e6 100644 Binary files a/templates/iii/quickstart.zip and b/templates/iii/quickstart.zip differ diff --git a/templates/motia/quickstart.zip b/templates/motia/quickstart.zip index 7308a78..d900870 100644 Binary files a/templates/motia/quickstart.zip and b/templates/motia/quickstart.zip differ