From 57355cc906638dfd12132d18ab944036e992dfc9 Mon Sep 17 00:00:00 2001 From: bitterpanda Date: Tue, 21 Apr 2026 14:19:32 +0200 Subject: [PATCH 01/19] add golang to mock server --- .../endpoint_protection_callbacks.rs | 53 ++++++++++++++++++- .../src/client/mock_server/malware_list.rs | 22 ++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/proxy-bin-l7/src/client/mock_server/endpoint_protection_callbacks.rs b/proxy-bin-l7/src/client/mock_server/endpoint_protection_callbacks.rs index c5e6d37c..cbc7b66f 100644 --- a/proxy-bin-l7/src/client/mock_server/endpoint_protection_callbacks.rs +++ b/proxy-bin-l7/src/client/mock_server/endpoint_protection_callbacks.rs @@ -388,6 +388,56 @@ async fn fetch_permissions(req: Request) -> impl IntoResponse { "rejected_packages": [] } }), + _ => default_ecosystem_policy.clone(), + }; + + let golang_policy = match token { + "policy-block-golang" => json!({ + "block_all_installs": true, + "request_installs": false, + "minimum_allowed_age_timestamp": null, + "exceptions": { + "allowed_packages": [], + "rejected_packages": [] + } + }), + "policy-allow-gorilla-mux-golang" => json!({ + "block_all_installs": false, + "request_installs": false, + "minimum_allowed_age_timestamp": null, + "exceptions": { + "allowed_packages": ["github.com/gorilla/mux"], // listed as malware in mock list + "rejected_packages": [] + } + }), + "policy-reject-gin-golang" => json!({ + "block_all_installs": false, + "request_installs": false, + "minimum_allowed_age_timestamp": null, + "exceptions": { + "allowed_packages": [], + "rejected_packages": ["github.com/gin-gonic/gin"] + } + }), + "policy-request-installs-golang" => json!({ + "block_all_installs": false, + "request_installs": true, + "minimum_allowed_age_timestamp": null, + "exceptions": { + "allowed_packages": [], + "rejected_packages": [] + } + }), + "policy-bypass-new-package-golang" => json!({ + "block_all_installs": false, + "request_installs": false, + // Cutoff set to far future (year ~2286): released_on (year ~2255) <= cutoff → not blocked + "minimum_allowed_age_timestamp": 9_999_999_999_i64, + "exceptions": { + "allowed_packages": [], + "rejected_packages": [] + } + }), _ => default_ecosystem_policy, }; @@ -404,7 +454,8 @@ async fn fetch_permissions(req: Request) -> impl IntoResponse { "nuget": nuget_policy, "chrome": chrome_policy, "open_vsx": open_vsx_policy, - "skills_sh": skills_sh_policy + "skills_sh": skills_sh_policy, + "golang": golang_policy } })) .into_response() diff --git a/proxy-bin-l7/src/client/mock_server/malware_list.rs b/proxy-bin-l7/src/client/mock_server/malware_list.rs index 2ac72c32..901fc012 100644 --- a/proxy-bin-l7/src/client/mock_server/malware_list.rs +++ b/proxy-bin-l7/src/client/mock_server/malware_list.rs @@ -27,6 +27,7 @@ pub(super) fn web_svc() -> impl Service impl Service impl IntoResponse { reason: Reason::Malware, }]) } + +pub const FRESH_GOLANG_MODULE_NAME: &str = "github.com/fresh-org/fresh-module"; +pub const FRESH_GOLANG_MODULE_VERSION: &str = "1.0.0"; +async fn released_golang() -> impl IntoResponse { + Json([ReleasedPackageData { + package_name: FRESH_GOLANG_MODULE_NAME.to_owned(), + version: PackageVersion::Semver( + PragmaticSemver::parse(FRESH_GOLANG_MODULE_VERSION).unwrap(), + ), + released_on: 9_000_000_000, + }]) +} + +async fn malware_golang() -> impl IntoResponse { + Json([ListDataEntry { + package_name: "github.com/gorilla/mux".to_owned(), + version: PackageVersion::Semver(PragmaticSemver::new_semver(1, 8, 0)), + reason: Reason::Malware, + }]) +} From 9aa42b833b0f22e76ed5c6b6aa364b9d94d13e92 Mon Sep 17 00:00:00 2001 From: bitterpanda Date: Tue, 21 Apr 2026 14:19:59 +0200 Subject: [PATCH 02/19] Add first version of golang firewall rule --- .../src/http/firewall/rule/golang/mod.rs | 285 ++++++++++++++++++ proxy-lib/src/http/firewall/rule/mod.rs | 1 + 2 files changed, 286 insertions(+) create mode 100644 proxy-lib/src/http/firewall/rule/golang/mod.rs diff --git a/proxy-lib/src/http/firewall/rule/golang/mod.rs b/proxy-lib/src/http/firewall/rule/golang/mod.rs new file mode 100644 index 00000000..b55c2e8c --- /dev/null +++ b/proxy-lib/src/http/firewall/rule/golang/mod.rs @@ -0,0 +1,285 @@ +use std::{fmt, str::FromStr}; + +use rama::utils::time::now_unix_ms; +use rama::{ + Service, + error::{BoxError, ErrorContext as _, extra::OpaqueError}, + graceful::ShutdownGuard, + http::{Request, Response, Uri}, + net::address::Domain, + telemetry::tracing, + utils::str::arcstr::{ArcStr, arcstr}, +}; + +use crate::{ + endpoint_protection::{PackagePolicyDecision, PolicyEvaluator, RemoteEndpointConfig}, + http::firewall::{ + domain_matcher::DomainMatcher, + events::{Artifact, BlockReason}, + }, + package::{ + malware_list::{LowerCaseEntryFormatter, RemoteMalwareList}, + released_packages_list::{LowerCaseReleasedPackageFormatter, RemoteReleasedPackagesList}, + version::{PackageVersion, PragmaticSemver}, + }, + storage::SyncCompactDataStorage, +}; + +#[cfg(feature = "pac")] +use crate::http::firewall::pac::PacScriptGenerator; + +use super::{BlockedRequest, RequestAction, Rule}; + +#[cfg(test)] +mod tests; + +pub(in crate::http::firewall) struct RuleGolang { + target_domains: DomainMatcher, + remote_malware_list: RemoteMalwareList, + remote_released_packages_list: RemoteReleasedPackagesList, + remote_endpoint_config: Option, + policy_evaluator: Option, +} + +impl RuleGolang { + pub(in crate::http::firewall) async fn try_new( + guard: ShutdownGuard, + remote_malware_list_https_client: C, + sync_storage: SyncCompactDataStorage, + policy_evaluator: Option, + remote_endpoint_config: Option, + ) -> Result + where + C: Service + Clone, + { + let remote_malware_list = RemoteMalwareList::try_new( + guard.clone(), + Uri::from_static("https://malware-list.aikido.dev/malware_golang.json"), + sync_storage.clone(), + remote_malware_list_https_client.clone(), + LowerCaseEntryFormatter, + ) + .await + .context("create remote malware list for golang block rule")?; + + let remote_released_packages_list = RemoteReleasedPackagesList::try_new( + guard, + Uri::from_static("https://malware-list.aikido.dev/releases/golang.json"), + sync_storage, + remote_malware_list_https_client, + LowerCaseReleasedPackageFormatter, + ) + .await + .context("create remote released packages list for golang block rule")?; + + Ok(Self { + target_domains: ["proxy.golang.org"].into_iter().collect(), + remote_malware_list, + remote_released_packages_list, + remote_endpoint_config, + policy_evaluator, + }) + } +} + +impl fmt::Debug for RuleGolang { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RuleGolang").finish() + } +} + +impl Rule for RuleGolang { + #[inline(always)] + fn match_domain(&self, domain: &Domain) -> bool { + self.target_domains.is_match(domain) + } + + #[cfg(feature = "pac")] + #[inline(always)] + fn collect_pac_domains(&self, generator: &mut PacScriptGenerator) { + for domain in self.target_domains.iter() { + generator.write_domain(&domain); + } + } + + async fn evaluate_request(&self, req: Request) -> Result { + let path = req.uri().path(); + if !is_zip_download(path) { + return Ok(RequestAction::Allow(req)); + } + self.evaluate_zip_request(req).await + } +} + +impl RuleGolang { + const DEFAULT_MIN_PACKAGE_AGE_SECS: i64 = 48 * 3600; + + fn get_package_age_cutoff_secs(&self) -> i64 { + let maybe_ts = self.remote_endpoint_config.as_ref().and_then(|c| { + c.get_ecosystem_config("golang") + .config() + .and_then(|cfg| cfg.minimum_allowed_age_timestamp) + }); + if let Some(ts_secs) = maybe_ts { + return ts_secs; + } + (now_unix_ms() / 1000) - Self::DEFAULT_MIN_PACKAGE_AGE_SECS + } + + fn blocked_artifact(package: &GoPackage) -> Artifact { + Artifact { + product: arcstr!("golang"), + identifier: ArcStr::from(package.fully_qualified_name.as_str()), + display_name: None, + version: Some(PackageVersion::Semver(package.version.clone())), + } + } + + fn is_package_listed_as_malware(&self, package: &GoPackage) -> bool { + self.remote_malware_list.has_entries_with_version( + &package.fully_qualified_name, + PackageVersion::Semver(package.version.clone()), + ) + } + + async fn evaluate_zip_request(&self, req: Request) -> Result { + let path = req.uri().path().trim_start_matches('/'); + + let Some(package) = parse_package_from_path(path) else { + tracing::debug!("Golang url: {path} is not a module zip download: passthrough"); + return Ok(RequestAction::Allow(req)); + }; + + tracing::debug!( + http.url.path = %path, + package.name = %package.fully_qualified_name, + package.version = %package.version, + "Go module zip download request" + ); + + if let Some(policy_evaluator) = self.policy_evaluator.as_ref() { + let decision = + policy_evaluator.evaluate_package_install("golang", &package.fully_qualified_name); + + match decision { + PackagePolicyDecision::Allow => { + return Ok(RequestAction::Allow(req)); + } + PackagePolicyDecision::Defer => {} + decision => { + return Ok(RequestAction::Block(BlockedRequest::blocked( + req, + Self::blocked_artifact(&package), + super::block_reason_for(decision), + ))); + } + } + } + + if self.is_package_listed_as_malware(&package) { + tracing::warn!("Blocked malware from {}", package.fully_qualified_name); + return Ok(RequestAction::Block(BlockedRequest::blocked( + req, + Self::blocked_artifact(&package), + BlockReason::Malware, + ))); + } + + let cutoff_secs = self.get_package_age_cutoff_secs(); + if self.remote_released_packages_list.is_recently_released( + &package.fully_qualified_name, + Some(&PackageVersion::Semver(package.version.clone())), + cutoff_secs, + ) { + tracing::info!( + http.url.path = %path, + package = %package.fully_qualified_name, + "blocked golang zip download: package released too recently" + ); + return Ok(RequestAction::Block(BlockedRequest::blocked( + req, + Self::blocked_artifact(&package), + BlockReason::NewPackage, + ))); + } + + tracing::debug!("Golang url: {path} does not contain malware: passthrough"); + Ok(RequestAction::Allow(req)) + } +} + +pub(super) struct GoPackage { + pub(super) fully_qualified_name: String, + pub(super) version: PragmaticSemver, +} + +fn is_zip_download(path: &str) -> bool { + path.ends_with(".zip") && path.contains("/@v/") +} + +/// Parses a Go module proxy zip URL path into a normalized module name and version. +/// +/// Expected path shape: `/{module_path}/@v/{version}.zip` +/// +/// Go's module proxy encodes uppercase letters as `!` + lowercase (e.g., `AikidoSec` → +/// `!aikido!sec`, percent-encoded as `%21aikido%21sec`). We lowercase the whole path +/// to normalise for malware-list lookup (which uses `LowerCaseEntryFormatter`). +pub(super) fn parse_package_from_path(path: &str) -> Option { + let path = path.trim_matches('/'); + + let (module_path_raw, rest) = path.split_once("/@v/")?; + let version_raw = rest.strip_suffix(".zip")?; + + if module_path_raw.is_empty() || version_raw.is_empty() { + return None; + } + + // Percent-decode the module path (handles %21 → !) + let module_path_decoded = percent_decode(module_path_raw); + // Lowercase for consistent malware-list lookup + let module_name = module_path_decoded.to_ascii_lowercase(); + + // Go versions always carry a 'v' prefix in the proxy URL; strip it for semver parsing + let version_str = version_raw.strip_prefix('v').unwrap_or(version_raw); + let version = PragmaticSemver::from_str(version_str) + .inspect_err(|err| { + tracing::debug!( + "failed to parse golang module ({module_name}) version (raw = {version_raw}): err = {err}" + ); + }) + .ok()?; + + Some(GoPackage { + fully_qualified_name: module_name, + version, + }) +} + +/// Decodes percent-encoded characters in a URL path segment. +/// Only handles the ASCII subset relevant to Go module paths (`%21` → `!`, etc.). +fn percent_decode(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let bytes = s.as_bytes(); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' && i + 2 < bytes.len() { + if let (Some(h), Some(l)) = (hex_val(bytes[i + 1]), hex_val(bytes[i + 2])) { + out.push((h << 4 | l) as char); + i += 3; + continue; + } + } + out.push(bytes[i] as char); + i += 1; + } + out +} + +fn hex_val(b: u8) -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} diff --git a/proxy-lib/src/http/firewall/rule/mod.rs b/proxy-lib/src/http/firewall/rule/mod.rs index e00a2332..ecbbfd39 100644 --- a/proxy-lib/src/http/firewall/rule/mod.rs +++ b/proxy-lib/src/http/firewall/rule/mod.rs @@ -50,6 +50,7 @@ pub(crate) fn block_reason_for(decision: PackagePolicyDecision) -> BlockReason { pub use super::pac::PacScriptGenerator; pub mod chrome; +pub mod golang; pub mod hijack; pub mod maven; pub mod npm; From d36839198d3b932e519b0f7e57e6666c1ff91ea2 Mon Sep 17 00:00:00 2001 From: bitterpanda Date: Tue, 21 Apr 2026 14:20:18 +0200 Subject: [PATCH 03/19] Enable golang firewall rule in firewall/mod.rs --- proxy-lib/src/http/firewall/mod.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/proxy-lib/src/http/firewall/mod.rs b/proxy-lib/src/http/firewall/mod.rs index 927a8847..d7a5a065 100644 --- a/proxy-lib/src/http/firewall/mod.rs +++ b/proxy-lib/src/http/firewall/mod.rs @@ -262,6 +262,16 @@ impl Firewall { .await .context("create block rule: open vsx")? .into_dyn(), + self::rule::golang::RuleGolang::try_new( + guard.clone(), + layered_client.clone(), + data.clone(), + policy_evaluator.clone(), + remote_endpoint_config.clone(), + ) + .await + .context("create block rule: golang")? + .into_dyn(), self::rule::skills_sh::RuleSkillsSh::try_new( guard, layered_client, From 6b2d539d1d6b4d5ab538240d57159b3d058013c4 Mon Sep 17 00:00:00 2001 From: bitterpanda Date: Tue, 21 Apr 2026 14:20:28 +0200 Subject: [PATCH 04/19] add e2e & test cases for golang firewall rule --- .../src/test/e2e/test_proxy/firewall_go.rs | 207 ++++++++++++++++++ proxy-bin-l7/src/test/e2e/test_proxy/mod.rs | 1 + .../src/http/firewall/rule/golang/tests.rs | 112 ++++++++++ 3 files changed, 320 insertions(+) create mode 100644 proxy-bin-l7/src/test/e2e/test_proxy/firewall_go.rs create mode 100644 proxy-lib/src/http/firewall/rule/golang/tests.rs diff --git a/proxy-bin-l7/src/test/e2e/test_proxy/firewall_go.rs b/proxy-bin-l7/src/test/e2e/test_proxy/firewall_go.rs new file mode 100644 index 00000000..0b7db38f --- /dev/null +++ b/proxy-bin-l7/src/test/e2e/test_proxy/firewall_go.rs @@ -0,0 +1,207 @@ +use rama::{ + http::{BodyExtractExt as _, StatusCode, service::client::HttpClientExt as _}, + telemetry::tracing, +}; + +use crate::{ + client::mock_server::malware_list::{FRESH_GOLANG_MODULE_NAME, FRESH_GOLANG_MODULE_VERSION}, + test::e2e, +}; + +#[tokio::test] +#[tracing_test::traced_test] +async fn test_go_malware_blocked() { + let runtime = e2e::runtime::get().await; + let client = runtime.client_with_http_proxy().await; + + // github.com/gorilla/mux v1.8.0 is listed as malware in the mock list + let resp = client + .get("https://proxy.golang.org/github.com/gorilla/mux/@v/v1.8.0.zip") + .send() + .await + .unwrap(); + + assert_eq!(StatusCode::FORBIDDEN, resp.status()); +} + +#[tokio::test] +#[tracing_test::traced_test] +async fn test_go_safe_package_allowed() { + let runtime = e2e::runtime::get().await; + let client = runtime.client_with_http_proxy().await; + + // gin is not in the malware list + let resp = client + .get("https://proxy.golang.org/github.com/gin-gonic/gin/@v/v1.9.1.zip") + .send() + .await + .unwrap(); + + assert_eq!(StatusCode::OK, resp.status()); +} + +#[tokio::test] +#[tracing_test::traced_test] +async fn test_go_non_zip_allowed() { + let runtime = e2e::runtime::get().await; + let client = runtime.client_with_http_proxy().await; + + // .mod files are not intercepted — only .zip downloads are blocked + let resp = client + .get("https://proxy.golang.org/github.com/gorilla/mux/@v/v1.8.0.mod") + .send() + .await + .unwrap(); + + assert_eq!(StatusCode::OK, resp.status()); +} + +#[tokio::test] +#[tracing_test::traced_test] +async fn test_go_list_endpoint_allowed() { + let runtime = e2e::runtime::get().await; + let client = runtime.client_with_http_proxy().await; + + // /@v/list is metadata, not intercepted + let resp = client + .get("https://proxy.golang.org/github.com/gorilla/mux/@v/list") + .send() + .await + .unwrap(); + + assert_eq!(StatusCode::OK, resp.status()); +} + +#[tokio::test] +#[tracing_test::traced_test] +async fn test_go_allows_different_version() { + let runtime = e2e::runtime::get().await; + let client = runtime.client_with_http_proxy().await; + + // gorilla/mux v1.7.0 is not in the malware list (only v1.8.0 is) + let resp = client + .get("https://proxy.golang.org/github.com/gorilla/mux/@v/v1.7.0.zip") + .send() + .await + .unwrap(); + + assert_eq!(StatusCode::OK, resp.status()); +} + +#[tokio::test] +#[tracing_test::traced_test] +async fn test_go_package_allowed_by_endpoint_policy_exception() { + let runtime = e2e::runtime::spawn_with_agent_identity( + "policy-allow-gorilla-mux-golang", + "mock_device", + &[], + ) + .await; + let client = runtime.client_with_http_proxy().await; + + // gorilla/mux v1.8.0 is malware, but the allowed_packages exception overrides the malware check + let resp = client + .get("https://proxy.golang.org/github.com/gorilla/mux/@v/v1.8.0.zip") + .send() + .await + .unwrap(); + + assert_eq!(StatusCode::OK, resp.status()); +} + +#[tokio::test] +#[tracing_test::traced_test] +async fn test_go_package_blocked_by_endpoint_policy_block_all() { + let runtime = + e2e::runtime::spawn_with_agent_identity("policy-block-golang", "mock_device", &[]).await; + let client = runtime.client_with_http_proxy().await; + + // gin is not malware, but block_all_installs blocks it + let resp = client + .get("https://proxy.golang.org/github.com/gin-gonic/gin/@v/v1.9.1.zip") + .send() + .await + .unwrap(); + + assert_eq!(StatusCode::FORBIDDEN, resp.status()); +} + +#[tokio::test] +#[tracing_test::traced_test] +async fn test_go_package_blocked_by_endpoint_policy_rejected_package() { + let runtime = + e2e::runtime::spawn_with_agent_identity("policy-reject-gin-golang", "mock_device", &[]) + .await; + let client = runtime.client_with_http_proxy().await; + + // gin is in rejected_packages — blocked even though it's not malware + let resp = client + .get("https://proxy.golang.org/github.com/gin-gonic/gin/@v/v1.9.1.zip") + .send() + .await + .unwrap(); + + assert_eq!(StatusCode::FORBIDDEN, resp.status()); +} + +#[tokio::test] +#[tracing_test::traced_test] +async fn test_go_package_blocked_by_endpoint_policy_request_installs() { + let runtime = e2e::runtime::spawn_with_agent_identity( + "policy-request-installs-golang", + "mock_device", + &[], + ) + .await; + let client = runtime.client_with_http_proxy().await; + + // gin is not malware, but request_installs requires approval for all installs + let resp = client + .get("https://proxy.golang.org/github.com/gin-gonic/gin/@v/v1.9.1.zip") + .send() + .await + .unwrap(); + + assert_eq!(StatusCode::FORBIDDEN, resp.status()); +} + +#[tokio::test] +#[tracing_test::traced_test] +async fn test_go_new_package_blocked() { + let runtime = e2e::runtime::get().await; + let client = runtime.client_with_http_proxy().await; + + let module = FRESH_GOLANG_MODULE_NAME; + let ver = FRESH_GOLANG_MODULE_VERSION; + let url = format!("https://proxy.golang.org/{module}/@v/v{ver}.zip"); + + let resp = client.get(url).send().await.unwrap(); + + assert_eq!(StatusCode::FORBIDDEN, resp.status()); + + let payload = resp.try_into_string().await.unwrap(); + assert!( + payload.to_lowercase().contains("24 hours") || payload.to_lowercase().contains("vetted"), + "expected blocked response to mention 24-hour vetting, got: {payload}" + ); +} + +#[tokio::test] +#[tracing_test::traced_test] +async fn test_go_new_package_not_blocked_via_policy_cutoff() { + let runtime = e2e::runtime::spawn_with_agent_identity( + "policy-bypass-new-package-golang", + "mock_device", + &[], + ) + .await; + let client = runtime.client_with_http_proxy().await; + + let module = FRESH_GOLANG_MODULE_NAME; + let ver = FRESH_GOLANG_MODULE_VERSION; + let url = format!("https://proxy.golang.org/{module}/@v/v{ver}.zip"); + + let resp = client.get(url).send().await.unwrap(); + + assert_eq!(StatusCode::OK, resp.status()); +} diff --git a/proxy-bin-l7/src/test/e2e/test_proxy/mod.rs b/proxy-bin-l7/src/test/e2e/test_proxy/mod.rs index a1399691..5991464d 100644 --- a/proxy-bin-l7/src/test/e2e/test_proxy/mod.rs +++ b/proxy-bin-l7/src/test/e2e/test_proxy/mod.rs @@ -1,6 +1,7 @@ mod no_firewall; mod firewall_chrome; +mod firewall_go; mod firewall_maven; mod firewall_npm; mod firewall_nuget; diff --git a/proxy-lib/src/http/firewall/rule/golang/tests.rs b/proxy-lib/src/http/firewall/rule/golang/tests.rs new file mode 100644 index 00000000..00b76b03 --- /dev/null +++ b/proxy-lib/src/http/firewall/rule/golang/tests.rs @@ -0,0 +1,112 @@ +use crate::package::version::PragmaticSemver; + +use super::parse_package_from_path; + +fn assert_parsed(path: &str, expected_name: &str, expected_version: &str) { + let pkg = parse_package_from_path(path) + .unwrap_or_else(|| panic!("expected package to parse: {path}")); + assert_eq!(pkg.fully_qualified_name, expected_name, "path: {path}"); + assert_eq!( + pkg.version, + PragmaticSemver::parse(expected_version).unwrap(), + "path: {path}" + ); +} + +#[test] +fn test_parse_simple_module() { + assert_parsed( + "github.com/gorilla/mux/@v/v1.8.0.zip", + "github.com/gorilla/mux", + "1.8.0", + ); +} + +#[test] +fn test_parse_with_leading_slash() { + assert_parsed( + "/github.com/gorilla/mux/@v/v1.8.0.zip", + "github.com/gorilla/mux", + "1.8.0", + ); +} + +#[test] +fn test_parse_deep_module_path() { + assert_parsed( + "github.com/aikidosec/firewall-go/cmd/zen-go/@v/v1.0.0.zip", + "github.com/aikidosec/firewall-go/cmd/zen-go", + "1.0.0", + ); +} + +#[test] +fn test_parse_percent_encoded_uppercase() { + // %21 is !, and !x encodes uppercase X in Go's module proxy protocol + // github.com/%21aikido%21sec/firewall-go → github.com/!aikido!sec/firewall-go → lowercase + assert_parsed( + "github.com/%21aikido%21sec/firewall-go/cmd/zen-go/@v/v1.0.0.zip", + "github.com/!aikido!sec/firewall-go/cmd/zen-go", + "1.0.0", + ); +} + +#[test] +fn test_parse_golang_org_module() { + assert_parsed( + "golang.org/x/sys/@v/v0.15.0.zip", + "golang.org/x/sys", + "0.15.0", + ); +} + +#[test] +fn test_parse_prerelease_version() { + assert_parsed( + "github.com/foo/bar/@v/v1.0.0-alpha.1.zip", + "github.com/foo/bar", + "1.0.0-alpha.1", + ); +} + +#[test] +fn test_reject_mod_file() { + assert!( + parse_package_from_path("github.com/gorilla/mux/@v/v1.8.0.mod").is_none(), + "should not parse .mod files" + ); +} + +#[test] +fn test_reject_info_file() { + assert!( + parse_package_from_path("github.com/gorilla/mux/@v/v1.8.0.info").is_none(), + "should not parse .info files" + ); +} + +#[test] +fn test_reject_list_endpoint() { + assert!( + parse_package_from_path("github.com/gorilla/mux/@v/list").is_none(), + "should not parse /@v/list" + ); +} + +#[test] +fn test_reject_no_at_v_segment() { + assert!( + parse_package_from_path("github.com/gorilla/mux/v1.8.0.zip").is_none(), + "should not parse paths without /@v/" + ); +} + +#[test] +fn test_parse_pseudo_version() { + // Pseudo-versions are used for commits not tagged with a semver + assert_parsed( + "github.com/some/pkg/@v/v0.0.0-20231215000000-abc1234def56.zip", + "github.com/some/pkg", + "0.0.0-20231215000000-abc1234def56", + ); +} From 6f2b2bb8daeb0e34402a3a0fe7672691b07bef49 Mon Sep 17 00:00:00 2001 From: bitterpanda Date: Wed, 22 Apr 2026 01:27:19 +0200 Subject: [PATCH 05/19] Create a go_module_unescape.rs --- .../firewall/rule/golang/module_escape.rs | 27 +++++ .../rule/golang/module_escape_tests.rs | 103 ++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 proxy-lib/src/http/firewall/rule/golang/module_escape.rs create mode 100644 proxy-lib/src/http/firewall/rule/golang/module_escape_tests.rs diff --git a/proxy-lib/src/http/firewall/rule/golang/module_escape.rs b/proxy-lib/src/http/firewall/rule/golang/module_escape.rs new file mode 100644 index 00000000..333803a2 --- /dev/null +++ b/proxy-lib/src/http/firewall/rule/golang/module_escape.rs @@ -0,0 +1,27 @@ +/// Reverses Go's module path escaping (`golang.org/x/mod/module.EscapePath`). +/// +/// Go encodes each uppercase letter as `!` + lowercase so module caches work on +/// case-insensitive filesystems — e.g. `AikidoSec` → `!aikido!sec`. `!a`–`!z` are +/// each replaced by the corresponding uppercase letter; any `!` not followed by a +/// lowercase letter is passed through unchanged. +pub(super) fn go_module_unescape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + if c == '!' { + if let Some(&next) = chars.peek() { + if next.is_ascii_lowercase() { + out.push(next.to_ascii_uppercase()); + chars.next(); + continue; + } + } + } + out.push(c); + } + out +} + +#[cfg(test)] +#[path = "module_escape_tests.rs"] +mod tests; diff --git a/proxy-lib/src/http/firewall/rule/golang/module_escape_tests.rs b/proxy-lib/src/http/firewall/rule/golang/module_escape_tests.rs new file mode 100644 index 00000000..77a2945b --- /dev/null +++ b/proxy-lib/src/http/firewall/rule/golang/module_escape_tests.rs @@ -0,0 +1,103 @@ +use super::go_module_unescape; + +#[test] +fn empty_string() { + assert_eq!(go_module_unescape(""), ""); +} + +#[test] +fn no_uppercase() { + // Common case — all-lowercase paths are not modified at all + assert_eq!(go_module_unescape("gorilla"), "gorilla"); +} + +#[test] +fn all_lowercase_proxy_path() { + assert_eq!( + go_module_unescape("github.com/gorilla/mux"), + "github.com/gorilla/mux" + ); +} + +#[test] +fn single_leading_uppercase() { + // "Azure" → escaped as "!azure" + assert_eq!(go_module_unescape("!azure"), "Azure"); +} + +#[test] +fn multiple_uppercases() { + // "AikidoSec" → escaped as "!aikido!sec" + assert_eq!(go_module_unescape("!aikido!sec"), "AikidoSec"); +} + +#[test] +fn uppercase_in_org_only() { + // "GoogleCloudPlatform/cloudsql-proxy" → "!google!cloud!platform/cloudsql-proxy" + assert_eq!( + go_module_unescape("!google!cloud!platform/cloudsql-proxy"), + "GoogleCloudPlatform/cloudsql-proxy" + ); +} + +#[test] +fn uppercase_in_repo_only() { + assert_eq!(go_module_unescape("org/!my!repo"), "org/MyRepo"); +} + +#[test] +fn mixed_segments_only_first_encoded() { + // "Azure/azure-sdk-for-go" — org is capitalized, repo is not + assert_eq!( + go_module_unescape("!azure/azure-sdk-for-go"), + "Azure/azure-sdk-for-go" + ); +} + +#[test] +fn consecutive_uppercases() { + // "ABC" → "!a!b!c" + assert_eq!(go_module_unescape("!a!b!c"), "ABC"); +} + +#[test] +fn sirupsen_logrus() { + // Real-world: "Sirupsen/logrus" is a well-known module with an uppercase org + assert_eq!(go_module_unescape("!sirupsen/logrus"), "Sirupsen/logrus"); +} + +#[test] +fn full_proxy_path_with_escaping() { + // Full path as it appears after percent-decoding the proxy URL + assert_eq!( + go_module_unescape("github.com/!sirupsen/logrus"), + "github.com/Sirupsen/logrus" + ); +} + +// --- Invalid / defensive cases --- + +#[test] +fn invalid_escape_bang_at_end() { + // Trailing `!` with nothing after it — passed through unchanged + assert_eq!(go_module_unescape("foo!"), "foo!"); +} + +#[test] +fn invalid_escape_bang_followed_by_uppercase() { + // `!!bar` — first `!` is not a valid escape (followed by `!`, not a-z) so it passes + // through; the second `!b` is a valid escape and becomes `B`. + // Go never produces `!!` in real paths, but each `!` is processed independently. + assert_eq!(go_module_unescape("foo!!bar"), "foo!Bar"); +} + +#[test] +fn invalid_escape_bang_followed_by_digit() { + // `!1` — not a valid Go escape sequence, passed through + assert_eq!(go_module_unescape("foo!1bar"), "foo!1bar"); +} + +#[test] +fn invalid_escape_bang_followed_by_space() { + assert_eq!(go_module_unescape("foo! bar"), "foo! bar"); +} From 2ed7da2021eb55c455ac6e54f53dd61a6a1dcfbf Mon Sep 17 00:00:00 2001 From: bitterpanda Date: Wed, 22 Apr 2026 01:28:18 +0200 Subject: [PATCH 06/19] golang: Move from mod.rs -> parser.rs & add test cases --- .../src/http/firewall/rule/golang/mod.rs | 76 -------- .../src/http/firewall/rule/golang/parser.rs | 117 +++++++++++ .../http/firewall/rule/golang/parser_tests.rs | 184 ++++++++++++++++++ .../src/http/firewall/rule/golang/tests.rs | 113 +---------- 4 files changed, 302 insertions(+), 188 deletions(-) create mode 100644 proxy-lib/src/http/firewall/rule/golang/parser.rs create mode 100644 proxy-lib/src/http/firewall/rule/golang/parser_tests.rs diff --git a/proxy-lib/src/http/firewall/rule/golang/mod.rs b/proxy-lib/src/http/firewall/rule/golang/mod.rs index b55c2e8c..dee1916d 100644 --- a/proxy-lib/src/http/firewall/rule/golang/mod.rs +++ b/proxy-lib/src/http/firewall/rule/golang/mod.rs @@ -207,79 +207,3 @@ impl RuleGolang { Ok(RequestAction::Allow(req)) } } - -pub(super) struct GoPackage { - pub(super) fully_qualified_name: String, - pub(super) version: PragmaticSemver, -} - -fn is_zip_download(path: &str) -> bool { - path.ends_with(".zip") && path.contains("/@v/") -} - -/// Parses a Go module proxy zip URL path into a normalized module name and version. -/// -/// Expected path shape: `/{module_path}/@v/{version}.zip` -/// -/// Go's module proxy encodes uppercase letters as `!` + lowercase (e.g., `AikidoSec` → -/// `!aikido!sec`, percent-encoded as `%21aikido%21sec`). We lowercase the whole path -/// to normalise for malware-list lookup (which uses `LowerCaseEntryFormatter`). -pub(super) fn parse_package_from_path(path: &str) -> Option { - let path = path.trim_matches('/'); - - let (module_path_raw, rest) = path.split_once("/@v/")?; - let version_raw = rest.strip_suffix(".zip")?; - - if module_path_raw.is_empty() || version_raw.is_empty() { - return None; - } - - // Percent-decode the module path (handles %21 → !) - let module_path_decoded = percent_decode(module_path_raw); - // Lowercase for consistent malware-list lookup - let module_name = module_path_decoded.to_ascii_lowercase(); - - // Go versions always carry a 'v' prefix in the proxy URL; strip it for semver parsing - let version_str = version_raw.strip_prefix('v').unwrap_or(version_raw); - let version = PragmaticSemver::from_str(version_str) - .inspect_err(|err| { - tracing::debug!( - "failed to parse golang module ({module_name}) version (raw = {version_raw}): err = {err}" - ); - }) - .ok()?; - - Some(GoPackage { - fully_qualified_name: module_name, - version, - }) -} - -/// Decodes percent-encoded characters in a URL path segment. -/// Only handles the ASCII subset relevant to Go module paths (`%21` → `!`, etc.). -fn percent_decode(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - let bytes = s.as_bytes(); - let mut i = 0; - while i < bytes.len() { - if bytes[i] == b'%' && i + 2 < bytes.len() { - if let (Some(h), Some(l)) = (hex_val(bytes[i + 1]), hex_val(bytes[i + 2])) { - out.push((h << 4 | l) as char); - i += 3; - continue; - } - } - out.push(bytes[i] as char); - i += 1; - } - out -} - -fn hex_val(b: u8) -> Option { - match b { - b'0'..=b'9' => Some(b - b'0'), - b'a'..=b'f' => Some(b - b'a' + 10), - b'A'..=b'F' => Some(b - b'A' + 10), - _ => None, - } -} diff --git a/proxy-lib/src/http/firewall/rule/golang/parser.rs b/proxy-lib/src/http/firewall/rule/golang/parser.rs new file mode 100644 index 00000000..324cd778 --- /dev/null +++ b/proxy-lib/src/http/firewall/rule/golang/parser.rs @@ -0,0 +1,117 @@ +use std::str::FromStr; + +use rama::telemetry::tracing; + +use crate::package::version::PragmaticSemver; + +#[path = "module_escape.rs"] +mod module_escape; +use module_escape::go_module_unescape; + +pub(super) struct GoPackage { + pub(super) fully_qualified_name: String, + pub(super) version: PragmaticSemver, +} + +pub(super) fn is_zip_download(path: &str) -> bool { + path.ends_with(".zip") && path.contains("/@v/") +} + +/// Parses a Go module proxy zip URL path into a normalized module name and version. +/// +/// Expected path shape: `/{module_path}/@v/{version}.zip` +/// +/// Go module paths go through two layers of encoding in proxy URLs: +/// 1. Module escaping (`golang.org/x/mod/module.EscapePath`): each uppercase letter +/// becomes `!` + its lowercase equivalent — e.g. `AikidoSec` → `!aikido!sec`. +/// 2. Percent-encoding of `!` in the URL: `!` → `%21`. +/// +/// So `github.com/AikidoSec/firewall-go` appears in the URL as +/// `github.com/%21aikido%21sec/firewall-go`. +/// +/// We reverse both layers then lowercase for malware-list lookup +/// (which uses `LowerCaseEntryFormatter`). +pub(super) fn parse_package_from_path(path: &str) -> Option { + let path = path.trim_matches('/'); + + let (module_path_raw, rest) = path.split_once("/@v/")?; + let version_raw = rest.strip_suffix(".zip")?; + + if module_path_raw.is_empty() || version_raw.is_empty() { + return None; + } + + let module_name = normalize_module_path(module_path_raw); + + // Go versions always carry a 'v' prefix in the proxy URL; strip it for semver parsing + let version_str = version_raw.strip_prefix('v').unwrap_or(version_raw); + let version = PragmaticSemver::from_str(version_str) + .inspect_err(|err| { + tracing::debug!( + "failed to parse golang module ({module_name}) version (raw = {version_raw}): err = {err}" + ); + }) + .ok()?; + + Some(GoPackage { + fully_qualified_name: module_name, + version, + }) +} + +/// Parses a Go module proxy list URL path into a normalized module name. +/// +/// Expected path shape: `/{module_path}/@v/list` +pub(super) fn parse_module_from_list_path(path: &str) -> Option { + let path = path.trim_matches('/'); + let module_path_raw = path.strip_suffix("/@v/list")?; + if module_path_raw.is_empty() { + return None; + } + Some(normalize_module_path(module_path_raw)) +} + +/// Normalizes a raw module path segment from a Go proxy URL to a canonical lowercase name. +/// +/// Reverses both encoding layers applied by the Go toolchain: +/// 1. Percent-decode (`%21` → `!`) +/// 2. Module-unescape (`!x` → uppercase `X`, per `golang.org/x/mod/module.UnescapePath`) +/// Then lowercases the result for consistent malware-list lookup. +fn normalize_module_path(raw: &str) -> String { + let percent_decoded = percent_decode(raw); + let unescaped = go_module_unescape(&percent_decoded); + unescaped.to_ascii_lowercase() +} + +/// Decodes percent-encoded characters in a URL path segment. +/// Only handles the ASCII subset relevant to Go module paths (`%21` → `!`, etc.). +fn percent_decode(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let bytes = s.as_bytes(); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' && i + 2 < bytes.len() { + if let (Some(h), Some(l)) = (hex_val(bytes[i + 1]), hex_val(bytes[i + 2])) { + out.push((h << 4 | l) as char); + i += 3; + continue; + } + } + out.push(bytes[i] as char); + i += 1; + } + out +} + +fn hex_val(b: u8) -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} + +#[cfg(test)] +#[path = "parser_tests.rs"] +mod tests; diff --git a/proxy-lib/src/http/firewall/rule/golang/parser_tests.rs b/proxy-lib/src/http/firewall/rule/golang/parser_tests.rs new file mode 100644 index 00000000..1156c880 --- /dev/null +++ b/proxy-lib/src/http/firewall/rule/golang/parser_tests.rs @@ -0,0 +1,184 @@ +use crate::package::version::PragmaticSemver; + +use super::{parse_module_from_list_path, parse_package_from_path}; + +fn assert_parsed(path: &str, expected_name: &str, expected_version: &str) { + let pkg = parse_package_from_path(path) + .unwrap_or_else(|| panic!("expected package to parse: {path}")); + assert_eq!(pkg.fully_qualified_name, expected_name, "path: {path}"); + assert_eq!( + pkg.version, + PragmaticSemver::parse(expected_version).unwrap(), + "path: {path}" + ); +} + +// --- parse_package_from_path tests (zip downloads) --- + +#[test] +fn test_parse_simple_module() { + assert_parsed( + "github.com/gorilla/mux/@v/v1.8.0.zip", + "github.com/gorilla/mux", + "1.8.0", + ); +} + +#[test] +fn test_parse_with_leading_slash() { + assert_parsed( + "/github.com/gorilla/mux/@v/v1.8.0.zip", + "github.com/gorilla/mux", + "1.8.0", + ); +} + +#[test] +fn test_parse_deep_module_path() { + assert_parsed( + "github.com/aikidosec/firewall-go/cmd/zen-go/@v/v1.0.0.zip", + "github.com/aikidosec/firewall-go/cmd/zen-go", + "1.0.0", + ); +} + +#[test] +fn test_parse_percent_encoded_uppercase() { + // %21 is !, and !x encodes uppercase X in Go's module proxy protocol + // github.com/%21aikido%21sec → percent-decode → github.com/!aikido!sec + // → module-unescape → github.com/AikidoSec → lowercase → github.com/aikidosec + assert_parsed( + "github.com/%21aikido%21sec/firewall-go/cmd/zen-go/@v/v1.0.0.zip", + "github.com/aikidosec/firewall-go/cmd/zen-go", + "1.0.0", + ); +} + +#[test] +fn test_parse_golang_org_module() { + assert_parsed( + "golang.org/x/sys/@v/v0.15.0.zip", + "golang.org/x/sys", + "0.15.0", + ); +} + +#[test] +fn test_parse_prerelease_version() { + assert_parsed( + "github.com/foo/bar/@v/v1.0.0-alpha.1.zip", + "github.com/foo/bar", + "1.0.0-alpha.1", + ); +} + +#[test] +fn test_reject_mod_file() { + assert!( + parse_package_from_path("github.com/gorilla/mux/@v/v1.8.0.mod").is_none(), + "should not parse .mod files" + ); +} + +#[test] +fn test_reject_info_file() { + assert!( + parse_package_from_path("github.com/gorilla/mux/@v/v1.8.0.info").is_none(), + "should not parse .info files" + ); +} + +#[test] +fn test_reject_list_endpoint() { + assert!( + parse_package_from_path("github.com/gorilla/mux/@v/list").is_none(), + "should not parse /@v/list" + ); +} + +#[test] +fn test_reject_no_at_v_segment() { + assert!( + parse_package_from_path("github.com/gorilla/mux/v1.8.0.zip").is_none(), + "should not parse paths without /@v/" + ); +} + +#[test] +fn test_parse_pseudo_version() { + // Pseudo-versions are used for commits not tagged with a semver + assert_parsed( + "github.com/some/pkg/@v/v0.0.0-20231215000000-abc1234def56.zip", + "github.com/some/pkg", + "0.0.0-20231215000000-abc1234def56", + ); +} + +#[test] +fn test_parse_non_github_domain() { + // Malware entries can use arbitrary domains, not just github.com + assert_parsed( + "github.web.gylab.com/mrz1836/go-pandadoc/@v/v1.0.1.zip", + "github.web.gylab.com/mrz1836/go-pandadoc", + "1.0.1", + ); +} + +// --- parse_module_from_list_path tests --- + +#[test] +fn test_parse_list_path() { + assert_eq!( + parse_module_from_list_path("github.com/gorilla/mux/@v/list"), + Some("github.com/gorilla/mux".to_owned()) + ); +} + +#[test] +fn test_parse_list_path_leading_slash() { + assert_eq!( + parse_module_from_list_path("/github.com/gorilla/mux/@v/list"), + Some("github.com/gorilla/mux".to_owned()) + ); +} + +#[test] +fn test_parse_list_path_deep_module() { + assert_eq!( + parse_module_from_list_path("github.com/aikidosec/firewall-go/cmd/zen-go/@v/list"), + Some("github.com/aikidosec/firewall-go/cmd/zen-go".to_owned()) + ); +} + +#[test] +fn test_parse_list_path_percent_encoded() { + // %21 → !, then lowercased + assert_eq!( + parse_module_from_list_path("github.com/%21Aikido/pkg/@v/list"), + Some("github.com/!aikido/pkg".to_owned()) + ); +} + +#[test] +fn test_reject_list_path_no_module() { + assert!( + parse_module_from_list_path("/@v/list").is_none(), + "should return None when module path is empty" + ); +} + +#[test] +fn test_reject_non_list_path() { + assert!( + parse_module_from_list_path("github.com/gorilla/mux/@v/v1.8.0.zip").is_none(), + "should not parse zip paths as list" + ); +} + +#[test] +fn test_reject_list_path_wrong_suffix() { + assert!( + parse_module_from_list_path("github.com/gorilla/mux/@v/v1.8.0.mod").is_none(), + "should not parse non-list paths" + ); +} diff --git a/proxy-lib/src/http/firewall/rule/golang/tests.rs b/proxy-lib/src/http/firewall/rule/golang/tests.rs index 00b76b03..0be6e3c7 100644 --- a/proxy-lib/src/http/firewall/rule/golang/tests.rs +++ b/proxy-lib/src/http/firewall/rule/golang/tests.rs @@ -1,112 +1 @@ -use crate::package::version::PragmaticSemver; - -use super::parse_package_from_path; - -fn assert_parsed(path: &str, expected_name: &str, expected_version: &str) { - let pkg = parse_package_from_path(path) - .unwrap_or_else(|| panic!("expected package to parse: {path}")); - assert_eq!(pkg.fully_qualified_name, expected_name, "path: {path}"); - assert_eq!( - pkg.version, - PragmaticSemver::parse(expected_version).unwrap(), - "path: {path}" - ); -} - -#[test] -fn test_parse_simple_module() { - assert_parsed( - "github.com/gorilla/mux/@v/v1.8.0.zip", - "github.com/gorilla/mux", - "1.8.0", - ); -} - -#[test] -fn test_parse_with_leading_slash() { - assert_parsed( - "/github.com/gorilla/mux/@v/v1.8.0.zip", - "github.com/gorilla/mux", - "1.8.0", - ); -} - -#[test] -fn test_parse_deep_module_path() { - assert_parsed( - "github.com/aikidosec/firewall-go/cmd/zen-go/@v/v1.0.0.zip", - "github.com/aikidosec/firewall-go/cmd/zen-go", - "1.0.0", - ); -} - -#[test] -fn test_parse_percent_encoded_uppercase() { - // %21 is !, and !x encodes uppercase X in Go's module proxy protocol - // github.com/%21aikido%21sec/firewall-go → github.com/!aikido!sec/firewall-go → lowercase - assert_parsed( - "github.com/%21aikido%21sec/firewall-go/cmd/zen-go/@v/v1.0.0.zip", - "github.com/!aikido!sec/firewall-go/cmd/zen-go", - "1.0.0", - ); -} - -#[test] -fn test_parse_golang_org_module() { - assert_parsed( - "golang.org/x/sys/@v/v0.15.0.zip", - "golang.org/x/sys", - "0.15.0", - ); -} - -#[test] -fn test_parse_prerelease_version() { - assert_parsed( - "github.com/foo/bar/@v/v1.0.0-alpha.1.zip", - "github.com/foo/bar", - "1.0.0-alpha.1", - ); -} - -#[test] -fn test_reject_mod_file() { - assert!( - parse_package_from_path("github.com/gorilla/mux/@v/v1.8.0.mod").is_none(), - "should not parse .mod files" - ); -} - -#[test] -fn test_reject_info_file() { - assert!( - parse_package_from_path("github.com/gorilla/mux/@v/v1.8.0.info").is_none(), - "should not parse .info files" - ); -} - -#[test] -fn test_reject_list_endpoint() { - assert!( - parse_package_from_path("github.com/gorilla/mux/@v/list").is_none(), - "should not parse /@v/list" - ); -} - -#[test] -fn test_reject_no_at_v_segment() { - assert!( - parse_package_from_path("github.com/gorilla/mux/v1.8.0.zip").is_none(), - "should not parse paths without /@v/" - ); -} - -#[test] -fn test_parse_pseudo_version() { - // Pseudo-versions are used for commits not tagged with a semver - assert_parsed( - "github.com/some/pkg/@v/v0.0.0-20231215000000-abc1234def56.zip", - "github.com/some/pkg", - "0.0.0-20231215000000-abc1234def56", - ); -} +// Tests for URL parsing are in parser_tests.rs (referenced from parser.rs). From 1c5d4b8e6c4b10407b59e03cda8c65c0841678f6 Mon Sep 17 00:00:00 2001 From: bitterpanda Date: Wed, 22 Apr 2026 01:31:30 +0200 Subject: [PATCH 07/19] golang: min package age rewrites --- .../rule/golang/min_package_age/mod.rs | 132 ++++++++++++++ .../rule/golang/min_package_age/tests.rs | 172 ++++++++++++++++++ 2 files changed, 304 insertions(+) create mode 100644 proxy-lib/src/http/firewall/rule/golang/min_package_age/mod.rs create mode 100644 proxy-lib/src/http/firewall/rule/golang/min_package_age/tests.rs diff --git a/proxy-lib/src/http/firewall/rule/golang/min_package_age/mod.rs b/proxy-lib/src/http/firewall/rule/golang/min_package_age/mod.rs new file mode 100644 index 00000000..fe9907ff --- /dev/null +++ b/proxy-lib/src/http/firewall/rule/golang/min_package_age/mod.rs @@ -0,0 +1,132 @@ +use std::str::FromStr; + +use rama::{ + error::{BoxError, ErrorContext as _}, + http::{ + Body, Response, + body::util::BodyExt as _, + headers::{CacheControl, HeaderMapExt as _}, + layer::remove_header::{ + remove_cache_policy_headers, remove_cache_validation_response_headers, + remove_payload_metadata_headers, + }, + }, + telemetry::tracing, + utils::{str::arcstr::ArcStr, time::now_unix_ms}, +}; + +use crate::{ + http::firewall::{ + events::{Artifact, MinPackageAgeEvent}, + notifier::EventNotifier, + }, + package::{ + released_packages_list::RemoteReleasedPackagesList, + version::{PackageVersion, PragmaticSemver}, + }, +}; + +#[derive(Debug, Clone)] +pub(in crate::http::firewall) struct MinPackageAgeGolang { + notifier: Option, +} + +impl MinPackageAgeGolang { + pub fn new(notifier: Option) -> Self { + Self { notifier } + } + + pub async fn rewrite_list_response( + &self, + resp: Response, + module_name: &str, + released_packages: &RemoteReleasedPackagesList, + cutoff_secs: i64, + ) -> Result { + let (mut parts, body) = resp.into_parts(); + + let bytes = body + .collect() + .await + .context("collect golang list response body")? + .to_bytes(); + + let text = match std::str::from_utf8(&bytes) { + Ok(t) => t, + Err(_) => { + tracing::debug!("golang list response body is not valid UTF-8, passing through"); + return Ok(Response::from_parts(parts, Body::from(bytes))); + } + }; + + let mut suppressed: Vec = Vec::new(); + let mut kept: Vec<&str> = Vec::new(); + + for line in text.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + let version_str = line.strip_prefix('v').unwrap_or(line); + match PragmaticSemver::from_str(version_str) { + Ok(parsed) => { + let version = PackageVersion::Semver(parsed); + if released_packages.is_recently_released( + module_name, + Some(&version), + cutoff_secs, + ) { + suppressed.push(version); + } else { + kept.push(line); + } + } + Err(_) => { + kept.push(line); + } + } + } + + if suppressed.is_empty() { + return Ok(Response::from_parts(parts, Body::from(bytes))); + } + + tracing::info!( + module = %module_name, + suppressed_versions = ?suppressed, + "Go module list rewritten: suppressed too-young versions" + ); + + remove_cache_policy_headers(&mut parts.headers); + remove_cache_validation_response_headers(&mut parts.headers); + remove_payload_metadata_headers(&mut parts.headers); + parts.headers.typed_insert(CacheControl::new().with_no_cache()); + + let new_body = kept.join("\n") + "\n"; + + self.notify_rewrite(module_name, suppressed).await; + + Ok(Response::from_parts(parts, Body::from(new_body))) + } + + async fn notify_rewrite(&self, module_name: &str, suppressed: Vec) { + let Some(notifier) = &self.notifier else { + return; + }; + let identifier: ArcStr = module_name.into(); + let event = MinPackageAgeEvent { + ts_ms: now_unix_ms(), + artifact: Artifact { + product: "golang".into(), + identifier: identifier.clone(), + display_name: None, + version: None, + }, + suppressed_versions: suppressed, + }; + notifier.notify_min_package_age(event).await; + } +} + +#[cfg(test)] +mod tests; diff --git a/proxy-lib/src/http/firewall/rule/golang/min_package_age/tests.rs b/proxy-lib/src/http/firewall/rule/golang/min_package_age/tests.rs new file mode 100644 index 00000000..da1f3909 --- /dev/null +++ b/proxy-lib/src/http/firewall/rule/golang/min_package_age/tests.rs @@ -0,0 +1,172 @@ +use rama::{ + http::{Body, BodyExtractExt as _}, + utils::time::now_unix_ms, +}; + +use crate::package::released_packages_list::{ + LowerCaseReleasedPackageFormatter, ReleasedPackageData, RemoteReleasedPackagesList, +}; + +use super::*; + +fn make_list_response(body: &str) -> Response { + Response::builder() + .header("content-type", "text/plain; charset=UTF-8") + .body(Body::from(body.to_owned())) + .unwrap() +} + +fn make_list_response_with_cache(body: &str) -> Response { + Response::builder() + .header("content-type", "text/plain; charset=UTF-8") + .header("cache-control", "public, max-age=60") + .header("etag", "abc123") + .header("last-modified", "Wed, 01 Jan 2020 00:00:00 GMT") + .body(Body::from(body.to_owned())) + .unwrap() +} + +fn make_released_packages(entries: &[(&str, &str, u64)]) -> RemoteReleasedPackagesList { + let now_secs = now_unix_ms() / 1000; + RemoteReleasedPackagesList::from_entries_for_tests( + entries + .iter() + .map(|(package_name, version, hours_ago)| ReleasedPackageData { + package_name: (*package_name).to_owned(), + version: version.parse().unwrap(), + released_on: now_secs - (*hours_ago as i64 * 3600), + }) + .collect(), + now_secs, + LowerCaseReleasedPackageFormatter, + ) +} + +fn default_cutoff_secs() -> i64 { + (now_unix_ms() / 1000) - (48 * 3600) +} + +#[tokio::test] +async fn filters_recently_released_versions() { + let body = "v1.0.0\nv2.0.0\n"; + let list = make_released_packages(&[ + ("github.com/gorilla/mux", "1.0.0", 72), // old — keep + ("github.com/gorilla/mux", "2.0.0", 1), // new — suppress + ]); + + let result = MinPackageAgeGolang::new(None) + .rewrite_list_response( + make_list_response(body), + "github.com/gorilla/mux", + &list, + default_cutoff_secs(), + ) + .await + .unwrap(); + + assert_eq!(result.headers().get("cache-control").unwrap(), "no-cache"); + let text = result.try_into_string().await.unwrap(); + assert!(text.contains("v1.0.0"), "old version should be kept"); + assert!(!text.contains("v2.0.0"), "new version should be suppressed"); +} + +#[tokio::test] +async fn passthrough_when_nothing_filtered() { + let body = "v1.0.0\nv1.1.0\n"; + let list = make_released_packages(&[ + ("github.com/gorilla/mux", "1.0.0", 72), + ("github.com/gorilla/mux", "1.1.0", 96), + ]); + let resp = make_list_response_with_cache(body); + + let result = MinPackageAgeGolang::new(None) + .rewrite_list_response(resp, "github.com/gorilla/mux", &list, default_cutoff_secs()) + .await + .unwrap(); + + // cache headers must be preserved when nothing was filtered + assert_eq!(result.headers().get("etag").unwrap(), "abc123"); + assert_eq!( + result.headers().get("cache-control").unwrap(), + "public, max-age=60" + ); + let text = result.try_into_string().await.unwrap(); + assert!(text.contains("v1.0.0")); + assert!(text.contains("v1.1.0")); +} + +#[tokio::test] +async fn filters_deep_multi_segment_module_path() { + // Mirrors a real malware-list entry: + // { "package_name": "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/es", + // "version": "v1.3.84", "released_on": 1776804062 } + let body = "v1.3.83\nv1.3.84\n"; + let list = make_released_packages(&[ + ( + "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/es", + "1.3.83", + 72, + ), + ( + "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/es", + "1.3.84", + 1, + ), + ]); + + let result = MinPackageAgeGolang::new(None) + .rewrite_list_response( + make_list_response(body), + "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/es", + &list, + default_cutoff_secs(), + ) + .await + .unwrap(); + + let text = result.try_into_string().await.unwrap(); + assert!(text.contains("v1.3.83"), "old version should be kept"); + assert!(!text.contains("v1.3.84"), "new version should be suppressed"); +} + +#[tokio::test] +async fn keeps_unparseable_version_lines() { + let body = "v1.0.0\nnot-a-version\n"; + let list = make_released_packages(&[("github.com/gorilla/mux", "1.0.0", 72)]); + + let result = MinPackageAgeGolang::new(None) + .rewrite_list_response( + make_list_response(body), + "github.com/gorilla/mux", + &list, + default_cutoff_secs(), + ) + .await + .unwrap(); + + let text = result.try_into_string().await.unwrap(); + assert!(text.contains("v1.0.0")); + assert!(text.contains("not-a-version"), "unparseable lines must be kept"); +} + +#[tokio::test] +async fn strips_cache_headers_only_when_modified() { + let body = "v1.0.0\nv2.0.0\n"; + let list = make_released_packages(&[ + ("github.com/gorilla/mux", "1.0.0", 72), + ("github.com/gorilla/mux", "2.0.0", 1), + ]); + let resp = make_list_response_with_cache(body); + + let result = MinPackageAgeGolang::new(None) + .rewrite_list_response(resp, "github.com/gorilla/mux", &list, default_cutoff_secs()) + .await + .unwrap(); + + assert!(result.headers().get("etag").is_none(), "etag should be stripped"); + assert!( + result.headers().get("last-modified").is_none(), + "last-modified should be stripped" + ); + assert_eq!(result.headers().get("cache-control").unwrap(), "no-cache"); +} From d87190f022d37998c5976ceb605827ac8f04ae0f Mon Sep 17 00:00:00 2001 From: bitterpanda Date: Wed, 22 Apr 2026 01:32:51 +0200 Subject: [PATCH 08/19] golang: Add match_http_response_payload_inspection_request --- proxy-lib/src/http/firewall/mod.rs | 6 +- .../src/http/firewall/rule/golang/mod.rs | 58 +++++++++++++++++-- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/proxy-lib/src/http/firewall/mod.rs b/proxy-lib/src/http/firewall/mod.rs index d7a5a065..b7ba8b8c 100644 --- a/proxy-lib/src/http/firewall/mod.rs +++ b/proxy-lib/src/http/firewall/mod.rs @@ -48,7 +48,10 @@ use crate::{ remote_app_passthrough_list::RemoteAppPassthroughList, }, http::firewall::rule::{ - DynRule, npm::min_package_age::MinPackageAge, pypi::min_package_age::MinPackageAgePyPI, + DynRule, + golang::min_package_age::MinPackageAgeGolang, + npm::min_package_age::MinPackageAge, + pypi::min_package_age::MinPackageAgePyPI, }, storage::SyncCompactDataStorage, utils::{env::network_service_identifier, token::AgentIdentity}, @@ -268,6 +271,7 @@ impl Firewall { data.clone(), policy_evaluator.clone(), remote_endpoint_config.clone(), + Some(MinPackageAgeGolang::new(notifier.clone())), ) .await .context("create block rule: golang")? diff --git a/proxy-lib/src/http/firewall/rule/golang/mod.rs b/proxy-lib/src/http/firewall/rule/golang/mod.rs index dee1916d..bc1cf3b9 100644 --- a/proxy-lib/src/http/firewall/rule/golang/mod.rs +++ b/proxy-lib/src/http/firewall/rule/golang/mod.rs @@ -1,11 +1,12 @@ -use std::{fmt, str::FromStr}; +use std::fmt; use rama::utils::time::now_unix_ms; use rama::{ Service, error::{BoxError, ErrorContext as _, extra::OpaqueError}, + extensions::ExtensionsRef as _, graceful::ShutdownGuard, - http::{Request, Response, Uri}, + http::{Request, Response, StatusCode, Uri}, net::address::Domain, telemetry::tracing, utils::str::arcstr::{ArcStr, arcstr}, @@ -16,11 +17,12 @@ use crate::{ http::firewall::{ domain_matcher::DomainMatcher, events::{Artifact, BlockReason}, + layer::OriginalRequestUri, }, package::{ malware_list::{LowerCaseEntryFormatter, RemoteMalwareList}, released_packages_list::{LowerCaseReleasedPackageFormatter, RemoteReleasedPackagesList}, - version::{PackageVersion, PragmaticSemver}, + version::PackageVersion, }, storage::SyncCompactDataStorage, }; @@ -28,7 +30,15 @@ use crate::{ #[cfg(feature = "pac")] use crate::http::firewall::pac::PacScriptGenerator; -use super::{BlockedRequest, RequestAction, Rule}; +use super::{ + BlockedRequest, HttpRequestMatcherView, HttpResponseMatcherView, RequestAction, Rule, +}; + +mod parser; +use parser::{GoPackage, is_zip_download, parse_package_from_path}; + +pub mod min_package_age; +use min_package_age::MinPackageAgeGolang; #[cfg(test)] mod tests; @@ -39,6 +49,7 @@ pub(in crate::http::firewall) struct RuleGolang { remote_released_packages_list: RemoteReleasedPackagesList, remote_endpoint_config: Option, policy_evaluator: Option, + maybe_min_package_age: Option, } impl RuleGolang { @@ -48,6 +59,7 @@ impl RuleGolang { sync_storage: SyncCompactDataStorage, policy_evaluator: Option, remote_endpoint_config: Option, + min_package_age: Option, ) -> Result where C: Service + Clone, @@ -78,6 +90,7 @@ impl RuleGolang { remote_released_packages_list, remote_endpoint_config, policy_evaluator, + maybe_min_package_age: min_package_age, }) } } @@ -109,6 +122,43 @@ impl Rule for RuleGolang { } self.evaluate_zip_request(req).await } + + fn match_http_response_payload_inspection_request( + &self, + req: HttpRequestMatcherView<'_>, + ) -> bool { + self.maybe_min_package_age.is_some() + && parser::parse_module_from_list_path(req.uri.path()).is_some() + } + + fn match_http_response_payload_inspection_response( + &self, + resp: HttpResponseMatcherView<'_>, + ) -> bool { + resp.status == StatusCode::OK + } + + async fn evaluate_response(&self, resp: Response) -> Result { + let Some(min_package_age) = &self.maybe_min_package_age else { + return Ok(resp); + }; + let Some(OriginalRequestUri(uri)) = + resp.extensions().get::().cloned() + else { + return Ok(resp); + }; + let Some(module_name) = parser::parse_module_from_list_path(uri.path()) else { + return Ok(resp); + }; + min_package_age + .rewrite_list_response( + resp, + &module_name, + &self.remote_released_packages_list, + self.get_package_age_cutoff_secs(), + ) + .await + } } impl RuleGolang { From 0368f6f64a1c89feb77b404d88a8a1ad37358e90 Mon Sep 17 00:00:00 2001 From: bitterpanda Date: Wed, 22 Apr 2026 02:09:34 +0200 Subject: [PATCH 09/19] use make_response_uncacheable & update to use req_uri --- .../firewall/rule/golang/min_package_age/mod.rs | 15 ++------------- proxy-lib/src/http/firewall/rule/golang/mod.rs | 11 ++--------- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/proxy-lib/src/http/firewall/rule/golang/min_package_age/mod.rs b/proxy-lib/src/http/firewall/rule/golang/min_package_age/mod.rs index fe9907ff..edd1d3cf 100644 --- a/proxy-lib/src/http/firewall/rule/golang/min_package_age/mod.rs +++ b/proxy-lib/src/http/firewall/rule/golang/min_package_age/mod.rs @@ -2,15 +2,7 @@ use std::str::FromStr; use rama::{ error::{BoxError, ErrorContext as _}, - http::{ - Body, Response, - body::util::BodyExt as _, - headers::{CacheControl, HeaderMapExt as _}, - layer::remove_header::{ - remove_cache_policy_headers, remove_cache_validation_response_headers, - remove_payload_metadata_headers, - }, - }, + http::{Body, Response, body::util::BodyExt as _}, telemetry::tracing, utils::{str::arcstr::ArcStr, time::now_unix_ms}, }; @@ -97,10 +89,7 @@ impl MinPackageAgeGolang { "Go module list rewritten: suppressed too-young versions" ); - remove_cache_policy_headers(&mut parts.headers); - remove_cache_validation_response_headers(&mut parts.headers); - remove_payload_metadata_headers(&mut parts.headers); - parts.headers.typed_insert(CacheControl::new().with_no_cache()); + super::super::make_response_uncacheable(&mut parts.headers); let new_body = kept.join("\n") + "\n"; diff --git a/proxy-lib/src/http/firewall/rule/golang/mod.rs b/proxy-lib/src/http/firewall/rule/golang/mod.rs index bc1cf3b9..a6742e28 100644 --- a/proxy-lib/src/http/firewall/rule/golang/mod.rs +++ b/proxy-lib/src/http/firewall/rule/golang/mod.rs @@ -4,7 +4,6 @@ use rama::utils::time::now_unix_ms; use rama::{ Service, error::{BoxError, ErrorContext as _, extra::OpaqueError}, - extensions::ExtensionsRef as _, graceful::ShutdownGuard, http::{Request, Response, StatusCode, Uri}, net::address::Domain, @@ -17,7 +16,6 @@ use crate::{ http::firewall::{ domain_matcher::DomainMatcher, events::{Artifact, BlockReason}, - layer::OriginalRequestUri, }, package::{ malware_list::{LowerCaseEntryFormatter, RemoteMalwareList}, @@ -138,16 +136,11 @@ impl Rule for RuleGolang { resp.status == StatusCode::OK } - async fn evaluate_response(&self, resp: Response) -> Result { + async fn evaluate_response(&self, resp: Response, req_uri: &Uri) -> Result { let Some(min_package_age) = &self.maybe_min_package_age else { return Ok(resp); }; - let Some(OriginalRequestUri(uri)) = - resp.extensions().get::().cloned() - else { - return Ok(resp); - }; - let Some(module_name) = parser::parse_module_from_list_path(uri.path()) else { + let Some(module_name) = parser::parse_module_from_list_path(req_uri.path()) else { return Ok(resp); }; min_package_age From 327d7d74d217cdc1173d8e7e0695832a988b42f0 Mon Sep 17 00:00:00 2001 From: bitterpanda Date: Wed, 22 Apr 2026 02:19:27 +0200 Subject: [PATCH 10/19] delete empty tests.rs file --- proxy-lib/src/http/firewall/rule/golang/mod.rs | 3 --- proxy-lib/src/http/firewall/rule/golang/tests.rs | 1 - 2 files changed, 4 deletions(-) delete mode 100644 proxy-lib/src/http/firewall/rule/golang/tests.rs diff --git a/proxy-lib/src/http/firewall/rule/golang/mod.rs b/proxy-lib/src/http/firewall/rule/golang/mod.rs index a6742e28..d60e9164 100644 --- a/proxy-lib/src/http/firewall/rule/golang/mod.rs +++ b/proxy-lib/src/http/firewall/rule/golang/mod.rs @@ -38,9 +38,6 @@ use parser::{GoPackage, is_zip_download, parse_package_from_path}; pub mod min_package_age; use min_package_age::MinPackageAgeGolang; -#[cfg(test)] -mod tests; - pub(in crate::http::firewall) struct RuleGolang { target_domains: DomainMatcher, remote_malware_list: RemoteMalwareList, diff --git a/proxy-lib/src/http/firewall/rule/golang/tests.rs b/proxy-lib/src/http/firewall/rule/golang/tests.rs deleted file mode 100644 index 0be6e3c7..00000000 --- a/proxy-lib/src/http/firewall/rule/golang/tests.rs +++ /dev/null @@ -1 +0,0 @@ -// Tests for URL parsing are in parser_tests.rs (referenced from parser.rs). From 2908677b8edfbda056164d2c475adfdc5edd2a26 Mon Sep 17 00:00:00 2001 From: bitterpanda Date: Wed, 22 Apr 2026 02:19:48 +0200 Subject: [PATCH 11/19] Use rama's percent_encoding instead of our own --- .../src/http/firewall/rule/golang/parser.rs | 33 ++----------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/proxy-lib/src/http/firewall/rule/golang/parser.rs b/proxy-lib/src/http/firewall/rule/golang/parser.rs index 324cd778..5676af07 100644 --- a/proxy-lib/src/http/firewall/rule/golang/parser.rs +++ b/proxy-lib/src/http/firewall/rule/golang/parser.rs @@ -1,6 +1,6 @@ use std::str::FromStr; -use rama::telemetry::tracing; +use rama::{net::uri::util::percent_encoding, telemetry::tracing}; use crate::package::version::PragmaticSemver; @@ -78,40 +78,11 @@ pub(super) fn parse_module_from_list_path(path: &str) -> Option { /// 2. Module-unescape (`!x` → uppercase `X`, per `golang.org/x/mod/module.UnescapePath`) /// Then lowercases the result for consistent malware-list lookup. fn normalize_module_path(raw: &str) -> String { - let percent_decoded = percent_decode(raw); + let percent_decoded = percent_encoding::percent_decode_str(raw).decode_utf8_lossy(); let unescaped = go_module_unescape(&percent_decoded); unescaped.to_ascii_lowercase() } -/// Decodes percent-encoded characters in a URL path segment. -/// Only handles the ASCII subset relevant to Go module paths (`%21` → `!`, etc.). -fn percent_decode(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - let bytes = s.as_bytes(); - let mut i = 0; - while i < bytes.len() { - if bytes[i] == b'%' && i + 2 < bytes.len() { - if let (Some(h), Some(l)) = (hex_val(bytes[i + 1]), hex_val(bytes[i + 2])) { - out.push((h << 4 | l) as char); - i += 3; - continue; - } - } - out.push(bytes[i] as char); - i += 1; - } - out -} - -fn hex_val(b: u8) -> Option { - match b { - b'0'..=b'9' => Some(b - b'0'), - b'a'..=b'f' => Some(b - b'a' + 10), - b'A'..=b'F' => Some(b - b'A' + 10), - _ => None, - } -} - #[cfg(test)] #[path = "parser_tests.rs"] mod tests; From 7108ddeb04537fec77140ec35edfcaf27c5b7dd7 Mon Sep 17 00:00:00 2001 From: bitterpanda Date: Wed, 22 Apr 2026 02:21:41 +0200 Subject: [PATCH 12/19] simplify parser.rs comments --- .../src/http/firewall/rule/golang/parser.rs | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/proxy-lib/src/http/firewall/rule/golang/parser.rs b/proxy-lib/src/http/firewall/rule/golang/parser.rs index 5676af07..6a10db5a 100644 --- a/proxy-lib/src/http/firewall/rule/golang/parser.rs +++ b/proxy-lib/src/http/firewall/rule/golang/parser.rs @@ -17,20 +17,9 @@ pub(super) fn is_zip_download(path: &str) -> bool { path.ends_with(".zip") && path.contains("/@v/") } -/// Parses a Go module proxy zip URL path into a normalized module name and version. -/// -/// Expected path shape: `/{module_path}/@v/{version}.zip` -/// -/// Go module paths go through two layers of encoding in proxy URLs: -/// 1. Module escaping (`golang.org/x/mod/module.EscapePath`): each uppercase letter -/// becomes `!` + its lowercase equivalent — e.g. `AikidoSec` → `!aikido!sec`. -/// 2. Percent-encoding of `!` in the URL: `!` → `%21`. -/// -/// So `github.com/AikidoSec/firewall-go` appears in the URL as -/// `github.com/%21aikido%21sec/firewall-go`. -/// -/// We reverse both layers then lowercase for malware-list lookup -/// (which uses `LowerCaseEntryFormatter`). +/// Parses a Go module proxy zip URL path (`/{module}/@v/{version}.zip`) into a normalized +/// module name and version. Reverses both encoding layers (percent-encoding + Go module escaping) +/// and lowercases for malware-list lookup. pub(super) fn parse_package_from_path(path: &str) -> Option { let path = path.trim_matches('/'); From ce94c5b5a1402ecfe2881810234271e9dab6dcad Mon Sep 17 00:00:00 2001 From: bitterpanda Date: Wed, 22 Apr 2026 03:26:52 +0200 Subject: [PATCH 13/19] linting for golang pr --- proxy-lib/src/http/firewall/mod.rs | 4 +--- .../firewall/rule/golang/min_package_age/tests.rs | 15 ++++++++++++--- proxy-lib/src/http/firewall/rule/golang/mod.rs | 4 +--- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/proxy-lib/src/http/firewall/mod.rs b/proxy-lib/src/http/firewall/mod.rs index b7ba8b8c..8635f2ae 100644 --- a/proxy-lib/src/http/firewall/mod.rs +++ b/proxy-lib/src/http/firewall/mod.rs @@ -48,9 +48,7 @@ use crate::{ remote_app_passthrough_list::RemoteAppPassthroughList, }, http::firewall::rule::{ - DynRule, - golang::min_package_age::MinPackageAgeGolang, - npm::min_package_age::MinPackageAge, + DynRule, golang::min_package_age::MinPackageAgeGolang, npm::min_package_age::MinPackageAge, pypi::min_package_age::MinPackageAgePyPI, }, storage::SyncCompactDataStorage, diff --git a/proxy-lib/src/http/firewall/rule/golang/min_package_age/tests.rs b/proxy-lib/src/http/firewall/rule/golang/min_package_age/tests.rs index da1f3909..3f0c866c 100644 --- a/proxy-lib/src/http/firewall/rule/golang/min_package_age/tests.rs +++ b/proxy-lib/src/http/firewall/rule/golang/min_package_age/tests.rs @@ -126,7 +126,10 @@ async fn filters_deep_multi_segment_module_path() { let text = result.try_into_string().await.unwrap(); assert!(text.contains("v1.3.83"), "old version should be kept"); - assert!(!text.contains("v1.3.84"), "new version should be suppressed"); + assert!( + !text.contains("v1.3.84"), + "new version should be suppressed" + ); } #[tokio::test] @@ -146,7 +149,10 @@ async fn keeps_unparseable_version_lines() { let text = result.try_into_string().await.unwrap(); assert!(text.contains("v1.0.0")); - assert!(text.contains("not-a-version"), "unparseable lines must be kept"); + assert!( + text.contains("not-a-version"), + "unparseable lines must be kept" + ); } #[tokio::test] @@ -163,7 +169,10 @@ async fn strips_cache_headers_only_when_modified() { .await .unwrap(); - assert!(result.headers().get("etag").is_none(), "etag should be stripped"); + assert!( + result.headers().get("etag").is_none(), + "etag should be stripped" + ); assert!( result.headers().get("last-modified").is_none(), "last-modified should be stripped" diff --git a/proxy-lib/src/http/firewall/rule/golang/mod.rs b/proxy-lib/src/http/firewall/rule/golang/mod.rs index d60e9164..47f6920e 100644 --- a/proxy-lib/src/http/firewall/rule/golang/mod.rs +++ b/proxy-lib/src/http/firewall/rule/golang/mod.rs @@ -28,9 +28,7 @@ use crate::{ #[cfg(feature = "pac")] use crate::http::firewall::pac::PacScriptGenerator; -use super::{ - BlockedRequest, HttpRequestMatcherView, HttpResponseMatcherView, RequestAction, Rule, -}; +use super::{BlockedRequest, HttpRequestMatcherView, HttpResponseMatcherView, RequestAction, Rule}; mod parser; use parser::{GoPackage, is_zip_download, parse_package_from_path}; From 680cb89dcc5722d66d29cb87e867ecb58f11556c Mon Sep 17 00:00:00 2001 From: bitterpanda Date: Wed, 22 Apr 2026 03:52:48 +0200 Subject: [PATCH 14/19] Fix clippy issues --- .../http/firewall/rule/golang/module_escape.rs | 15 +++++++-------- proxy-lib/src/http/firewall/rule/golang/parser.rs | 1 + 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/proxy-lib/src/http/firewall/rule/golang/module_escape.rs b/proxy-lib/src/http/firewall/rule/golang/module_escape.rs index 333803a2..2bb7b982 100644 --- a/proxy-lib/src/http/firewall/rule/golang/module_escape.rs +++ b/proxy-lib/src/http/firewall/rule/golang/module_escape.rs @@ -8,14 +8,13 @@ pub(super) fn go_module_unescape(s: &str) -> String { let mut out = String::with_capacity(s.len()); let mut chars = s.chars().peekable(); while let Some(c) = chars.next() { - if c == '!' { - if let Some(&next) = chars.peek() { - if next.is_ascii_lowercase() { - out.push(next.to_ascii_uppercase()); - chars.next(); - continue; - } - } + if c == '!' + && let Some(&next) = chars.peek() + && next.is_ascii_lowercase() + { + out.push(next.to_ascii_uppercase()); + chars.next(); + continue; } out.push(c); } diff --git a/proxy-lib/src/http/firewall/rule/golang/parser.rs b/proxy-lib/src/http/firewall/rule/golang/parser.rs index 6a10db5a..78f9ce4d 100644 --- a/proxy-lib/src/http/firewall/rule/golang/parser.rs +++ b/proxy-lib/src/http/firewall/rule/golang/parser.rs @@ -65,6 +65,7 @@ pub(super) fn parse_module_from_list_path(path: &str) -> Option { /// Reverses both encoding layers applied by the Go toolchain: /// 1. Percent-decode (`%21` → `!`) /// 2. Module-unescape (`!x` → uppercase `X`, per `golang.org/x/mod/module.UnescapePath`) +/// /// Then lowercases the result for consistent malware-list lookup. fn normalize_module_path(raw: &str) -> String { let percent_decoded = percent_encoding::percent_decode_str(raw).decode_utf8_lossy(); From 6cc6de7088582c309186cf6f49d321af41636962 Mon Sep 17 00:00:00 2001 From: bitterpanda Date: Thu, 23 Apr 2026 13:28:42 +0200 Subject: [PATCH 15/19] Merge update after huge refactor --- proxy-lib/src/http/firewall/mod.rs | 6 +- .../rule/golang/min_package_age/mod.rs | 27 +++--- .../src/http/firewall/rule/golang/mod.rs | 88 ++++++++++--------- 3 files changed, 65 insertions(+), 56 deletions(-) diff --git a/proxy-lib/src/http/firewall/mod.rs b/proxy-lib/src/http/firewall/mod.rs index 12b9eef9..0b169e56 100644 --- a/proxy-lib/src/http/firewall/mod.rs +++ b/proxy-lib/src/http/firewall/mod.rs @@ -49,7 +49,10 @@ use crate::{ }, http::firewall::{ notifier::EventNotifier, - rule::{DynRule, npm::min_package_age::MinPackageAge}, + rule::{ + DynRule, golang::min_package_age::MinPackageAgeGolang, + npm::min_package_age::MinPackageAge, + }, }, storage::SyncCompactDataStorage, utils::{env::network_service_identifier, token::AgentIdentity}, @@ -220,7 +223,6 @@ impl Firewall { guard.clone(), layered_client.clone(), data.clone(), - policy_evaluator.clone(), remote_endpoint_config.clone(), Some(MinPackageAgeGolang::new(notifier.clone())), ) diff --git a/proxy-lib/src/http/firewall/rule/golang/min_package_age/mod.rs b/proxy-lib/src/http/firewall/rule/golang/min_package_age/mod.rs index edd1d3cf..52664979 100644 --- a/proxy-lib/src/http/firewall/rule/golang/min_package_age/mod.rs +++ b/proxy-lib/src/http/firewall/rule/golang/min_package_age/mod.rs @@ -4,18 +4,22 @@ use rama::{ error::{BoxError, ErrorContext as _}, http::{Body, Response, body::util::BodyExt as _}, telemetry::tracing, - utils::{str::arcstr::ArcStr, time::now_unix_ms}, + utils::str::arcstr::ArcStr, }; use crate::{ - http::firewall::{ - events::{Artifact, MinPackageAgeEvent}, - notifier::EventNotifier, + http::{ + firewall::{ + events::{Artifact, MinPackageAgeEvent}, + notifier::EventNotifier, + }, + headers::make_response_uncacheable, }, package::{ released_packages_list::RemoteReleasedPackagesList, version::{PackageVersion, PragmaticSemver}, }, + utils::time::SystemTimestampMilliseconds, }; #[derive(Debug, Clone)] @@ -32,8 +36,8 @@ impl MinPackageAgeGolang { &self, resp: Response, module_name: &str, - released_packages: &RemoteReleasedPackagesList, - cutoff_secs: i64, + released_packages: &RemoteReleasedPackagesList, + cutoff_ts: SystemTimestampMilliseconds, ) -> Result { let (mut parts, body) = resp.into_parts(); @@ -63,11 +67,8 @@ impl MinPackageAgeGolang { match PragmaticSemver::from_str(version_str) { Ok(parsed) => { let version = PackageVersion::Semver(parsed); - if released_packages.is_recently_released( - module_name, - Some(&version), - cutoff_secs, - ) { + let name = super::GolangPackageName::from(module_name); + if released_packages.is_recently_released(&name, Some(&version), cutoff_ts) { suppressed.push(version); } else { kept.push(line); @@ -89,7 +90,7 @@ impl MinPackageAgeGolang { "Go module list rewritten: suppressed too-young versions" ); - super::super::make_response_uncacheable(&mut parts.headers); + make_response_uncacheable(&mut parts.headers); let new_body = kept.join("\n") + "\n"; @@ -104,7 +105,7 @@ impl MinPackageAgeGolang { }; let identifier: ArcStr = module_name.into(); let event = MinPackageAgeEvent { - ts_ms: now_unix_ms(), + ts_ms: SystemTimestampMilliseconds::now(), artifact: Artifact { product: "golang".into(), identifier: identifier.clone(), diff --git a/proxy-lib/src/http/firewall/rule/golang/mod.rs b/proxy-lib/src/http/firewall/rule/golang/mod.rs index 47f6920e..19707ff9 100644 --- a/proxy-lib/src/http/firewall/rule/golang/mod.rs +++ b/proxy-lib/src/http/firewall/rule/golang/mod.rs @@ -1,9 +1,9 @@ use std::fmt; -use rama::utils::time::now_unix_ms; use rama::{ Service, error::{BoxError, ErrorContext as _, extra::OpaqueError}, + extensions::ExtensionsRef as _, graceful::ShutdownGuard, http::{Request, Response, StatusCode, Uri}, net::address::Domain, @@ -12,19 +12,27 @@ use rama::{ }; use crate::{ - endpoint_protection::{PackagePolicyDecision, PolicyEvaluator, RemoteEndpointConfig}, - http::firewall::{ - domain_matcher::DomainMatcher, - events::{Artifact, BlockReason}, + endpoint_protection::{ + EcosystemKey, PackagePolicyDecision, PolicyEvaluator, RemoteEndpointConfig, + }, + http::{ + RequestMetaUri, + firewall::{ + domain_matcher::DomainMatcher, + events::{Artifact, BlockReason}, + }, }, package::{ - malware_list::{LowerCaseEntryFormatter, RemoteMalwareList}, - released_packages_list::{LowerCaseReleasedPackageFormatter, RemoteReleasedPackagesList}, - version::PackageVersion, + malware_list::RemoteMalwareList, name_formatter::LowerCasePackageName, + released_packages_list::RemoteReleasedPackagesList, version::PackageVersion, }, storage::SyncCompactDataStorage, + utils::time::{SystemDuration, SystemTimestampMilliseconds}, }; +type GolangPackageName = LowerCasePackageName; +const GOLANG_ECOSYSTEM_KEY: EcosystemKey = EcosystemKey::from_static("golang"); + #[cfg(feature = "pac")] use crate::http::firewall::pac::PacScriptGenerator; @@ -38,10 +46,9 @@ use min_package_age::MinPackageAgeGolang; pub(in crate::http::firewall) struct RuleGolang { target_domains: DomainMatcher, - remote_malware_list: RemoteMalwareList, - remote_released_packages_list: RemoteReleasedPackagesList, - remote_endpoint_config: Option, - policy_evaluator: Option, + remote_malware_list: RemoteMalwareList, + remote_released_packages_list: RemoteReleasedPackagesList, + policy_evaluator: Option>, maybe_min_package_age: Option, } @@ -50,7 +57,6 @@ impl RuleGolang { guard: ShutdownGuard, remote_malware_list_https_client: C, sync_storage: SyncCompactDataStorage, - policy_evaluator: Option, remote_endpoint_config: Option, min_package_age: Option, ) -> Result @@ -62,26 +68,27 @@ impl RuleGolang { Uri::from_static("https://malware-list.aikido.dev/malware_golang.json"), sync_storage.clone(), remote_malware_list_https_client.clone(), - LowerCaseEntryFormatter, ) .await .context("create remote malware list for golang block rule")?; let remote_released_packages_list = RemoteReleasedPackagesList::try_new( - guard, + guard.clone(), Uri::from_static("https://malware-list.aikido.dev/releases/golang.json"), sync_storage, remote_malware_list_https_client, - LowerCaseReleasedPackageFormatter, ) .await .context("create remote released packages list for golang block rule")?; + let policy_evaluator = remote_endpoint_config + .clone() + .map(|config| PolicyEvaluator::new(guard, GOLANG_ECOSYSTEM_KEY.clone(), config)); + Ok(Self { target_domains: ["proxy.golang.org"].into_iter().collect(), remote_malware_list, remote_released_packages_list, - remote_endpoint_config, policy_evaluator, maybe_min_package_age: min_package_age, }) @@ -131,11 +138,15 @@ impl Rule for RuleGolang { resp.status == StatusCode::OK } - async fn evaluate_response(&self, resp: Response, req_uri: &Uri) -> Result { + async fn evaluate_response(&self, resp: Response) -> Result { let Some(min_package_age) = &self.maybe_min_package_age else { return Ok(resp); }; - let Some(module_name) = parser::parse_module_from_list_path(req_uri.path()) else { + let Some(module_name) = resp + .extensions() + .get_ref::() + .and_then(|RequestMetaUri(uri)| parser::parse_module_from_list_path(uri.path())) + else { return Ok(resp); }; min_package_age @@ -143,25 +154,20 @@ impl Rule for RuleGolang { resp, &module_name, &self.remote_released_packages_list, - self.get_package_age_cutoff_secs(), + self.get_package_age_cutoff_ts(), ) .await } } impl RuleGolang { - const DEFAULT_MIN_PACKAGE_AGE_SECS: i64 = 48 * 3600; - - fn get_package_age_cutoff_secs(&self) -> i64 { - let maybe_ts = self.remote_endpoint_config.as_ref().and_then(|c| { - c.get_ecosystem_config("golang") - .config() - .and_then(|cfg| cfg.minimum_allowed_age_timestamp) - }); - if let Some(ts_secs) = maybe_ts { - return ts_secs; - } - (now_unix_ms() / 1000) - Self::DEFAULT_MIN_PACKAGE_AGE_SECS + const DEFAULT_MIN_PACKAGE_AGE: SystemDuration = SystemDuration::days(2); + + fn get_package_age_cutoff_ts(&self) -> SystemTimestampMilliseconds { + self.policy_evaluator + .as_ref() + .map(|c| c.package_age_cutoff_ts(Self::DEFAULT_MIN_PACKAGE_AGE)) + .unwrap_or_else(|| SystemTimestampMilliseconds::now() - Self::DEFAULT_MIN_PACKAGE_AGE) } fn blocked_artifact(package: &GoPackage) -> Artifact { @@ -174,10 +180,9 @@ impl RuleGolang { } fn is_package_listed_as_malware(&self, package: &GoPackage) -> bool { - self.remote_malware_list.has_entries_with_version( - &package.fully_qualified_name, - PackageVersion::Semver(package.version.clone()), - ) + let name = GolangPackageName::from(package.fully_qualified_name.as_str()); + self.remote_malware_list + .has_entries_with_version(&name, &PackageVersion::Semver(package.version.clone())) } async fn evaluate_zip_request(&self, req: Request) -> Result { @@ -196,8 +201,8 @@ impl RuleGolang { ); if let Some(policy_evaluator) = self.policy_evaluator.as_ref() { - let decision = - policy_evaluator.evaluate_package_install("golang", &package.fully_qualified_name); + let name = GolangPackageName::from(package.fully_qualified_name.as_str()); + let decision = policy_evaluator.evaluate_package_install(&name); match decision { PackagePolicyDecision::Allow => { @@ -223,11 +228,12 @@ impl RuleGolang { ))); } - let cutoff_secs = self.get_package_age_cutoff_secs(); + let cutoff_ts = self.get_package_age_cutoff_ts(); + let name = GolangPackageName::from(package.fully_qualified_name.as_str()); if self.remote_released_packages_list.is_recently_released( - &package.fully_qualified_name, + &name, Some(&PackageVersion::Semver(package.version.clone())), - cutoff_secs, + cutoff_ts, ) { tracing::info!( http.url.path = %path, From de66d5d9f0d5fe431577ebabfcd3b494ffd50fce Mon Sep 17 00:00:00 2001 From: bitterpanda Date: Mon, 27 Apr 2026 17:29:28 +0200 Subject: [PATCH 16/19] Fix merge --- proxy-lib/src/http/firewall/mod.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/proxy-lib/src/http/firewall/mod.rs b/proxy-lib/src/http/firewall/mod.rs index 34640b16..c44df611 100644 --- a/proxy-lib/src/http/firewall/mod.rs +++ b/proxy-lib/src/http/firewall/mod.rs @@ -52,9 +52,7 @@ use crate::{ notifier::EventNotifier, rule::{ DynRule, golang::min_package_age::MinPackageAgeGolang, - npm::min_package_age::MinPackageAge, - DynRule, npm::min_package_age::MinPackageAge, - vscode::min_package_age::MinPackageAgeVSCode, + npm::min_package_age::MinPackageAge, vscode::min_package_age::MinPackageAgeVSCode, }, }, storage::SyncCompactDataStorage, From f2b5d4cc84de8f819ce4020fab37e0eb9905310e Mon Sep 17 00:00:00 2001 From: bitterpanda Date: Mon, 27 Apr 2026 17:38:11 +0200 Subject: [PATCH 17/19] fix additonal merge conflicts with main refactors --- .../rule/golang/min_package_age/tests.rs | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/proxy-lib/src/http/firewall/rule/golang/min_package_age/tests.rs b/proxy-lib/src/http/firewall/rule/golang/min_package_age/tests.rs index 3f0c866c..a873f7d8 100644 --- a/proxy-lib/src/http/firewall/rule/golang/min_package_age/tests.rs +++ b/proxy-lib/src/http/firewall/rule/golang/min_package_age/tests.rs @@ -1,10 +1,11 @@ -use rama::{ - http::{Body, BodyExtractExt as _}, - utils::time::now_unix_ms, -}; - -use crate::package::released_packages_list::{ - LowerCaseReleasedPackageFormatter, ReleasedPackageData, RemoteReleasedPackagesList, +use rama::http::{Body, BodyExtractExt as _}; + +use crate::{ + package::{ + name_formatter::LowerCasePackageName, + released_packages_list::{ReleasedPackageData, RemoteReleasedPackagesList}, + }, + utils::time::{SystemDuration, SystemTimestampMilliseconds}, }; use super::*; @@ -26,24 +27,25 @@ fn make_list_response_with_cache(body: &str) -> Response { .unwrap() } -fn make_released_packages(entries: &[(&str, &str, u64)]) -> RemoteReleasedPackagesList { - let now_secs = now_unix_ms() / 1000; +fn make_released_packages( + entries: &[(&str, &str, u64)], +) -> RemoteReleasedPackagesList { + let now_ts = SystemTimestampMilliseconds::now(); RemoteReleasedPackagesList::from_entries_for_tests( entries .iter() .map(|(package_name, version, hours_ago)| ReleasedPackageData { package_name: (*package_name).to_owned(), version: version.parse().unwrap(), - released_on: now_secs - (*hours_ago as i64 * 3600), + released_on: now_ts - SystemDuration::hours(*hours_ago as u16), }) .collect(), - now_secs, - LowerCaseReleasedPackageFormatter, + now_ts, ) } -fn default_cutoff_secs() -> i64 { - (now_unix_ms() / 1000) - (48 * 3600) +fn default_cutoff_ts() -> SystemTimestampMilliseconds { + SystemTimestampMilliseconds::now() - SystemDuration::hours(48) } #[tokio::test] @@ -59,7 +61,7 @@ async fn filters_recently_released_versions() { make_list_response(body), "github.com/gorilla/mux", &list, - default_cutoff_secs(), + default_cutoff_ts(), ) .await .unwrap(); @@ -80,7 +82,7 @@ async fn passthrough_when_nothing_filtered() { let resp = make_list_response_with_cache(body); let result = MinPackageAgeGolang::new(None) - .rewrite_list_response(resp, "github.com/gorilla/mux", &list, default_cutoff_secs()) + .rewrite_list_response(resp, "github.com/gorilla/mux", &list, default_cutoff_ts()) .await .unwrap(); @@ -119,7 +121,7 @@ async fn filters_deep_multi_segment_module_path() { make_list_response(body), "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/es", &list, - default_cutoff_secs(), + default_cutoff_ts(), ) .await .unwrap(); @@ -142,7 +144,7 @@ async fn keeps_unparseable_version_lines() { make_list_response(body), "github.com/gorilla/mux", &list, - default_cutoff_secs(), + default_cutoff_ts(), ) .await .unwrap(); @@ -165,7 +167,7 @@ async fn strips_cache_headers_only_when_modified() { let resp = make_list_response_with_cache(body); let result = MinPackageAgeGolang::new(None) - .rewrite_list_response(resp, "github.com/gorilla/mux", &list, default_cutoff_secs()) + .rewrite_list_response(resp, "github.com/gorilla/mux", &list, default_cutoff_ts()) .await .unwrap(); From bc163ad17bd6c5623241075f85237df4cd723138 Mon Sep 17 00:00:00 2001 From: bitterpanda Date: Mon, 27 Apr 2026 17:55:59 +0200 Subject: [PATCH 18/19] Use SystemTimestamMilliseconds::MAX --- proxy-bin-l7/src/client/mock_server/malware_list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy-bin-l7/src/client/mock_server/malware_list.rs b/proxy-bin-l7/src/client/mock_server/malware_list.rs index 081953c2..84ca3ef4 100644 --- a/proxy-bin-l7/src/client/mock_server/malware_list.rs +++ b/proxy-bin-l7/src/client/mock_server/malware_list.rs @@ -247,7 +247,7 @@ async fn released_golang() -> impl IntoResponse { version: PackageVersion::Semver( PragmaticSemver::parse(FRESH_GOLANG_MODULE_VERSION).unwrap(), ), - released_on: 9_000_000_000, + released_on: SystemTimestampMilliseconds::MAX, }]) } From 4661138dd97d7fe414d5fae05d17e807463185a0 Mon Sep 17 00:00:00 2001 From: bitterpanda Date: Mon, 27 Apr 2026 18:25:24 +0200 Subject: [PATCH 19/19] Fix MAX --- proxy-bin-l7/src/client/mock_server/malware_list.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/proxy-bin-l7/src/client/mock_server/malware_list.rs b/proxy-bin-l7/src/client/mock_server/malware_list.rs index 84ca3ef4..febade99 100644 --- a/proxy-bin-l7/src/client/mock_server/malware_list.rs +++ b/proxy-bin-l7/src/client/mock_server/malware_list.rs @@ -242,12 +242,14 @@ async fn malware_open_vsx() -> impl IntoResponse { pub const FRESH_GOLANG_MODULE_NAME: &str = "github.com/fresh-org/fresh-module"; pub const FRESH_GOLANG_MODULE_VERSION: &str = "1.0.0"; async fn released_golang() -> impl IntoResponse { + // Use now() so the entry is always "recently released" relative to the default `now - 48h` + // cutoff, but NOT relative to the bypass policy's far-future cutoff (year ~2286). Json([ReleasedPackageData { package_name: FRESH_GOLANG_MODULE_NAME.to_owned(), version: PackageVersion::Semver( PragmaticSemver::parse(FRESH_GOLANG_MODULE_VERSION).unwrap(), ), - released_on: SystemTimestampMilliseconds::MAX, + released_on: SystemTimestampMilliseconds::now(), }]) }