diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index e9655d4..1d75288 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -38,11 +38,11 @@ jobs: rust: stable target: x86_64-unknown-linux-musl cross: true - - build: x86_64-macos - os: macos-latest - rust: stable - target: x86_64-apple-darwin - cross: false +# - build: x86_64-macos +# os: macos-latest +# rust: stable +# target: x86_64-apple-darwin +# cross: false # - build: aarch64-macos # os: macos-13-xlarge # rust: stable @@ -60,12 +60,10 @@ jobs: # git-scanning tests need entire git history to work fetch-depth: 0 - name: Install ${{ matrix.rust }}-${{ matrix.target }} toolchain - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@master with: - profile: minimal toolchain: ${{ matrix.rust }} - target: ${{ matrix.target }} - override: true + targets: ${{ matrix.target }} - name: Test uses: actions-rs/cargo@v1 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c4026c..828c7e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,4 +37,7 @@ ## 1.0.11 - The update from 1.0.10 caused a major bug where choctaw_hog was reporting 0 results. This was due to a change in the behavior of the Rust Git library. The change is fixed and tested on MacOS. -- Added the Slack hog - hante hog \ No newline at end of file +- Added the Slack hog - hante hog + +## 1.0.12 +- Updating Cargo edition from 2021 to 2024 \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 43eb08a..8696705 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,9 +5,9 @@ members = [ [package] name = "rusty_hogs" -version = "1.0.11" +version = "1.0.12" authors = ["Scott Cutler "] -edition = "2021" +edition = "2024" description = "This project provides a set of scanners that will use regular expressions to try and detect the presence of sensitive information such as API keys, passwords, and personal information. It includes a set of regular expressions by default, but will also accept a JSON object containing your custom regular expressions." homepage = "https://github.com/newrelic/rusty-hog" keywords = ["secret", "scanner", "regex", "rusty", "hog"] @@ -55,4 +55,4 @@ lto = true codegen-units = 1 [package.metadata.cross.target.x86_64-unknown-linux-musl] -dockerfile = "Dockerfile.lambda" +dockerfile = "Dockerfile.lambda" \ No newline at end of file diff --git a/Dockerfile.lambda b/Dockerfile.lambda index 171e91d..7c9feb5 100644 --- a/Dockerfile.lambda +++ b/Dockerfile.lambda @@ -5,7 +5,7 @@ FROM ${CROSS_BASE_IMAGE} ARG CFLAGS="" ARG LDFLAGS="" -ARG OPENSSL_BUILD_VER=3.0.12 +ARG OPENSSL_BUILD_VER=3.0.16 RUN cd /usr/local/src/ && curl -sLO https://www.openssl.org/source/openssl-${OPENSSL_BUILD_VER}.tar.gz && \ tar xzvf openssl-${OPENSSL_BUILD_VER}.tar.gz && cd openssl-${OPENSSL_BUILD_VER} && \ CROSS_COMPILE="x86_64-linux-musl-" ./Configure --prefix=/usr/local/openssl-${OPENSSL_BUILD_VER} linux-x86_64 && make && make install diff --git a/README.md b/README.md index 2dece9b..631dd3b 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,8 @@ Download and unzip the [latest ZIP](https://github.com/newrelic/rusty-hog/releas on the releases tab. Then, run each binary with `-h` to see the usage. ```shell script -wget https://github.com/newrelic/rusty-hog/releases/download/v1.0.11/rustyhogs-darwin-choctaw_hog-1.0.11.zip -unzip rustyhogs-darwin-choctaw_hog-1.0.11.zip +wget https://github.com/newrelic/rusty-hog/releases/download/v1.0.12/rustyhogs-darwin-choctaw_hog-1.0.12.zip +unzip rustyhogs-darwin-choctaw_hog-1.0.12.zip darwin_releases/choctaw_hog -h ``` @@ -61,8 +61,8 @@ Rusty Hog Docker images can be found at the authors personal DockerHub page [her A Docker Image is built for each Hog and for each release. So to use choctaw_hog you would run the following commands: ```shell script -docker pull wetfeet2000/choctaw_hog:1.0.10 -docker run -it --rm wetfeet2000/choctaw_hog:1.0.10 --help +docker pull wetfeet2000/choctaw_hog:1.0.12 +docker run -it --rm wetfeet2000/choctaw_hog:1.0.12 --help ``` ## How to build @@ -107,7 +107,7 @@ brew install rpm2cpio FiloSottile/musl-cross/musl-cross Then run `./build_lambda_macos.sh`. -The build script will build against OpenSSL 3.0.12. Use `export OPENSSL_BUILD_VER=3.0.12` to override. +The build script will build against OpenSSL 3.0.16. Use `export OPENSSL_BUILD_VER=3.0.16` to override. The build script will build against Amazon Linux kernel headers provided by their RPM; `export AMAZON_KERNEL_HEADERS_RPM_URL=...` to override where the RPM is downloaded from. (There's nothing preventing a different distribution's linux-headers RPM from being used, we just need the linux-headers to build openssl for Linux) diff --git a/build_lambda_macos.sh b/build_lambda_macos.sh index f300598..815bf31 100755 --- a/build_lambda_macos.sh +++ b/build_lambda_macos.sh @@ -5,7 +5,7 @@ if [[ "$(uname)" != "Darwin" ]]; then exit 1; fi -[ -z "${OPENSSL_BUILD_VER}" ] && OPENSSL_BUILD_VER="3.0.12" +[ -z "${OPENSSL_BUILD_VER}" ] && OPENSSL_BUILD_VER="3.0.16" [ -z "${AMAZON_KERNEL_HEADERS_RPM_URL}" ] && AMAZON_KERNEL_HEADERS_RPM_URL="http://packages.eu-central-1.amazonaws.com/2018.03/updates/adeeb554baf5/x86_64/Packages/kernel-headers-4.14.275-142.503.amzn1.x86_64.rpm" AMAZON_KERNEL_HEADERS_RPM="${AMAZON_KERNEL_HEADERS_RPM_URL##http*/}" diff --git a/crates/rusty-hog-scanner/Cargo.toml b/crates/rusty-hog-scanner/Cargo.toml index 964b533..7ab3f37 100644 --- a/crates/rusty-hog-scanner/Cargo.toml +++ b/crates/rusty-hog-scanner/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rusty_hog_scanner" -version = "0.1.0" -edition = "2021" +version = "0.1.1" +edition = "2024" [dependencies] serde = "1.0" diff --git a/crates/rusty-hog-scanner/src/lib.rs b/crates/rusty-hog-scanner/src/lib.rs index 450e9dd..f2c5aba 100644 --- a/crates/rusty-hog-scanner/src/lib.rs +++ b/crates/rusty-hog-scanner/src/lib.rs @@ -68,9 +68,9 @@ extern crate clap; use anyhow::Result; -use base64::{engine::general_purpose as Base64Engine, Engine as _}; +use base64::{Engine as _, engine::general_purpose as Base64Engine}; use clap::ArgMatches; -use log::{self, debug, error, info, LevelFilter}; +use log::{self, LevelFilter, debug, error, info}; use regex::bytes::{Match, Matches, Regex, RegexBuilder}; use serde::Serialize; use serde_derive::Deserialize; @@ -486,7 +486,10 @@ impl SecretScannerBuilder { let json_string: String = match json_string_result { Ok(s) => s, Err(e) => { - error!("Error reading allowlist JSON file, falling back to default allowlist rules: {:?}", e); + error!( + "Error reading allowlist JSON file, falling back to default allowlist rules: {:?}", + e + ); String::from(DEFAULT_ALLOWLIST_JSON) } }; @@ -557,7 +560,7 @@ impl SecretScannerBuilder { return Err(SimpleError::with( "Failed to parse the regex pattern from JSON", e, - )) + )); } }; Ok((x.0, v)) diff --git a/src/bin/ankamali_hog.rs b/src/bin/ankamali_hog.rs index 1716acf..1c64a62 100644 --- a/src/bin/ankamali_hog.rs +++ b/src/bin/ankamali_hog.rs @@ -41,7 +41,7 @@ use std::path::Path; #[tokio::main] async fn main() { let matches = Command::new("ankamali_hog") - .version("1.0.11") + .version("1.0.12") .author("Scott Cutler ") .about("Google Drive secret scanner in Rust.") .arg( @@ -152,8 +152,20 @@ async fn run(arg_matches: ArgMatches) -> Result<(), SimpleError> { let auth = drive3::oauth2::InstalledFlowAuthenticator::builder( secret, drive3::oauth2::InstalledFlowReturnMethod::HTTPRedirect, - ).build().await.unwrap(); - let mut hub = DriveHub::new(hyper::Client::builder().build(hyper_rustls::HttpsConnectorBuilder::new().with_native_roots().https_or_http().enable_http1().build()), auth); + ) + .build() + .await + .unwrap(); + let mut hub = DriveHub::new( + hyper::Client::builder().build( + hyper_rustls::HttpsConnectorBuilder::new() + .with_native_roots() + .https_or_http() + .enable_http1() + .build(), + ), + auth, + ); // get some initial info about the file let gdriveinfo = GDriveFileInfo::new(file_id, &hub).await.unwrap(); diff --git a/src/bin/berkshire_hog.rs b/src/bin/berkshire_hog.rs index 40bf138..a70a168 100644 --- a/src/bin/berkshire_hog.rs +++ b/src/bin/berkshire_hog.rs @@ -44,7 +44,7 @@ use std::collections::HashSet; /// Main entry function that uses the [clap crate](https://docs.rs/clap/2.33.0/clap/) fn main() { let matches = Command::new("berkshire_hog") - .version("1.0.11") + .version("1.0.12") .author("Scott Cutler ") .about("S3 secret hunter in Rust. Avoid bandwidth costs, run this within a VPC!") .arg(Arg::new("REGEX").long("regex").action(ArgAction::Set).help("Sets a custom regex JSON file")) diff --git a/src/bin/berkshire_hog_lambda.rs b/src/bin/berkshire_hog_lambda.rs index e8cfc50..85ddea2 100644 --- a/src/bin/berkshire_hog_lambda.rs +++ b/src/bin/berkshire_hog_lambda.rs @@ -16,8 +16,8 @@ extern crate s3; -use lambda_runtime::{handler_fn, Context, Error}; -use log::{self, warn, LevelFilter}; +use lambda_runtime::{Context, Error, handler_fn}; +use log::{self, LevelFilter, warn}; use rusty_hog_scanner::SecretScannerBuilder; use rusty_hogs::aws_scanning::{S3Finding, S3Scanner}; use s3::bucket::Bucket; diff --git a/src/bin/choctaw_hog.rs b/src/bin/choctaw_hog.rs index 65c15ba..39a0dd4 100644 --- a/src/bin/choctaw_hog.rs +++ b/src/bin/choctaw_hog.rs @@ -51,7 +51,7 @@ use rusty_hogs::git_scanning::GitScanner; /// Main entry function that uses the [clap crate](https://docs.rs/clap/2.33.0/clap/) fn main() { let matches = Command::new("choctaw_hog") - .version("1.0.11") + .version("1.0.12") .author("Scott Cutler ") .about("Git secret scanner in Rust") .arg(Arg::new("REGEX").short('r').long("regex").action(ArgAction::Set).value_name("REGEX").help("Sets a custom regex JSON file")) diff --git a/src/bin/duroc_hog.rs b/src/bin/duroc_hog.rs index 4639b72..3f7b6c4 100644 --- a/src/bin/duroc_hog.rs +++ b/src/bin/duroc_hog.rs @@ -58,7 +58,7 @@ pub struct FileFinding { pub path: String, pub reason: String, pub linenum: usize, - pub lineindextuples: Vec<(usize, usize)> + pub lineindextuples: Vec<(usize, usize)>, } const ZIPEXTENSIONS: &[&str] = &["zip"]; @@ -68,7 +68,7 @@ const GZEXTENSIONS: &[&str] = &["gz", "tgz"]; /// Main entry function that uses the [clap crate](https://docs.rs/clap/2.33.0/clap/) fn main() { let matches = Command::new("duroc_hog") - .version("1.0.11") + .version("1.0.12") .author("Scott Cutler ") .about("File system secret scanner in Rust") .arg( @@ -385,7 +385,7 @@ fn scan_bytes(input: Vec, ss: &SecretScanner, path: String) -> HashSet".parse().unwrap()); strings_found.push(result); - lineindextuples.push((m.start(),m.end())); + lineindextuples.push((m.start(), m.end())); } if !strings_found.is_empty() { findings.insert(FileFinding { @@ -393,7 +393,7 @@ fn scan_bytes(input: Vec, ss: &SecretScanner, path: String) -> HashSet Result { let dir_path = dir.path().to_str().unwrap(); diff --git a/src/bin/essex_hog.rs b/src/bin/essex_hog.rs index 37a3da4..87b51f2 100644 --- a/src/bin/essex_hog.rs +++ b/src/bin/essex_hog.rs @@ -33,16 +33,16 @@ extern crate clap; extern crate hyper; extern crate hyper_rustls; -use base64::{engine::general_purpose as Base64Engine, Engine as _}; +use base64::{Engine as _, engine::general_purpose as Base64Engine}; use clap::{Arg, ArgAction, ArgMatches, Command}; +use encoding::DecoderTrap; use encoding::all::ASCII; use encoding::types::Encoding; -use encoding::DecoderTrap; use hyper::body; use hyper::header::AUTHORIZATION; use hyper::http::Request; use hyper::http::StatusCode; -use hyper::{client, Body, Client}; +use hyper::{Body, Client, client}; use log::{self, debug, error, info}; use rusty_hog_scanner::SecretScannerBuilder; use rusty_hog_scanner::{RustyHogMatch, SecretScanner}; @@ -74,7 +74,7 @@ pub struct ConfluencePage { #[tokio::main] async fn main() { let matches = Command::new("gottingen_hog") - .version("1.0.11") + .version("1.0.12") .author("Emily Cain , Scott Cutler") .about("Confluence secret scanner in Rust.") .arg( diff --git a/src/bin/gottingen_hog.rs b/src/bin/gottingen_hog.rs index f03589f..b2a6830 100644 --- a/src/bin/gottingen_hog.rs +++ b/src/bin/gottingen_hog.rs @@ -26,16 +26,16 @@ extern crate clap; extern crate hyper; extern crate hyper_rustls; -use base64::{engine::general_purpose as Base64Engine, Engine as _}; +use base64::{Engine as _, engine::general_purpose as Base64Engine}; use clap::{Arg, ArgAction, ArgMatches, Command}; +use encoding::DecoderTrap; use encoding::all::ASCII; use encoding::types::Encoding; -use encoding::DecoderTrap; use hyper::body; use hyper::header::AUTHORIZATION; use hyper::http::Request; use hyper::http::StatusCode; -use hyper::{client, Body, Client}; +use hyper::{Body, Client, client}; use log::{self, debug, error, info}; use rusty_hog_scanner::SecretScannerBuilder; use rusty_hog_scanner::{RustyHogMatch, SecretScanner}; @@ -60,7 +60,7 @@ pub struct JiraFinding { #[tokio::main] async fn main() { let matches: ArgMatches = Command::new("gottingen_hog") - .version("1.0.11") + .version("1.0.12") .author("Emily Cain ") .about("Jira secret scanner in Rust.") .arg( diff --git a/src/bin/hante_hog.rs b/src/bin/hante_hog.rs index 122a8a4..75270b2 100644 --- a/src/bin/hante_hog.rs +++ b/src/bin/hante_hog.rs @@ -32,14 +32,14 @@ extern crate hyper; extern crate hyper_rustls; use clap::{Arg, ArgAction, ArgMatches, Command}; +use encoding::DecoderTrap; use encoding::all::ASCII; use encoding::types::Encoding; -use encoding::DecoderTrap; use hyper::body; use hyper::header::AUTHORIZATION; use hyper::http::Request; use hyper::http::StatusCode; -use hyper::{client, Body, Client, Method}; +use hyper::{Body, Client, Method, client}; use log::{self, debug, error, info}; use rusty_hog_scanner::SecretScannerBuilder; use rusty_hog_scanner::{RustyHogMatch, SecretScanner}; @@ -65,7 +65,7 @@ pub struct SlackFinding { #[tokio::main] async fn main() { let matches: ArgMatches = Command::new("hante_hog") - .version("1.0.11") + .version("1.0.12") .author("Joao Henrique Machado Silva ") .about("Slack secret scanner in Rust.") .arg( diff --git a/src/git_scanning.rs b/src/git_scanning.rs index 64981dc..a6076f6 100644 --- a/src/git_scanning.rs +++ b/src/git_scanning.rs @@ -42,7 +42,7 @@ //! assert_eq!(findings.len(), 8); //! ``` -use chrono::{DateTime}; +use chrono::DateTime; use chrono::Utc; use encoding::all::ASCII; use encoding::{DecoderTrap, Encoding}; @@ -244,7 +244,8 @@ impl GitScanner { .decode(&new_line, DecoderTrap::Ignore) .unwrap_or_else(|_| "".parse().unwrap()), date: DateTime::from_timestamp(commit.time().seconds(), 0) - .expect("Failed to parse timestamp").to_string(), + .expect("Failed to parse timestamp") + .to_string(), strings_found: secrets.clone(), path, reason: reason.clone(), diff --git a/src/google_scanning.rs b/src/google_scanning.rs index 8876475..cd75890 100644 --- a/src/google_scanning.rs +++ b/src/google_scanning.rs @@ -31,6 +31,7 @@ //! //! ```no_run //! # extern crate hyper_rustls; +//! # extern crate google_drive3 as drive3; //! # extern crate yup_oauth2 as oauth2; //! use rusty_hog_scanner::SecretScannerBuilder; //! use rusty_hogs::google_scanning::{GDriveScanner, GDriveFileInfo}; @@ -47,10 +48,13 @@ //! # let handle = rt.handle(); //! # rt.block_on(async { //! // Start with GDrive auth - based on example code from drive3 API and yup-oauth2 -//! # let secret = yup_oauth2::read_application_secret(Path::new(oauthsecretfile)) +//! # let secret = drive3::oauth2::read_application_secret(Path::new(oauthsecretfile)) //! # .await //! # .expect(oauthsecretfile); -//! # let auth = InstalledFlowAuthenticator::builder(secret, InstalledFlowReturnMethod::HTTPRedirect) +//! # let auth = drive3::oauth2::InstalledFlowAuthenticator::builder( +//! # secret, +//! # drive3::oauth2::InstalledFlowReturnMethod::HTTPRedirect +//! # ) //! # .persist_tokens_to_disk(Path::new(oauthtokenfile)) //! # .build() //! # .await @@ -174,7 +178,7 @@ impl GDriveFileInfo { return Err(SimpleError::new(format!( "failed accessing Google Metadata API {:?}", e - ))) + ))); } };