diff --git a/Cargo.lock b/Cargo.lock index 377b0d22b84..430219462be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8578,8 +8578,10 @@ dependencies = [ "camino", "chrono", "clap", + "glob", "jiff", "omicron-workspace-hack", + "rayon", "sigpipe", "uuid", ] diff --git a/dev-tools/oxlog/Cargo.toml b/dev-tools/oxlog/Cargo.toml index c556356c3f2..62ffa87ec40 100644 --- a/dev-tools/oxlog/Cargo.toml +++ b/dev-tools/oxlog/Cargo.toml @@ -12,7 +12,9 @@ anyhow.workspace = true camino.workspace = true chrono.workspace = true clap.workspace = true +glob.workspace = true jiff.workspace = true +rayon.workspace = true sigpipe.workspace = true uuid.workspace = true omicron-workspace-hack.workspace = true diff --git a/dev-tools/oxlog/src/bin/oxlog.rs b/dev-tools/oxlog/src/bin/oxlog.rs index 05e0596bf50..f984c0b56ef 100644 --- a/dev-tools/oxlog/src/bin/oxlog.rs +++ b/dev-tools/oxlog/src/bin/oxlog.rs @@ -5,11 +5,18 @@ //! Tool for discovering oxide related logfiles on sleds use clap::{ArgAction, Args, Parser, Subcommand}; +use glob::Pattern; use jiff::civil::DateTime; use jiff::tz::TimeZone; use jiff::{Span, Timestamp}; use oxlog::{DateRange, Filter, LogFile, Zones}; use std::collections::BTreeSet; +use std::num::NonZeroUsize; +use std::str::FromStr; + +/// The number of threads to given to the Rayon thread pool. +/// The default thread-per-physical core is excessive on a Gimlet. +const MAX_THREADS: usize = 12; #[derive(Debug, Parser)] #[command(version)] @@ -25,8 +32,8 @@ enum Commands { /// List logs for a given service Logs { - /// The name of the zone - zone: String, + /// The glob pattern to match against zone names + zone_glob: GlobPattern, /// The name of the service to list logs for service: Option, @@ -49,15 +56,26 @@ enum Commands { after: Option, }, - /// List the names of all services in a zone, from the perspective of oxlog. + /// List the names of all services in matching zones, from the perspective of oxlog. /// Use these names with `oxlog logs` to filter output to logs from a /// specific service. Services { - /// The name of the zone - zone: String, + /// The glob pattern to match against zone names + zone_glob: GlobPattern, }, } +#[derive(Clone, Debug)] +struct GlobPattern(Pattern); + +impl FromStr for GlobPattern { + type Err = glob::PatternError; + + fn from_str(s: &str) -> Result { + Pattern::new(s).map(GlobPattern) + } +} + #[derive(Args, Debug)] #[group(required = true, multiple = true)] struct FilterArgs { @@ -109,6 +127,12 @@ fn parse_timestamp( fn main() -> Result<(), anyhow::Error> { sigpipe::reset(); + let num_threads = std::thread::available_parallelism() + .map(NonZeroUsize::get) + .unwrap_or(MAX_THREADS) + .min(MAX_THREADS); + rayon::ThreadPoolBuilder::new().num_threads(num_threads).build_global()?; + let cli = Cli::parse(); match cli.command { @@ -118,7 +142,14 @@ fn main() -> Result<(), anyhow::Error> { } Ok(()) } - Commands::Logs { zone, service, metadata, filter, before, after } => { + Commands::Logs { + zone_glob, + service, + metadata, + filter, + before, + after, + } => { let zones = Zones::load()?; let date_range = match (before, after) { (None, None) => None, @@ -145,44 +176,46 @@ fn main() -> Result<(), anyhow::Error> { ); }; - let logs = zones.zone_logs(&zone, filter); - for (svc_name, svc_logs) in logs { - if let Some(service) = &service { - if svc_name != service.as_str() { - continue; + let zones = zones.matching_zone_logs(&zone_glob.0, filter); + for logs in zones { + for (svc_name, svc_logs) in logs { + if let Some(service) = &service { + if svc_name != service.as_str() { + continue; + } } - } - if filter.current { - if let Some(current) = &svc_logs.current { - if metadata { - print_metadata(current); - } else { - println!("{}", current.path); + if filter.current { + if let Some(current) = &svc_logs.current { + if metadata { + print_metadata(current); + } else { + println!("{}", current.path); + } } } - } - if filter.archived { - for f in &svc_logs.archived { - if metadata { - print_metadata(f); - } else { - println!("{}", f.path); + if filter.archived { + for f in &svc_logs.archived { + if metadata { + print_metadata(f); + } else { + println!("{}", f.path); + } } } - } - if filter.extra { - for f in &svc_logs.extra { - if metadata { - print_metadata(f); - } else { - println!("{}", f.path); + if filter.extra { + for f in &svc_logs.extra { + if metadata { + print_metadata(f); + } else { + println!("{}", f.path); + } } } } } Ok(()) } - Commands::Services { zone } => { + Commands::Services { zone_glob } => { let zones = Zones::load()?; // We want all logs that exist, anywhere, so we can find their @@ -197,8 +230,11 @@ fn main() -> Result<(), anyhow::Error> { // Collect a unique set of services, based on the logs in the // specified zone - let services: BTreeSet = - zones.zone_logs(&zone, filter).into_keys().collect(); + let services: BTreeSet = zones + .matching_zone_logs(&zone_glob.0, filter) + .into_iter() + .flat_map(|l| l.into_keys()) + .collect(); for svc in services { println!("{}", svc); diff --git a/dev-tools/oxlog/src/lib.rs b/dev-tools/oxlog/src/lib.rs index 6edb49a4766..584bc1331e4 100644 --- a/dev-tools/oxlog/src/lib.rs +++ b/dev-tools/oxlog/src/lib.rs @@ -8,7 +8,9 @@ use anyhow::Context; use camino::{Utf8DirEntry, Utf8Path, Utf8PathBuf}; +use glob::Pattern; use jiff::Timestamp; +use rayon::prelude::*; use std::collections::BTreeMap; use std::io; use uuid::Uuid; @@ -186,8 +188,8 @@ impl SvcLogs { /// scattered across several different directories -- and we care more /// about filename than which directory they are in. pub fn sort_by_file_name(&mut self) { - self.archived.sort_unstable_by(LogFile::file_name_cmp); - self.extra.sort_unstable_by(LogFile::file_name_cmp); + self.archived.par_sort_unstable_by(LogFile::file_name_cmp); + self.extra.par_sort_unstable_by(LogFile::file_name_cmp); } } @@ -347,9 +349,21 @@ impl Zones { } sort_logs(&mut output); - output } + + /// Return log files for all zones whose names match `zone_pattern` + pub fn matching_zone_logs( + &self, + zone_pattern: &Pattern, + filter: Filter, + ) -> Vec> { + self.zones + .par_iter() + .filter(|(zone, _)| zone_pattern.matches(zone)) + .map(|(zone, _)| self.zone_logs(zone, filter)) + .collect() + } } fn sort_logs(output: &mut BTreeMap) {