From 3c07c0ef8bf9040dd3660b3f34f10c93c93055bc Mon Sep 17 00:00:00 2001 From: Will Chandler Date: Fri, 18 Apr 2025 16:12:37 -0400 Subject: [PATCH] oxlog: Glob match zone names `oxlog logs` currently requires a precise match for the zone name. This is inconvenient when attempting to search across multiple zones with similar names. For example, when searching all Crucible downstairs, or using Pilot to query Nexus logs on multiple sleds. Use the `glob` crate to take a pattern to match against the zone name for the `services` and `logs` subcommands, potentially allowing multiple zones to be returned. If more than one zone is matched, then the logs and services are sorted separately. Zones with a large number of archived logs may take an extended period to walk and sort, and when matching against multiple zones this becomes uncomfortably slow. To claw back some performance, use Rayon to find files and sort in parallel. We limit the size of its thread pool to a maximum of eight threads to avoid soaking up all available threads on a sled. On smaller machines we fall back to the number of logical threads. Example usage: $ oxlog services 'oxz_crucible_[!p]*' # All non-pantry Crucible zones $ oxlog logs 'oxz_nexus_*' --current # The nexus zone --- Cargo.lock | 2 + dev-tools/oxlog/Cargo.toml | 2 + dev-tools/oxlog/src/bin/oxlog.rs | 106 +++++++++++++++++++++---------- dev-tools/oxlog/src/lib.rs | 20 +++++- 4 files changed, 92 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 377b0d22b84..430219462be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8578,8 +8578,10 @@ dependencies = [ "camino", "chrono", "clap", + "glob", "jiff", "omicron-workspace-hack", + "rayon", "sigpipe", "uuid", ] diff --git a/dev-tools/oxlog/Cargo.toml b/dev-tools/oxlog/Cargo.toml index c556356c3f2..62ffa87ec40 100644 --- a/dev-tools/oxlog/Cargo.toml +++ b/dev-tools/oxlog/Cargo.toml @@ -12,7 +12,9 @@ anyhow.workspace = true camino.workspace = true chrono.workspace = true clap.workspace = true +glob.workspace = true jiff.workspace = true +rayon.workspace = true sigpipe.workspace = true uuid.workspace = true omicron-workspace-hack.workspace = true diff --git a/dev-tools/oxlog/src/bin/oxlog.rs b/dev-tools/oxlog/src/bin/oxlog.rs index 05e0596bf50..f984c0b56ef 100644 --- a/dev-tools/oxlog/src/bin/oxlog.rs +++ b/dev-tools/oxlog/src/bin/oxlog.rs @@ -5,11 +5,18 @@ //! Tool for discovering oxide related logfiles on sleds use clap::{ArgAction, Args, Parser, Subcommand}; +use glob::Pattern; use jiff::civil::DateTime; use jiff::tz::TimeZone; use jiff::{Span, Timestamp}; use oxlog::{DateRange, Filter, LogFile, Zones}; use std::collections::BTreeSet; +use std::num::NonZeroUsize; +use std::str::FromStr; + +/// The number of threads to given to the Rayon thread pool. +/// The default thread-per-physical core is excessive on a Gimlet. +const MAX_THREADS: usize = 12; #[derive(Debug, Parser)] #[command(version)] @@ -25,8 +32,8 @@ enum Commands { /// List logs for a given service Logs { - /// The name of the zone - zone: String, + /// The glob pattern to match against zone names + zone_glob: GlobPattern, /// The name of the service to list logs for service: Option, @@ -49,15 +56,26 @@ enum Commands { after: Option, }, - /// List the names of all services in a zone, from the perspective of oxlog. + /// List the names of all services in matching zones, from the perspective of oxlog. /// Use these names with `oxlog logs` to filter output to logs from a /// specific service. Services { - /// The name of the zone - zone: String, + /// The glob pattern to match against zone names + zone_glob: GlobPattern, }, } +#[derive(Clone, Debug)] +struct GlobPattern(Pattern); + +impl FromStr for GlobPattern { + type Err = glob::PatternError; + + fn from_str(s: &str) -> Result { + Pattern::new(s).map(GlobPattern) + } +} + #[derive(Args, Debug)] #[group(required = true, multiple = true)] struct FilterArgs { @@ -109,6 +127,12 @@ fn parse_timestamp( fn main() -> Result<(), anyhow::Error> { sigpipe::reset(); + let num_threads = std::thread::available_parallelism() + .map(NonZeroUsize::get) + .unwrap_or(MAX_THREADS) + .min(MAX_THREADS); + rayon::ThreadPoolBuilder::new().num_threads(num_threads).build_global()?; + let cli = Cli::parse(); match cli.command { @@ -118,7 +142,14 @@ fn main() -> Result<(), anyhow::Error> { } Ok(()) } - Commands::Logs { zone, service, metadata, filter, before, after } => { + Commands::Logs { + zone_glob, + service, + metadata, + filter, + before, + after, + } => { let zones = Zones::load()?; let date_range = match (before, after) { (None, None) => None, @@ -145,44 +176,46 @@ fn main() -> Result<(), anyhow::Error> { ); }; - let logs = zones.zone_logs(&zone, filter); - for (svc_name, svc_logs) in logs { - if let Some(service) = &service { - if svc_name != service.as_str() { - continue; + let zones = zones.matching_zone_logs(&zone_glob.0, filter); + for logs in zones { + for (svc_name, svc_logs) in logs { + if let Some(service) = &service { + if svc_name != service.as_str() { + continue; + } } - } - if filter.current { - if let Some(current) = &svc_logs.current { - if metadata { - print_metadata(current); - } else { - println!("{}", current.path); + if filter.current { + if let Some(current) = &svc_logs.current { + if metadata { + print_metadata(current); + } else { + println!("{}", current.path); + } } } - } - if filter.archived { - for f in &svc_logs.archived { - if metadata { - print_metadata(f); - } else { - println!("{}", f.path); + if filter.archived { + for f in &svc_logs.archived { + if metadata { + print_metadata(f); + } else { + println!("{}", f.path); + } } } - } - if filter.extra { - for f in &svc_logs.extra { - if metadata { - print_metadata(f); - } else { - println!("{}", f.path); + if filter.extra { + for f in &svc_logs.extra { + if metadata { + print_metadata(f); + } else { + println!("{}", f.path); + } } } } } Ok(()) } - Commands::Services { zone } => { + Commands::Services { zone_glob } => { let zones = Zones::load()?; // We want all logs that exist, anywhere, so we can find their @@ -197,8 +230,11 @@ fn main() -> Result<(), anyhow::Error> { // Collect a unique set of services, based on the logs in the // specified zone - let services: BTreeSet = - zones.zone_logs(&zone, filter).into_keys().collect(); + let services: BTreeSet = zones + .matching_zone_logs(&zone_glob.0, filter) + .into_iter() + .flat_map(|l| l.into_keys()) + .collect(); for svc in services { println!("{}", svc); diff --git a/dev-tools/oxlog/src/lib.rs b/dev-tools/oxlog/src/lib.rs index 6edb49a4766..584bc1331e4 100644 --- a/dev-tools/oxlog/src/lib.rs +++ b/dev-tools/oxlog/src/lib.rs @@ -8,7 +8,9 @@ use anyhow::Context; use camino::{Utf8DirEntry, Utf8Path, Utf8PathBuf}; +use glob::Pattern; use jiff::Timestamp; +use rayon::prelude::*; use std::collections::BTreeMap; use std::io; use uuid::Uuid; @@ -186,8 +188,8 @@ impl SvcLogs { /// scattered across several different directories -- and we care more /// about filename than which directory they are in. pub fn sort_by_file_name(&mut self) { - self.archived.sort_unstable_by(LogFile::file_name_cmp); - self.extra.sort_unstable_by(LogFile::file_name_cmp); + self.archived.par_sort_unstable_by(LogFile::file_name_cmp); + self.extra.par_sort_unstable_by(LogFile::file_name_cmp); } } @@ -347,9 +349,21 @@ impl Zones { } sort_logs(&mut output); - output } + + /// Return log files for all zones whose names match `zone_pattern` + pub fn matching_zone_logs( + &self, + zone_pattern: &Pattern, + filter: Filter, + ) -> Vec> { + self.zones + .par_iter() + .filter(|(zone, _)| zone_pattern.matches(zone)) + .map(|(zone, _)| self.zone_logs(zone, filter)) + .collect() + } } fn sort_logs(output: &mut BTreeMap) {