oxlog: Glob match zone names

`oxlog logs` currently requires a precise match for the zone name. This
is inconvenient when attempting to search across multiple zones with
similar names. For example, when searching all Crucible downstairs, or
using Pilot to query Nexus logs on multiple sleds.

Use the `glob` crate to take a pattern to match against the zone name
for the `services` and `logs` subcommands, potentially allowing multiple
zones to be returned. If more than one zone is matched, then the logs
and services are sorted separately.

Zones with a large number of archived logs may take an extended period
to walk and sort, and when matching against multiple zones this becomes
uncomfortably slow. To claw back some performance, use Rayon to find
files and sort in parallel. We limit the size of its thread pool to a
maximum of eight threads to avoid soaking up all available threads on a
sled. On smaller machines we fall back to the number of logical threads.

Example usage:

  $ oxlog services 'oxz_crucible_[!p]*' # All non-pantry Crucible zones
  $ oxlog logs 'oxz_nexus_*' --current  # The nexus zone

Author: wfchandler

Cargo.lock

@@ -12,7 +12,9 @@ anyhow.workspace = true
 camino.workspace = true
 chrono.workspace = true
 clap.workspace = true
+glob.workspace = true
 jiff.workspace = true
+rayon.workspace = true
 sigpipe.workspace = true
 uuid.workspace = true
 omicron-workspace-hack.workspace = true

dev-tools/oxlog/Cargo.toml

@@ -5,11 +5,18 @@
 //! Tool for discovering oxide related logfiles on sleds
 
 use clap::{ArgAction, Args, Parser, Subcommand};
+use glob::Pattern;
 use jiff::civil::DateTime;
 use jiff::tz::TimeZone;
 use jiff::{Span, Timestamp};
 use oxlog::{DateRange, Filter, LogFile, Zones};
 use std::collections::BTreeSet;
+use std::num::NonZeroUsize;
+use std::str::FromStr;
+
+/// The number of threads to given to the Rayon thread pool.
+/// Use this limit to avoid consuming excessive CPU time on a sled.
+const MAX_THREADS: usize = 8;
 
 #[derive(Debug, Parser)]
 #[command(version)]
@@ -25,8 +32,8 @@ enum Commands {
 
     /// List logs for a given service
     Logs {
-        /// The name of the zone
-        zone: String,
+        /// The glob pattern to match against zone names
+        zone_glob: GlobPattern,
 
         /// The name of the service to list logs for
         service: Option<String>,
@@ -49,15 +56,26 @@ enum Commands {
         after: Option<Timestamp>,
     },
 
-    /// List the names of all services in a zone, from the perspective of oxlog.
+    /// List the names of all services in matching zones, from the perspective of oxlog.
     /// Use these names with `oxlog logs` to filter output to logs from a
     /// specific service.
     Services {
-        /// The name of the zone
-        zone: String,
+        /// The glob pattern to match against zone names
+        zone_glob: GlobPattern,
     },
 }
 
+#[derive(Clone, Debug)]
+struct GlobPattern(Pattern);
+
+impl FromStr for GlobPattern {
+    type Err = glob::PatternError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Pattern::new(s).map(GlobPattern)
+    }
+}
+
 #[derive(Args, Debug)]
 #[group(required = true, multiple = true)]
 struct FilterArgs {
@@ -109,6 +127,12 @@ fn parse_timestamp(
 fn main() -> Result<(), anyhow::Error> {
     sigpipe::reset();
 
+    let num_threads = std::thread::available_parallelism()
+        .map(NonZeroUsize::get)
+        .unwrap_or(MAX_THREADS)
+        .min(MAX_THREADS);
+    rayon::ThreadPoolBuilder::new().num_threads(num_threads).build_global()?;
+
     let cli = Cli::parse();
 
     match cli.command {
@@ -118,7 +142,14 @@ fn main() -> Result<(), anyhow::Error> {
             }
             Ok(())
         }
-        Commands::Logs { zone, service, metadata, filter, before, after } => {
+        Commands::Logs {
+            zone_glob,
+            service,
+            metadata,
+            filter,
+            before,
+            after,
+        } => {
             let zones = Zones::load()?;
             let date_range = match (before, after) {
                 (None, None) => None,
@@ -145,44 +176,46 @@ fn main() -> Result<(), anyhow::Error> {
                 );
             };
 
-            let logs = zones.zone_logs(&zone, filter);
-            for (svc_name, svc_logs) in logs {
-                if let Some(service) = &service {
-                    if svc_name != service.as_str() {
-                        continue;
+            let zones = zones.zone_logs(&zone_glob.0, filter);
+            for logs in zones {
+                for (svc_name, svc_logs) in logs {
+                    if let Some(service) = &service {
+                        if svc_name != service.as_str() {
+                            continue;
+                        }
                     }
-                }
-                if filter.current {
-                    if let Some(current) = &svc_logs.current {
-                        if metadata {
-                            print_metadata(current);
-                        } else {
-                            println!("{}", current.path);
+                    if filter.current {
+                        if let Some(current) = &svc_logs.current {
+                            if metadata {
+                                print_metadata(current);
+                            } else {
+                                println!("{}", current.path);
+                            }
                         }
                     }
-                }
-                if filter.archived {
-                    for f in &svc_logs.archived {
-                        if metadata {
-                            print_metadata(f);
-                        } else {
-                            println!("{}", f.path);
+                    if filter.archived {
+                        for f in &svc_logs.archived {
+                            if metadata {
+                                print_metadata(f);
+                            } else {
+                                println!("{}", f.path);
+                            }
                         }
                     }
-                }
-                if filter.extra {
-                    for f in &svc_logs.extra {
-                        if metadata {
-                            print_metadata(f);
-                        } else {
-                            println!("{}", f.path);
+                    if filter.extra {
+                        for f in &svc_logs.extra {
+                            if metadata {
+                                print_metadata(f);
+                            } else {
+                                println!("{}", f.path);
+                            }
                         }
                     }
                 }
             }
             Ok(())
         }
-        Commands::Services { zone } => {
+        Commands::Services { zone_glob } => {
             let zones = Zones::load()?;
 
             // We want all logs that exist, anywhere, so we can find their
@@ -197,8 +230,11 @@ fn main() -> Result<(), anyhow::Error> {
 
             // Collect a unique set of services, based on the logs in the
             // specified zone
-            let services: BTreeSet<String> =
-                zones.zone_logs(&zone, filter).into_keys().collect();
+            let services: BTreeSet<String> = zones
+                .zone_logs(&zone_glob.0, filter)
+                .into_iter()
+                .flat_map(|l| l.into_keys())
+                .collect();
 
             for svc in services {
                 println!("{}", svc);

dev-tools/oxlog/src/bin/oxlog.rs

@@ -8,7 +8,9 @@
 
 use anyhow::Context;
 use camino::{Utf8DirEntry, Utf8Path, Utf8PathBuf};
+use glob::Pattern;
 use jiff::Timestamp;
+use rayon::prelude::*;
 use std::collections::BTreeMap;
 use std::io;
 use uuid::Uuid;
@@ -186,8 +188,8 @@ impl SvcLogs {
     /// scattered across several different directories -- and we care more
     /// about filename than which directory they are in.
     pub fn sort_by_file_name(&mut self) {
-        self.archived.sort_unstable_by(LogFile::file_name_cmp);
-        self.extra.sort_unstable_by(LogFile::file_name_cmp);
+        self.archived.par_sort_unstable_by(LogFile::file_name_cmp);
+        self.extra.par_sort_unstable_by(LogFile::file_name_cmp);
     }
 }
 
@@ -302,51 +304,56 @@ impl Zones {
         Ok(Zones { zones })
     }
 
-    /// Return log files organized by service name
+    /// Return log files for all matching zone names organized by service name
     pub fn zone_logs(
         &self,
-        zone: &str,
+        zone_pattern: &Pattern,
         filter: Filter,
-    ) -> BTreeMap<ServiceName, SvcLogs> {
-        let mut output = BTreeMap::new();
-        let Some(paths) = self.zones.get(zone) else {
-            return BTreeMap::new();
-        };
-        // Some rotated files exist in `paths.primary` that we track as
-        // 'archived'. These files have not yet been migrated into the debug
-        // directory.
-        if filter.current || filter.archived {
-            load_svc_logs(
-                paths.primary.clone(),
-                &mut output,
-                filter.show_empty,
-                filter.date_range,
-            );
-        }
+    ) -> Vec<BTreeMap<ServiceName, SvcLogs>> {
+        let output: Vec<BTreeMap<String, SvcLogs>> = self
+            .zones
+            .par_iter()
+            .filter(|(zone, _)| zone_pattern.matches(zone))
+            .filter_map(|(_, paths)| {
+                let mut local = BTreeMap::new();
+                // Some rotated files exist in `paths.primary` that we track as
+                // 'archived'. These files have not yet been migrated into the debug
+                // directory.
+                if filter.current || filter.archived {
+                    load_svc_logs(
+                        paths.primary.clone(),
+                        &mut local,
+                        filter.show_empty,
+                        filter.date_range,
+                    );
+                }
 
-        if filter.archived {
-            for dir in paths.debug.clone() {
-                load_svc_logs(
-                    dir,
-                    &mut output,
-                    filter.show_empty,
-                    filter.date_range,
-                );
-            }
-        }
-        if filter.extra {
-            for (svc_name, dir) in paths.extra.clone() {
-                load_extra_logs(
-                    dir,
-                    svc_name,
-                    &mut output,
-                    filter.show_empty,
-                    filter.date_range,
-                );
-            }
-        }
+                if filter.archived {
+                    for dir in paths.debug.clone() {
+                        load_svc_logs(
+                            dir,
+                            &mut local,
+                            filter.show_empty,
+                            filter.date_range,
+                        );
+                    }
+                }
+                if filter.extra {
+                    for (svc_name, dir) in paths.extra.clone() {
+                        load_extra_logs(
+                            dir,
+                            svc_name,
+                            &mut local,
+                            filter.show_empty,
+                            filter.date_range,
+                        );
+                    }
+                }
 
-        sort_logs(&mut output);
+                sort_logs(&mut local);
+                Some(local)
+            })
+            .collect();
 
         output
     }